04_parallel_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <font color=\"red\"> *Exercise*: starting ipyparallel </font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternative location for configuration files on Windows: `%PROGRAMDATA%\\jupyter\\`"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <font color=\"blue\"> Demo, direct interface </blue>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import ipyparallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ktw/anaconda2/lib/python2.7/site-packages/ipyparallel/client/client.py:442: RuntimeWarning: \n",
      "            Controller appears to be listening on localhost, but not on this machine.\n",
      "            If this is true, you should specify Client(...,sshserver='you@130.229.168.241')\n",
      "            or instruct your controller to listen on an external IP.\n",
      "  RuntimeWarning)\n"
     ]
    }
   ],
   "source": [
    "rc = ipyparallel.Client()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The `ids` attribute of Client instance shows identifiers of engines that IPython detected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rc.ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<DirectView [0, 1, 2, 3]>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dview = rc[:]\n",
    "dview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<DirectView all>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dview2 = rc.direct_view()\n",
    "dview2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "rc.direct_view?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <font color=\"blue\"> Demo, %px magic </blue>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%px?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%px import os, time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] 1481548185.94\n",
      "[stdout:1] 1481548185.94\n",
      "[stdout:2] 1481548185.95\n",
      "[stdout:3] 1481548185.95\n"
     ]
    }
   ],
   "source": [
    "%px print(time.time())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%px a = os.getpid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] 74595\n",
      "[stdout:1] 74596\n",
      "[stdout:2] 74598\n",
      "[stdout:3] 74597\n"
     ]
    }
   ],
   "source": [
    "%px print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] 74595\n",
      "[stdout:1] 74596\n",
      "[stdout:2] 74598\n",
      "[stdout:3] 74597\n"
     ]
    }
   ],
   "source": [
    "%px print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] 74595\n",
      "[stdout:1] 74596\n",
      "[stdout:2] 74598\n",
      "[stdout:3] 74597\n"
     ]
    }
   ],
   "source": [
    "%pxresult"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[stdout:0] 74595\n",
      "[stdout:1] 74596\n",
      "[stdout:2] 74598\n"
     ]
    }
   ],
   "source": [
    "%%px --targets :-1\n",
    "    print(os.getpid())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <font color=\"red\"> *Exercise: Estimating $\\pi$ with parallel Monte Carlo* </font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import ipyparallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 3]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clients = ipyparallel.Client()\n",
    "clients.ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<DirectView [0, 1, 2, 3]>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dview = clients[:]\n",
    "dview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def mc_pi(n):\n",
    "#    import numpy as np\n",
    "    count = 0\n",
    "    p = np.random.rand(n,2)\n",
    "    for i in range(n):\n",
    "        x = p[i,0]\n",
    "        y = p[i,1]\n",
    "        if (x**2 + y**2) <= 1.0:\n",
    "            count += 1\n",
    "    return float(count)/float(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.140236\n",
      "3.140232\n",
      "3.143732\n",
      "3.142344\n",
      "1 loop, best of 3: 1.25 s per loop\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "est_pi = mc_pi(10**6)\n",
    "print(est_pi*4) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "May need to import modules on the engines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AsyncResult: execute>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dview.execute(\"import numpy as np\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.572378\n",
      "1.571444\n",
      "1.570164\n",
      "1.570518\n",
      "1 loop, best of 3: 650 ms per loop\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "results = dview.map_sync(mc_pi, [500000]*2)\n",
    "print(np.sum(results))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Speed up of factor 2 on my mac, since only two physical cores (with hyperthreading enabled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}