IPython Parallel Push/Execute/Pull Demo

ipy_parallel_push_exec_pull.ipynb

{
 "metadata": {
  "name": "ipy-parallel-push-exec-pull"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": "IPython Parallel Push/Execute/Pull Demo"
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": "This notebook performs a very simple dot product of the same two random (square) matrices on all ipengines. It then gathers the results from all ipengines and confirms that all results are the same. This mostly demos how the push, execute, and pull pipeline works in the IPython Parallel client."
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "1. Allocate random matrices locally"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "import numpy as np",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 113
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "N = 2000",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 114
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "a = np.random.randn(N, N)",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 115
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "b = np.random.randn(N, N)",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 116
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "2. Compute their dot product locally"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "c = np.dot(a,b)",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 117
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "3. Initialize the IPython Parallel Client"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "from IPython.parallel import Client",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 118
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "cl = Client()",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 119
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "cl.ids",
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 120,
       "text": "[0, 1, 2]"
      }
     ],
     "prompt_number": 120
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "4. Fetch a 'view' from the IPython parallel client"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "v = cl[:]",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 121
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "5. Import NumPy on all IPEngines"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "v.execute('import numpy as np', block=True)",
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "6. Push both random matrices to all IPEngines"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "v.push(dict(a=a, b=b), block=True)",
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 122,
       "text": "[None, None, None]"
      }
     ],
     "prompt_number": 122
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "7. Compute dot product on all IPEngines"
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": "NOTE: We add 'print c' to the execute statement below in order to demo the display_outputs() function which shows the stdout, if any, of all remote engines as a result of executing the command. In general you do not need to print variables in order to interact with them."
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "r = v.execute('c = np.dot(a,b); print c', block=True)",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 123
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "r.display_outputs()",
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": "[stdout:0] \n[[  -4.67202217  104.39252912  -78.37783997 ...,  -29.30709274\n     6.66403977   29.92526411]\n [  39.22517891   73.37959579 -140.66938625 ...,  -30.40772693\n     9.21601775   29.80644042]\n [  31.60094895  -26.90207753   42.07903369 ...,    6.16564342  -40.9135798\n   -57.82278587]\n ..., \n [ -25.57059809  -63.64479316   22.76541772 ...,    1.68819694  -12.9543937\n   -61.07450388]\n [  15.3646617    57.48457183  -34.15856279 ...,    7.34530517\n   -16.26106107  -17.58811896]\n [ -13.26202943   75.42065094   74.86911918 ...,   -0.55862744\n     6.72326022   10.38620045]]\n[stdout:1] \n[[  -4.67202217  104.39252912  -78.37783997 ...,  -29.30709274\n     6.66403977   29.92526411]\n [  39.22517891   73.37959579 -140.66938625 ...,  -30.40772693\n     9.21601775   29.80644042]\n [  31.60094895  -26.90207753   42.07903369 ...,    6.16564342  -40.9135798\n   -57.82278587]\n ..., \n [ -25.57059809  -63.64479316   22.76541772 ...,    1.68819694  -12.9543937\n   -61.07450388]\n [  15.3646617    57.48457183  -34.15856279 ...,    7.34530517\n   -16.26106107  -17.58811896]\n [ -13.26202943   75.42065094   74.86911918 ...,   -0.55862744\n     6.72326022   10.38620045]]\n[stdout:2] \n[[  -4.67202217  104.39252912  -78.37783997 ...,  -29.30709274\n     6.66403977   29.92526411]\n [  39.22517891   73.37959579 -140.66938625 ...,  -30.40772693\n     9.21601775   29.80644042]\n [  31.60094895  -26.90207753   42.07903369 ...,    6.16564342  -40.9135798\n   -57.82278587]\n ..., \n [ -25.57059809  -63.64479316   22.76541772 ...,    1.68819694  -12.9543937\n   -61.07450388]\n [  15.3646617    57.48457183  -34.15856279 ...,    7.34530517\n   -16.26106107  -17.58811896]\n [ -13.26202943   75.42065094   74.86911918 ...,   -0.55862744\n     6.72326022   10.38620045]]\n"
      }
     ],
     "prompt_number": 124
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "8. Pull the results from all IPEngines"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "dot = v.pull('c', block=True)",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 125
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "print dot",
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": "[array([[  -4.67202217,  104.39252912,  -78.37783997, ...,  -29.30709274,\n           6.66403977,   29.92526411],\n       [  39.22517891,   73.37959579, -140.66938625, ...,  -30.40772693,\n           9.21601775,   29.80644042],\n       [  31.60094895,  -26.90207753,   42.07903369, ...,    6.16564342,\n         -40.9135798 ,  -57.82278587],\n       ..., \n       [ -25.57059809,  -63.64479316,   22.76541772, ...,    1.68819694,\n         -12.9543937 ,  -61.07450388],\n       [  15.3646617 ,   57.48457183,  -34.15856279, ...,    7.34530517,\n         -16.26106107,  -17.58811896],\n       [ -13.26202943,   75.42065094,   74.86911918, ...,   -0.55862744,\n           6.72326022,   10.38620045]]), array([[  -4.67202217,  104.39252912,  -78.37783997, ...,  -29.30709274,\n           6.66403977,   29.92526411],\n       [  39.22517891,   73.37959579, -140.66938625, ...,  -30.40772693,\n           9.21601775,   29.80644042],\n       [  31.60094895,  -26.90207753,   42.07903369, ...,    6.16564342,\n         -40.9135798 ,  -57.82278587],\n       ..., \n       [ -25.57059809,  -63.64479316,   22.76541772, ...,    1.68819694,\n         -12.9543937 ,  -61.07450388],\n       [  15.3646617 ,   57.48457183,  -34.15856279, ...,    7.34530517,\n         -16.26106107,  -17.58811896],\n       [ -13.26202943,   75.42065094,   74.86911918, ...,   -0.55862744,\n           6.72326022,   10.38620045]]), array([[  -4.67202217,  104.39252912,  -78.37783997, ...,  -29.30709274,\n           6.66403977,   29.92526411],\n       [  39.22517891,   73.37959579, -140.66938625, ...,  -30.40772693,\n           9.21601775,   29.80644042],\n       [  31.60094895,  -26.90207753,   42.07903369, ...,    6.16564342,\n         -40.9135798 ,  -57.82278587],\n       ..., \n       [ -25.57059809,  -63.64479316,   22.76541772, ...,    1.68819694,\n         -12.9543937 ,  -61.07450388],\n       [  15.3646617 ,   57.48457183,  -34.15856279, ...,    7.34530517,\n         -16.26106107,  -17.58811896],\n       [ -13.26202943,   75.42065094,   74.86911918, ...,   -0.55862744,\n           6.72326022,   10.38620045]])]\n"
      }
     ],
     "prompt_number": 126
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": "9. Check remote results against local results"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "assert np.allclose(c, dot[0])",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 127
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "assert np.allclose(dot[0], dot[1])",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 128
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": "assert np.allclose(dot[1], dot[2])",
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 129
    }
   ],
   "metadata": {}
  }
 ]
}