Skip to content

Instantly share code, notes, and snippets.

@minrk

minrk/Load-balancing methods.ipynb

Created October 17, 2013 19:04
Show Gist options
  • Save minrk/7030408 to your computer and use it in GitHub Desktop.
Save minrk/7030408 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
Load-balancing methods.ipynb
{
"metadata": {
"name": ""
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Using methods on remote objects as task functions"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from IPython import parallel\n",
"rc = parallel.Client()\n",
"ref = parallel.Reference"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 23
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%precision 2"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 24,
"text": [
"u'%.2f'"
]
}
],
"prompt_number": 24
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"rc.ids\n",
"dview = rc[:]\n",
"dview"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 25,
"text": [
"<DirectView [0, 1, 2, 3,...]>"
]
}
],
"prompt_number": 25
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"dview.scatter('rank', rc.ids, flatten=True)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 26,
"text": [
"<AsyncResult: scatter>"
]
}
],
"prompt_number": 26
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"%%px\n",
"import numpy as np\n",
"A = np.random.random((rank+1,100))\n",
"B = np.random.random((100,rank+1))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 27
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"lview = rc.load_balanced_view()\n",
"ref = parallel.Reference"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 28
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"You can use `parallel.Reference` to refer to the task function and/or any of its arguments\n",
"that should be loaded from the remote namespace."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"C_async = lview.apply(ref('A.dot'), ref('B'))\n",
"C_async"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 29,
"text": [
"<AsyncResult: A.dot>"
]
}
],
"prompt_number": 29
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"C_async.get()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 30,
"text": [
"array([[ 23.86, 26.7 , 27.13, 23.23, 26.71, 25.79],\n",
" [ 23.68, 25.63, 26.76, 21.04, 26.23, 23.78],\n",
" [ 23.53, 24.25, 26.79, 21.83, 24.83, 23.12],\n",
" [ 24.2 , 27.3 , 24.49, 20.01, 24.84, 23.64],\n",
" [ 25.97, 28.96, 26. , 24.07, 27.9 , 25.87],\n",
" [ 25.77, 25.53, 25.47, 22.08, 26.1 , 25.75]])"
]
}
],
"prompt_number": 30
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from IPython.display import display\n",
"\n",
"async_dots = [ lview.apply(ref('A.dot'), ref('B')) for id in rc.ids ]\n",
"for ar in async_dots:\n",
" C = ar.get()\n",
" print \"A.dot(B) on engine %s:\" % ar.engine_id\n",
" print C.shape\n",
" print C"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"A.dot(B) on engine 6:\n",
"(7, 7)\n",
"[[ 25.08 24.8 29.35 27.23 24.49 24.09 26.03]\n",
" [ 26.54 26.62 29.57 29.18 26.74 26.19 25.74]\n",
" [ 25.34 24.96 27.09 26.84 25.74 23.97 23.97]\n",
" [ 23. 23.16 25.79 25.14 24.54 23.96 23.52]\n",
" [ 25.53 26.05 27.65 27.44 27.65 25.58 24.86]\n",
" [ 21.61 23.5 24.21 25.27 23.6 21.76 21.81]\n",
" [ 23.72 23.13 25.99 22.32 23.04 21.49 22.98]]\n",
"A.dot(B) on engine 4:\n",
"(5, 5)\n",
"[[ 20.04 21.75 23.82 18.53 23.31]\n",
" [ 25. 25.96 27.3 23.83 26.82]\n",
" [ 20.85 24.51 24.28 21.6 22.52]\n",
" [ 21.82 22.95 23.72 21.93 23.16]\n",
" [ 23.79 24.13 26.42 24.55 26.46]]\n",
"A.dot(B) on engine 3:\n",
"(4, 4)\n",
"[[ 21.25 24.62 24.64 25.93]\n",
" [ 25.83 28. 24.42 27.74]\n",
" [ 24.81 24.13 23.41 26.73]\n",
" [ 23.04 24.79 22.56 27.36]]\n",
"A.dot(B) on engine 2:\n",
"(3, 3)\n",
"[[ 25.91 26.31 24.97]\n",
" [ 24.06 24.77 24.61]\n",
" [ 25.55 25.38 25.31]]\n",
"A.dot(B) on engine 1:\n",
"(2, 2)\n",
"[[ 23.97 27.08]\n",
" [ 25.86 25.33]]\n",
"A.dot(B) on engine 0:\n",
"(1, 1)\n",
"[[ 23.53]]\n",
"A.dot(B) on engine 7:\n",
"(8, 8)\n",
"[[ 23.05 24.24 24.83 25.14 23.69 27.05 23.99 23.09]\n",
" [ 27.54 25.62 29.51 27.88 27.66 31.78 29.02 26.88]\n",
" [ 27.86 24.5 27.1 25.37 25.34 30.25 25.3 24.9 ]\n",
" [ 22.33 21.18 24.85 22.48 23.36 24.74 24.24 23.68]\n",
" [ 27.11 26.53 26.65 26.48 24.81 28.17 27.07 25.46]\n",
" [ 24.47 23.89 25.77 25.08 24.72 30.19 24.37 25.16]\n",
" [ 26. 24.8 26.19 27.6 25.6 29.89 25.47 24.5 ]\n",
" [ 21.97 21.59 24.88 22.72 22.52 24.59 24.3 22.05]]\n",
"A.dot(B) on engine 5:\n",
"(6, 6)\n",
"[[ 23.86 26.7 27.13 23.23 26.71 25.79]\n",
" [ 23.68 25.63 26.76 21.04 26.23 23.78]\n",
" [ 23.53 24.25 26.79 21.83 24.83 23.12]\n",
" [ 24.2 27.3 24.49 20.01 24.84 23.64]\n",
" [ 25.97 28.96 26. 24.07 27.9 25.87]\n",
" [ 25.77 25.53 25.47 22.08 26.1 25.75]]\n"
]
}
],
"prompt_number": 31
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Or you can just rely on module globals, which resolves to `__main__`,\n",
"the user namespace."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"async_dots = [ lview.apply_async(lambda : A.dot(B)) for id in rc.ids ]\n",
"for ar in async_dots:\n",
" C = ar.get()\n",
" print \"A.dot(B) on engine %s:\" % ar.engine_id\n",
" print C.shape\n",
" print C"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"A.dot(B) on engine 6:\n",
"(7, 7)\n",
"[[ 25.08 24.8 29.35 27.23 24.49 24.09 26.03]\n",
" [ 26.54 26.62 29.57 29.18 26.74 26.19 25.74]\n",
" [ 25.34 24.96 27.09 26.84 25.74 23.97 23.97]\n",
" [ 23. 23.16 25.79 25.14 24.54 23.96 23.52]\n",
" [ 25.53 26.05 27.65 27.44 27.65 25.58 24.86]\n",
" [ 21.61 23.5 24.21 25.27 23.6 21.76 21.81]\n",
" [ 23.72 23.13 25.99 22.32 23.04 21.49 22.98]]\n",
"A.dot(B) on engine 4:\n",
"(5, 5)\n",
"[[ 20.04 21.75 23.82 18.53 23.31]\n",
" [ 25. 25.96 27.3 23.83 26.82]\n",
" [ 20.85 24.51 24.28 21.6 22.52]\n",
" [ 21.82 22.95 23.72 21.93 23.16]\n",
" [ 23.79 24.13 26.42 24.55 26.46]]\n",
"A.dot(B) on engine 3:\n",
"(4, 4)\n",
"[[ 21.25 24.62 24.64 25.93]\n",
" [ 25.83 28. 24.42 27.74]\n",
" [ 24.81 24.13 23.41 26.73]\n",
" [ 23.04 24.79 22.56 27.36]]\n",
"A.dot(B) on engine 2:\n",
"(3, 3)\n",
"[[ 25.91 26.31 24.97]\n",
" [ 24.06 24.77 24.61]\n",
" [ 25.55 25.38 25.31]]\n",
"A.dot(B) on engine 1:\n",
"(2, 2)\n",
"[[ 23.97 27.08]\n",
" [ 25.86 25.33]]\n",
"A.dot(B) on engine 0:\n",
"(1, 1)\n",
"[[ 23.53]]\n",
"A.dot(B) on engine 7:\n",
"(8, 8)\n",
"[[ 23.05 24.24 24.83 25.14 23.69 27.05 23.99 23.09]\n",
" [ 27.54 25.62 29.51 27.88 27.66 31.78 29.02 26.88]\n",
" [ 27.86 24.5 27.1 25.37 25.34 30.25 25.3 24.9 ]\n",
" [ 22.33 21.18 24.85 22.48 23.36 24.74 24.24 23.68]\n",
" [ 27.11 26.53 26.65 26.48 24.81 28.17 27.07 25.46]\n",
" [ 24.47 23.89 25.77 25.08 24.72 30.19 24.37 25.16]\n",
" [ 26. 24.8 26.19 27.6 25.6 29.89 25.47 24.5 ]\n",
" [ 21.97 21.59 24.88 22.72 22.52 24.59 24.3 22.05]]\n",
"A.dot(B) on engine 5:\n",
"(6, 6)\n",
"[[ 23.86 26.7 27.13 23.23 26.71 25.79]\n",
" [ 23.68 25.63 26.76 21.04 26.23 23.78]\n",
" [ 23.53 24.25 26.79 21.83 24.83 23.12]\n",
" [ 24.2 27.3 24.49 20.01 24.84 23.64]\n",
" [ 25.97 28.96 26. 24.07 27.9 25.87]\n",
" [ 25.77 25.53 25.47 22.08 26.1 25.75]]\n"
]
}
],
"prompt_number": 32
}
],
"metadata": {}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment