Created
January 6, 2013 20:49
-
-
Save minrk/4470122 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"metadata": { | |
"name": "SO14184621" | |
}, | |
"nbformat": 3, | |
"nbformat_minor": 0, | |
"worksheets": [ | |
{ | |
"cells": [ | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"For SO Question [14184621](http://stackoverflow.com/questions/14184621)" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Connect to the cluster" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"from IPython.parallel import Client\n", | |
"rc = Client()\n", | |
"lview = rc.load_balanced_view()\n", | |
"dv = rc[:]" | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [], | |
"prompt_number": 1 | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Define a magic that runs a cell locally *and* remotely" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"def pxlocal(line, cell):\n", | |
" ip = get_ipython()\n", | |
" ip.run_cell_magic(\"px\", line, cell)\n", | |
" ip.run_cell(cell)\n", | |
"ip.register_magic_function(pxlocal, \"cell\") " | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [], | |
"prompt_number": 2 | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Use the magic to define a class interactively everywhere (including local)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"%%pxlocal\n", | |
"class MyClass(object):\n", | |
" def __init__(self, parameter):\n", | |
" self.parameter = parameter\n", | |
"\n", | |
" def update_something(self, some_data):\n", | |
" # do something smart here with some_data & internal state\n", | |
" self.parameter = some_data\n", | |
" \n", | |
" def compute_something(self, other_data):\n", | |
" # do something smart here with other data & internal state\n", | |
" return self.parameter * other_data" | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [], | |
"prompt_number": 3 | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"Run some tasks" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"def process(obj, some_data, other_data):\n", | |
" obj.update_something(some_data)\n", | |
" return obj.compute_something(other_data)\n", | |
"\n", | |
"tasks = []\n", | |
"some_instances = [MyClass(i) for i in range(5)]\n", | |
"data_source_1 = range(1,4)\n", | |
"data_source_2 = range(4,1,-1)\n", | |
"for obj in some_instances:\n", | |
" for some_data in data_source_1:\n", | |
" for other_data in data_source_2:\n", | |
" ar = lview.apply_async(process, obj, some_data, other_data)\n", | |
" tasks.append(ar)\n", | |
"\n", | |
"# wait for computation to end\n", | |
"results = [ar.get() for ar in tasks] " | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [], | |
"prompt_number": 4 | |
}, | |
{ | |
"cell_type": "code", | |
"collapsed": false, | |
"input": [ | |
"results[:18]" | |
], | |
"language": "python", | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"output_type": "pyout", | |
"prompt_number": 5, | |
"text": [ | |
"[4, 3, 2, 8, 6, 4, 12, 9, 6, 4, 3, 2, 8, 6, 4, 12, 9, 6]" | |
] | |
} | |
], | |
"prompt_number": 5 | |
} | |
], | |
"metadata": {} | |
} | |
] | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment