Skip to content

Instantly share code, notes, and snippets.

@wasade
Created March 18, 2016 21:41
Show Gist options
  • Save wasade/84446acb2f1407ca19d9 to your computer and use it in GitHub Desktop.
Save wasade/84446acb2f1407ca19d9 to your computer and use it in GitHub Desktop.
Example using joblib for running systemcalls in parallel
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"import subprocess\n",
"import functools\n",
"from IPython.core.magic import line_magic, Magics, magics_class\n",
"import joblib\n",
"\n",
"\n",
"def systemcall(to_exec):\n",
" proc = subprocess.Popen(to_exec, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)\n",
" stdout, stderr = proc.communicate()\n",
" retval = proc.returncode\n",
"\n",
" return(stdout, stderr, retval)\n",
"\n",
"@magics_class\n",
"class JobManager(Magics):\n",
" jobs = []\n",
" ncpus = 2\n",
" \n",
" def __init__(self, shell, systemcaller):\n",
" super(JobManager, self).__init__(shell)\n",
" self.systemcall = systemcaller\n",
" \n",
" @line_magic\n",
" def stage(self, line):\n",
" self.jobs.append(line)\n",
" \n",
" @line_magic\n",
" def execute(self, line):\n",
" to_run = self.jobs\n",
" self.jobs = []\n",
" \n",
" if to_run:\n",
" return joblib.Parallel(n_jobs=self.ncpus)(joblib.delayed(self.systemcall)(f) for f in to_run)\n",
" else:\n",
" return []\n",
" \n",
"ip = get_ipython()\n",
"magics = JobManager(ip, systemcall)\n",
"ip.register_magics(magics)"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"collapsed": false
},
"outputs": [
{
"data": {
"text/plain": [
"[(b'sandbar.local\\n', b'', 0),\n",
" (b'14:38 up 22:29, 7 users, load averages: 1.80 1.79 1.69\\n', b'', 0),\n",
" (b'', b'', 0),\n",
" (b'', b'/bin/sh: foobar: command not found\\n', 127)]"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"%stage hostname\n",
"%stage uptime\n",
"%stage sleep 10\n",
"%stage foobar\n",
"%execute"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"def do_something(a, b):\n",
" return a+b\n",
"\n",
"def do_something_else(x, y=None, z=10):\n",
" if y is None:\n",
" return x\n",
" else:\n",
" return x + y + z\n",
" \n",
"jm.stage('hostname')\n",
"jm.stage('uptime')\n",
"jm.stage('sleep 10')\n",
"#jm.stage(do_something, 5, 10)\n",
"#jm.stage(do_something, 10, 15)\n",
"#jm.stage(do_something_else, 10)\n",
"#jm.stage(do_something_else, 10, 20, 30)\n",
"#jm.stage(do_something_else, 10, z=20, y=30)\n",
"#jm.stage(do_something_else, 10, y=None, z=30)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"%time jm.run()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"%stage hostname\n",
"%stage uptime\n",
"%stage do_something(5, 10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false
},
"outputs": [],
"source": [
"%execute"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.5.1"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment