launchers.ipynb

{
 "metadata": {
  "name": "launchers"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Set up a logger, so we can actually see the output (this is unnecessary, but helpful for debugging):"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import logging\n",
      "logger = logging.Logger('ipcluster')\n",
      "logger.setLevel(logging.DEBUG)\n",
      "logger.addHandler(logging.StreamHandler(sys.stdout))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Get some config that the launchers expect from ipcluster:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ip = get_ipython()\n",
      "config = ip.config\n",
      "profile_dir = ip.profile_dir.location # aka ipython_dir/profile_{name}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Import the Launcher classes used for starting controller/engine subprocesses"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from IPython.parallel.apps.launcher import LocalControllerLauncher, LocalEngineLauncher"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "There is also a LocalEngineSetLauncher class that allows you to start/stop a group of engines at a time"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Define some functions for simple start/stop of engines"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import time\n",
      "\n",
      "engines = []\n",
      "\n",
      "def add_engines(n, delay=0.1):\n",
      "    \"\"\"start n engines\"\"\"\n",
      "    for i in range(n):\n",
      "        e = LocalEngineLauncher(config=config, log=logger, profile_dir=profile_dir)\n",
      "        engines.append(e)\n",
      "        e.start()\n",
      "        if i + 1 < n:\n",
      "            time.sleep(delay)\n",
      "\n",
      "def cleanup_engines():\n",
      "    [ e.stop() for e in engines ]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Start the Controller:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "controller = LocalControllerLauncher(config=config, log=logger, profile_dir=profile_dir)\n",
      "controller.start()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalControllerLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipcontrollerapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44636\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2012-09-27 16:38:23,506.506 [IPControllerApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default'\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2012-09-27 16:38:23.830 [scheduler] Scheduler started [leastload]\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Start 5 engines:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "add_engines(5)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44641\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44642\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44643\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44644\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44645\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2012-09-27 16:38:27,935.935 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default'\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2012-09-27 16:38:27,932.932 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default'\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2012-09-27 16:38:27,997.997 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default'\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2012-09-27 16:38:27,998.998 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default'\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2012-09-27 16:38:28,017.017 [IPEngineApp] Using existing profile dir: u'/Users/minrk/.ipython/profile_default'\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Now we can work with the engines in the usual way:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from IPython import parallel\n",
      "rc = parallel.Client()\n",
      "rc.ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 10,
       "text": [
        "[0, 1, 2, 3, 4]"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rc[:]['a'] = 5\n",
      "rc[:]['a']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 11,
       "text": [
        "[5, 5, 5, 5, 5]"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "And we can stop them as well:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "engines[0].stop()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44641, 'exit_code': 0}\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rc.ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 13,
       "text": [
        "[0, 1, 3, 4]"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "And add some more"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "add_engines(3)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44676\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44677\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Starting LocalEngineLauncher: ['/usr/bin/python', '-c', 'from IPython.parallel.apps.ipengineapp import launch_new_instance; launch_new_instance()', '--profile-dir', u'/Users/minrk/.ipython/profile_default', '--cluster-id', u'', '--log-to-file', '--log-level=20']\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' started: 44678\n"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rc.ids"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 17,
       "text": [
        "[0, 1, 3, 4, 5, 6, 7]"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "And cleanup the whole cluster:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rc.shutdown(hub=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2012-09-27 16:39:05.590 [scheduler] Unhandled message type: u'shutdown_notice'\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44645, 'exit_code': 0}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44642, 'exit_code': 0}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44676, 'exit_code': 0}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44678, 'exit_code': 0}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44643, 'exit_code': 0}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44677, 'exit_code': 0}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44644, 'exit_code': 0}\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Process '/usr/bin/python' stopped: {'pid': 44636, 'exit_code': 0}\n"
       ]
      }
     ],
     "prompt_number": 18
    }
   ],
   "metadata": {}
  }
 ]
}