Parallel stdout.ipynb

{
 "metadata": {
  "gist_id": "6171348",
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Watching stdout of an AsyncResult as it arrives"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from IPython import parallel\n",
      "rc = parallel.Client()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def print_stuff(n):\n",
      "    \"\"\"dummy task that prints stuff\"\"\"\n",
      "    import os, sys\n",
      "    for i in range(n):\n",
      "        print i, os.getpid()\n",
      "        # stdout.flush is important, to ensure that stdout messages are\n",
      "        # published in a timely manner\n",
      "        sys.stdout.flush()\n",
      "        time.sleep(1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print_stuff(3)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0 99629\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1 99629\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "2 99629\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Create our view"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dview = rc[:]\n",
      "dview"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "<DirectView [0, 1, 2, 3]>"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Define a function that watches stdout as we wait for an AsyncResult to complete"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys\n",
      "import time\n",
      "from IPython.display import clear_output\n",
      "\n",
      "def wait_watching_stdout(ar, dt=1, truncate=1000):\n",
      "    while not ar.ready():\n",
      "        stdouts = ar.stdout\n",
      "        if not any(stdouts):\n",
      "            continue\n",
      "        # clear_output doesn't work in plain terminal / script environments\n",
      "        clear_output()\n",
      "        print '-' * 30\n",
      "        print \"%.3fs elapsed\" % ar.elapsed\n",
      "        print \"\"\n",
      "        for eid, stdout in zip(ar._targets, ar.stdout):\n",
      "            if stdout:\n",
      "                print \"[ stdout %2i ]\\n%s\" % (eid, stdout[-truncate:])\n",
      "        sys.stdout.flush()\n",
      "        time.sleep(dt)\n",
      "        "
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ar = dview.apply_async(print_stuff, 5)\n",
      "wait_watching_stdout(ar)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "------------------------------\n",
        "4.024s elapsed\n",
        "\n",
        "[ stdout 0 ]\n",
        "0 99588\n",
        "1 99588\n",
        "2 99588\n",
        "3 99588\n",
        "4 99588\n",
        "\n",
        "[ stdout 1 ]\n",
        "0 99587\n",
        "1 99587\n",
        "2 99587\n",
        "3 99587\n",
        "4 99587\n",
        "\n",
        "[ stdout 2 ]\n",
        "0 99589\n",
        "1 99589\n",
        "2 99589\n",
        "3 99589\n",
        "4 99589\n",
        "\n",
        "[ stdout 3 ]\n",
        "0 99594\n",
        "1 99594\n",
        "2 99594\n",
        "3 99594\n",
        "4 99594\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 12
    }
   ],
   "metadata": {}
  }
 ]
}