Interactive Classes.ipynb

{
 "metadata": {
  "gist_id": "5216756",
  "name": "Interactive Classes"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "heading",
     "level": 1,
     "metadata": {},
     "source": [
      "Working with interactively defined classes in IPython.parallel"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Create our Client"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from IPython import parallel\n",
      "client = parallel.Client()\n",
      "lbv = client.load_balanced_view()\n",
      "dview = client[:]\n",
      "dview"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 1,
       "text": [
        "<DirectView [0, 1]>"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "The problem"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Interactively defined classes can be annoying to use with IPython.parallel.\n",
      "This stems from how class instances are serialized by pickle"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# the real model object is much more involved...\n",
      "class MyModel(object):\n",
      "    def maximize(self, r):\n",
      "        return max(r, 4)\n",
      "\n",
      "def evaluate(r, model):\n",
      "    return model.maximize(r)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cPickle as pickle\n",
      "\n",
      "m = MyModel()\n",
      "print pickle.dumps(m)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "ccopy_reg\n",
        "_reconstructor\n",
        "p1\n",
        "(c__main__\n",
        "MyModel\n",
        "p2\n",
        "c__builtin__\n",
        "object\n",
        "p3\n",
        "NtRp4\n",
        ".\n"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Note how the pickled model is almost nothing but a reference to `__main__.MyModel` and `object`.\n",
      "This is how the instance will be reconstructed remotely.  But the probem is that MyModel is not defined on the engines:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = MyModel()\n",
      "lbv.apply_sync(evaluate, 2, model)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "RemoteError",
       "evalue": "AttributeError('DummyMod' object has no attribute 'MyModel')",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
        "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/kernel/zmq/serialize.pyc\u001b[0m in \u001b[0;36munpack_apply_message\u001b[1;34m(bufs, g, copy)\u001b[0m",
        "\u001b[0;32m    191\u001b[0m     \u001b[0margs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m    192\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'nargs'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;32m--> 193\u001b[1;33m         \u001b[0marg\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marg_bufs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0munserialize_object\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg_bufs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0m\u001b[0;32m    194\u001b[0m         \u001b[0margs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m    195\u001b[0m     \u001b[0margs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;32m/Users/minrk/dev/ip/mine/IPython/kernel/zmq/serialize.pyc\u001b[0m in \u001b[0;36munserialize_object\u001b[1;34m(buffers, g)\u001b[0m",
        "\u001b[0;32m    130\u001b[0m         \u001b[1;31m# a zmq message\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m    131\u001b[0m         \u001b[0mpobj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbytes\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;32m--> 132\u001b[1;33m     \u001b[0mcanned\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpobj\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0m\u001b[0;32m    133\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mistype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcanned\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msequence_types\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcanned\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m<\u001b[0m \u001b[0mMAX_ITEMS\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m    134\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcanned\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;31mAttributeError\u001b[0m: 'DummyMod' object has no attribute 'MyModel'"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "This is a common problem with relying on interactively defined names,\n",
      "but there are issues peculiar to classes, as opposed to locally defined functions."
     ]
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "The old way"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "In IPython 0.13, there are a few shortcomings:\n",
      "    \n",
      "1. Classes cannot be pushed, unlike functions.\n",
      "2. There is no way to use `%%px` to define a class *both* locally and remotely.\n",
      "\n",
      "So defining a class everywhere is a two-step process, either manual:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class MyModel1(object):\n",
      "    def maximize(self, r):\n",
      "        return max(r, 0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%px\n",
      "class MyModel1(object):\n",
      "    def maximize(self, r):\n",
      "        return max(r, 0)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = MyModel1()\n",
      "lbv.apply_sync(evaluate, 1, model)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 21,
       "text": [
        "1"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Or more automatic:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class MyModel2(object):\n",
      "    def maximize(self, r):\n",
      "        return max(r, 2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# this just executes the previous cell on all our engines\n",
      "# (note that In[-1] is *this* cell, so In[-2] is the previous one)\n",
      "dview.execute(In[-2], block=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 23,
       "text": [
        "<AsyncResult: finished>"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = MyModel2()\n",
      "lbv.apply_sync(evaluate, 1, model)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "2"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "The new way"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "In IPython 1.0, this has been alleviated to some degree, by making interactively defined classes pushable,\n",
      "and adding `%%px --local`.\n",
      "\n",
      "So now, you can use the same push step that is commonly used when you have multi-function tasks:"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class MyModel3(object):\n",
      "    def maximize(self, r):\n",
      "        return max(r, 3)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dview['MyModel3'] = MyModel3"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = MyModel3()\n",
      "lbv.apply_sync(evaluate, 0, model)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 27,
       "text": [
        "3"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Or you can simultaneously define the class both locally and remotely with `%%px --local`"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%%px --local\n",
      "# the real model object is much more involved...\n",
      "class MyModel4(object):\n",
      "    def maximize(self, r):\n",
      "        return max(r, 4)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = MyModel4()\n",
      "lbv.apply_sync(evaluate, 2, model)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 29,
       "text": [
        "4"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "heading",
     "level": 2,
     "metadata": {},
     "source": [
      "The new new way"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Now, if you really want bleeding edge, there is an [outstanding Pull Request](https://github.com/ipython/ipython/pull/3041)\n",
      "that lets you define dependencies for locally defined names,\n",
      "which will actually attach the local objects to the task,\n",
      "so they are guaranteed to be defined when the task runs"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class MyModel5(object):\n",
      "    def maximize(self, r):\n",
      "        return max(r, 5)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "@parallel.require(MyModel5)\n",
      "def evaluate(r, model):\n",
      "    return model.maximize(r)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = MyModel5()\n",
      "lbv.apply_sync(evaluate, 2, model)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "5"
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "<style>\n",
      "/* for presentations\n",
      "\n",
      "body {\n",
      "  font-size: 18px;\n",
      "}\n",
      "div.list_item {\n",
      "  height: 30px;\n",
      "}\n",
      ".toolbar select {\n",
      "  height: 30px;\n",
      "}\n",
      "\n",
      "/* the real stuff */\n",
      "\n",
      "div.cell.selected {\n",
      "  border: 1px solid white;\n",
      "  background-image: -webkit-gradient(\n",
      "    linear,\n",
      "    left top,\n",
      "    right top,\n",
      "    color-stop(0, rgb(200,200,200)),\n",
      "    color-stop(1, rgb(255,255,255))\n",
      "  );\n",
      "  background-size: 80px;\n",
      "  background-repeat: repeat-y;\n",
      "}\n",
      "div#notebook {\n",
      "  padding: 0px 120px 0px 50px;\n",
      "}\n",
      "\n",
      ".ui-widget {\n",
      "  font-family: Avenir, Century, Verdana, Arial, sans-serif;\n",
      "}\n",
      "\n",
      "body {\n",
      "  /* sans-serif fonts I like */\n",
      "  font-family: \"Proxima Nova\", proxima-nova, \"Avenir Next\", Avenir, \"Century Gothic\", Optima, \"Frutiger LT Std\", \"Univers LT Std\", sans-serif;\n",
      "  /* serif fonts I like */\n",
      "/*  font-family: Century, Baskerville, \"Adobe Garamond Pro\", \"Minion Pro\", serif;*/\n",
      "}\n",
      "\n",
      "div.CodeMirror {\n",
      "  font-family: \"Source Code Pro\", source-code-pro, \"Anonymous Pro\", Consolas, Inconsolata, monospace;\n",
      "}\n",
      "\n",
      "div.text_cell_render {\n",
      "  color: #333333;\n",
      "}\n",
      "\n",
      ".rendered_html h1, .rendered_html h2, .rendered_html h3, .rendered_html h4, .rendered_html h5 {\n",
      "/*  line-height: 1em;*/\n",
      "  margin-top: 16px;\n",
      "  margin-bottom: 16px;\n",
      "  border-bottom: 1px solid #aaa;\n",
      "}\n",
      "\n",
      "</style>"
     ]
    }
   ],
   "metadata": {}
  }
 ]
}