Paralellel_and_pandas.ipynb

{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd\n",
      "from IPython.parallel import Client, error"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Relevant part of the class. The error happens when trying to execute the code on the engines"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class Load_balanced_view(object):\n",
      "    def __init__(self, profile='default'):\n",
      "        #initialise a direct and a load ballanced view\n",
      "        self.c = Client(profile=profile)\n",
      "        self.dview = self.c[:]\n",
      "        self.lbview = self.c.load_balanced_view()\n",
      "        \n",
      "    def exec_on_engine(self, code, block=True):\n",
      "        \"\"\" \n",
      "        Execute the given code on all engines \n",
      "        \n",
      "        Parameters\n",
      "        ----------\n",
      "        to_execute: string or list of strings \n",
      "            command(s) to execute on all the nodes. Thought for short tasks,\n",
      "            like importing modules\n",
      "        block: bool\n",
      "            whether or not to wait until done to return. default: True\n",
      "        \"\"\"\n",
      "        #Six: Python 2 and 3 Compatibility Library\n",
      "        from six import string_types  #appropriate string type\n",
      "        if isinstance(code, string_types): # if it's a string\n",
      "            code = [code,]  # convert to list\n",
      "        # execute the required commands \n",
      "        # (better to do in block mode, avoids errors if command is slow)\n",
      "        for te in code:\n",
      "            try:\n",
      "                self.dview.execute(te, block=block)\n",
      "            except error.CompositeError as e:  # if an error occurs, print a single one, not one per engine\n",
      "                e.raise_exception()\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "My default profile (set up for ipy v0.13). \n",
      "\n",
      "In `ipython_config.py`:\n",
      "\n",
      "    c.InteractiveShellApp.pylab = 'qt'\n",
      "    c.InteractiveShellApp.pylab_import_all = False\n",
      "\n",
      "In `ipython_notebook_config.py`:\n",
      "\n",
      "    c.IPKernelApp.pylab = 'qt'\n",
      "    c.IPKernelApp.pylab_import_all = False\n",
      "\n",
      "In `startup/01-interactive.py`\n",
      "\n",
      "    #enable interactive plotting\n",
      "    plt.ion()\n",
      "\n",
      "    print(\"Interactive plot on\")\n",
      "\n",
      "Motivation:\n",
      "I mostlt use numpy and matplotlib and don't want to import them every time. \n",
      "Plus I don't like `from pylab import *` and I want interactive plotting (I guess that this affects mostly the console)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "parallel_env = Load_balanced_view(profile='default')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "I've create an \"empty\" profile with \n",
      "\n",
      "    ipython profile create empty --reset\n",
      "If I start the engines with `profile=empty` and execute the cell below, the whole code runs without problem"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#parallel_env = Load_balanced_view(profile='empty')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "If I download this notebook as `.py` and run it, I get the error in the same position as here, but the error message is less helpful:\n",
      "\n",
      "    File \"/home/montefra/.local/lib/python2.7/site-packages/IPython/parallel/error.py\", line 199, in raise_exception\n",
      "        raise RemoteError(en, ev, etb, ei)\n",
      "    RemoteError: NameError(name 'plt' is not defined)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "imports = ['import numpy as np',]\n",
      "parallel_env.exec_on_engine(imports)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "RemoteError",
       "evalue": "NameError(name 'plt' is not defined)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
        "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)\u001b[1;32m/home/montefra/.local/lib/python2.7/site-packages/IPython/utils/py3compat.pyc\u001b[0m in \u001b[0;36mexecfile\u001b[1;34m(fname, *where)\u001b[0m",
        "\u001b[0;32m    202\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m    203\u001b[0m                 \u001b[0mfilename\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfname\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;32m--> 204\u001b[1;33m             \u001b[0m__builtin__\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexecfile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilename\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0mwhere\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0m\u001b[1;32m/home/montefra/.config/ipython/profile_default/startup/01-interactive.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m",
        "\u001b[0;32m      1\u001b[0m \u001b[1;31m#enable interactive plotting\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;32m----> 2\u001b[1;33m \u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mion\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m      4\u001b[0m \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"Interactive plot on\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[0;32m      5\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m",
        "\u001b[1;31mNameError\u001b[0m: name 'plt' is not defined"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "imports = ['import numpy as np',]\n",
      "parallel_env.exec_on_engine(imports)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "imports = ['import pandas as pd',]\n",
      "parallel_env.exec_on_engine(imports)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "imports = ['import pandas as pd',]\n",
      "parallel_env.exec_on_engine(imports)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}