Quick and Dirty Introduction to Neural Networks

quickIntro2NN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pybrain\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "pd.core.format.set_option('notebook_repr_html',True)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quick and Dirty Introduction to Neural Networks\n",
    "\n",
    "[Fabio A. González](http://dis.unal.edu.co/~fgonza/), Universidad Nacional de Colombia\n",
    "\n",
    "## Artificial Neuron\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "<img src=\"http://upload.wikimedia.org/wikipedia/commons/thumb/6/60/ArtificialNeuronModel_english.png/600px-ArtificialNeuronModel_english.png\">\n",
    "\n",
    "$$o_j^{(n)} = f\\left(\\sum_{i\\; in\\; layer (n-1)}w_{ij}o_i^{(n-1)} \\right)$$\n",
    "\n",
    "## Step activation function\n",
    "<img src=\"https://c.mql5.com/2/4/act1.png\">\n",
    "\n",
    "## Logistic activation function\n",
    "\n",
    "$$f(x) = \\frac{1}{1 - e^{-(x-b)}}$$\n",
    "\n",
    "<img width= 300 src=\"http://upload.wikimedia.org/wikipedia/commons/thumb/b/b5/SigmoidFunction.png/400px-SigmoidFunction.png\">\n",
    "\n",
    "## Question: How to program an artificial neuron to calculate the *and* function?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table>\n",
    "<tr>\n",
    "<th>X</th>\n",
    "<th>Y</th>\n",
    "<th>X and Y</th>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>0</td>\n",
    "<td>0</td>\n",
    "<td style=\"text-align:center\">0</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>0</td>\n",
    "<td>1</td>\n",
    "<td style=\"text-align:center\">0</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>1</td>\n",
    "<td>0</td>\n",
    "<td style=\"text-align:center\">0</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>1</td>\n",
    "<td>1</td>\n",
    "<td style=\"text-align:center\">1</td>\n",
    "</tr>\n",
    "</table>\n",
    "\n",
    "<img width=500 src=\"2in-neuron.jpg\">\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1.31571817 -1.03630767  0.33810642]\n"
     ]
    }
   ],
   "source": [
    "from pybrain.tools.shortcuts import buildNetwork\n",
    "net = buildNetwork(2, 1, outclass=pybrain.SigmoidLayer)\n",
    "print net.params\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   X  Y  output\n",
       "0  0  0   0.212\n",
       "1  0  1   0.273\n",
       "2  1  0   0.087\n",
       "3  1  1   0.118"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def print_pred2(dataset, network):\n",
    "    df = pd.DataFrame(dataset.data['sample'][:dataset.getLength()],columns=['X', 'Y'])\n",
    "    prediction = np.round(network.activateOnDataset(dataset),3)\n",
    "    df['output'] = pd.DataFrame(prediction)\n",
    "    return df\n",
    "\n",
    "from pybrain.datasets import UnsupervisedDataSet, SupervisedDataSet\n",
    "D = UnsupervisedDataSet(2) # define a dataset in pybrain\n",
    "D.addSample([0,0])\n",
    "D.addSample([0,1])\n",
    "D.addSample([1,0])\n",
    "D.addSample([1,1])\n",
    "print_pred2(D, net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   X  Y  output\n",
       "0  0  0       0\n",
       "1  0  1       1\n",
       "2  1  0       1\n",
       "3  1  1       1"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.params[:] = [-40, 50,  50]\n",
    "print_pred2(D, net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question: How to program an artificial neuron to calculate the *xor* function?\n",
    "\n",
    "<table>\n",
    "<tr>\n",
    "<th>X</th>\n",
    "<th>Y</th>\n",
    "<th>X and Y</th>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>0</td>\n",
    "<td>0</td>\n",
    "<td style=\"text-align:center\">0</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>0</td>\n",
    "<td>1</td>\n",
    "<td style=\"text-align:center\">1</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>1</td>\n",
    "<td>0</td>\n",
    "<td style=\"text-align:center\">1</td>\n",
    "</tr>\n",
    "<tr>\n",
    "<td>1</td>\n",
    "<td>1</td>\n",
    "<td style=\"text-align:center\">0</td>\n",
    "</tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAYAAABccqhmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAE1lJREFUeJzt3V9sW+d5x/HvI5ISJUuUKMmWDVuBky7o0gHN2qLuPxRT\n0QJzc9EAGbAi7TasLbAAhYvdtUUvmuWiHQpsXVZkMII0y4oNWC7aAsuGIEGH1ugQJNkCNEm3OkPc\nNajt2PpHiZIpUuKfZxciWUqmRMo+JCW9vw9wIB7yNflY4vnx8PDleczdEZEw9fW6ABHpHQWASMAU\nACIBUwCIBEwBIBIwBYBIwFoGgJn9vZnNmtnPdxnzHTN708xeM7P3RFuiiHRKO3sATwFnd7rRzO4D\nfsvd7wb+DDgfUW0i0mEtA8Dd/wNY2mXIp4DvVce+DIyZ2VQ05YlIJ0VxDOAkcLlh/QpwKoL7FZEO\ni+ogoG1b1/xikQMgHsF9XAWmG9ZPVa/bwswUCiI94u7bX6SBaPYAngH+BMDMPggsu/tsBPcrPTY6\nOspHP/pRvvjFL3L+/HleeOEFVldXcfeOLA8//HDH7ruTy36vezct9wDM7J+B3wMmzewy8DCQAHD3\nx939WTO7z8wuATngc7f9zJN9oVKpUCgUWF1dJZPJcP36ddLpNKlUiv7+fhKJRP1nIpHodblyC1oG\ngLs/2MaYc9GUI/tJuVwml8uxuLjI1atXSSQSlMtlxsfHSaVSpFIpRkZGSKVSCoADKopjAHJIVSoV\n1tbWWFxcrG/8uVyOY8eO1ZdKpcLAwADDw8O3/XgzMzO3X3QPHNS6QQEgu2jcA2i8nMlkWFtbo1Kp\n0N/fz+joaCSPd1A3pINaNygAZBe1jb5UKpHL5VhYWCCZTJLNZnF3BgYGSKVSTE1p3tdBpQCQHdXe\nAqytrW25vlAokEwmGR0d5ejRo6ytrVEqlTAzzDY/bWq8LPuXAkD2rFgssrq6ytzcHL/+9a8ZGBig\nWCwyODjI4OAgyWSyfjke11NsP9NfR/ZsY2ODbDbL7Ows/f39VCoVcrkc6XSasbEx0uk06XSaRCKh\nANjn9NeRPdvY2GBlZYW5ubn624SlpSVOnDjBiRMnKJfLJBIJRkZGel2qtKAAkD3b2Niozwisbfyz\ns7Pkcrn6xp9KpSiVSr0uVVpQAMie1d4CrK2tkclkiMfjDA4OUi6XicfjjIyMcOzYMQXAAaAAkD0r\nl8uUy2XW19fr1yUSCcbHx0mn00xMTDAxMcH4+DgA8Xj8pkX2B/0lJBLuTj6fZ2lpiWvXrpFMJnF3\nFhcXGR4evmnRR4T7gwJAIrE9AGBzvkAmk2FycrK+xGKxSKYNSzQUABKJxgAA6pcXFxeZnp6mWCzW\nN3531x7APqEAkEjUPhGoBcHy8jLXrl0jk8lQKpWIxWIcOXKEycnJXpcqDRQAEonahl8oFFheXq5P\nBV5aWiIejzM8PMzk5CT5fL7lSSqkexQAEplmZ6DJ5/Nks1nm5ua4cuUKR44cob+/n8HBQQYGBrYs\nOqdA9ykApKNKpRKrq6ssLCwwNDREX18f6+vrjI2NMTY2xujoKGNjY/T19SkAekABIB1VKpW4ceMG\n8/PzmBnr6+tks1mmpqY4fvw4U1NT9PX1MTg4yNDQUK/LDY4CQDqqWCxy48aN+it/7TsEKysrbGxs\n0NfXx5EjR+qThqS7FADSUbW3ALVX/rm5ORKJBPl8HjNjaGiI8fFxNjY2el1qkBQA0lHlcpl8Pk8+\nn99yfTKZJJVKMT4+ztGjR8lms6RSqab30dfXV19isVj9stw+BYD0RKFQIJvNcv36dYaHh4nFYiwv\nLzcdm0wmGRoaYmhoiCNHjjA0NMTg4GCXKz6cFADSE+vr6ywvL3P9+nVisRgbGxvMzc01HVvbU6h9\nwSgWiykAIqIAkJ6oTRiKxWL1U4ztdAKRiYkJTp48WZ9RqO8SREcBID1ROyhY2/jn5+cZGBhoOvbE\niRP17xIMDQ0xMTHR5WoPLwWA9EShUKhv/K0O7K2srNSnE09MTGw5D4HcHgWA9ESlUqFSqVAsFluO\nXVpaYmFhgdnZWdLpNKOjoySTyS29CWs/Y7FYF6o/PBQAsu81noR0cHAQMyOfz2/pTVi7rADYGwWA\n7HuNAdDX10exWGRlZaXen7A2o1BTifdOASD7Xi0AatOJs9ks8/Pz3HHHHayvr2+ZUSh7owCQfa8W\nALWNf3Z2luHh4Zs2fp2FeO8UALLvFYvF+icGNf39/cTjcYaGhkin0xw/fpxCobClR2Fjn0JpTgEg\nB9L2k5DWphNnMpmb+hMmk0mFwA4UAHIgbQ+A2ozCxcXFem/CdDoNbH6XQJpTAMiBVDsJaSaT2TKd\neHFxkRMnTlAoFIDNjX9sbEx7ADtQAMiB1LgHUDvnwPz8PJlMhkKhgJmRTCbrewHSnAJADqRaANRe\n+WutyDOZDACDg4OMjo7qLMQttAwAMzsLPArEgO+6+7e23T4J/BNwvHp/f+Xu/xB9qSK/4e71Twca\n9fX1cfz4cVZXV+sBITvb9bQqZhYDHgPOAu8CHjSze7YNOwf8zN1/F5gB/trMtGchcgC0Oq/SGeCS\nu7/l7kXgaeD+bWOuAbVzOaWARXfXjAyRA6DVK/VJ4HLD+hXgA9vGPAH82MzeBkaAP4yuPBHppFYB\n0M7Rk68Br7r7jJm9A/iRmd3r7qut/qGIRO/ChQtcuHChrbGtAuAqMN2wPs3mXkCjDwPfAHD3X5rZ\nr4B3Aq+0VYGIRGpmZoaZmZn6+iOPPLLj2FYB8Apwt5mdBt4GPg08uG3MG8AngBfMbIrNjf//9liz\nSCRKpRK5XI7FxUWuXbtGKpViYGBAvQh3sGsAuHvJzM4Bz7P5MeCT7n7RzB6q3v448E3gKTN7jc2D\nil9290yH6xZpSr0I98a6NUnCzDQbQzpueHi4fqKQxqWxF2Ht59jYWK/L7Qozw92bzoXW5/VyqKgX\n4d4oAORQUS/CvVEAyKGiXoR7owCQIKgXYXMKAAmCehE2pwCQIKgXYXMKAAmCehE2pwCQIKgXYXMK\nAAmCehE2pwAQ2SakXoQKAJFtQupFqAAQ2SakXoQKAJFtQupFqAAQ2SakXoQKAJE2HNZehAoAkTYc\n1l6ECgCRNhzWXoQKAJE2HNZehAoAkTYc1l6ECgCRNuzUixDY8tXhiYkJJiYmGBgYIB6PE4/HicVi\n9Z/7jQJA5DY0vh0YHh4mHo9TLpcZGRlhZGSE4eHh+qIAEDlkaichXVhYIJFIUC6XWVtbY3Jysr64\nO/39/fvy0wEFgMhtaNwDKJVK9TA4deoUa2truDsDAwM7nn+w1xQAIrehttGXy+X6xj84OEgul6NS\nqdQ3/v16FmIFgMhtqAVALpfbMiW4XC6TTCYZHR1lamqqrfMQ9IICQOQ2uHvTj/1u3LhBJpNhdna2\nfkKRQqFAMpm8qUdhL085rgAQ6YDGsxD39/dTqVTIZrP1/oS1HoWJREIBIHLY1M5CPDAwQKVSqU8j\nrvUlLJfLJBKJHc9M3C0KAJEOqJ1LoPE7BAsLC6ytrW3Z+CuVSk/rVACIdEChUNjyyt/f3086naZc\nLhOPxxkZGeHYsWMKAJHDqDZtOJfL1a/LZrMMDw+TSqWYmJjg2LFjrKys4O4960WoABDpkkqlQi6X\nI5PJcPXq1foZh9PpdM96ESoARLqkXC6Ty+VYXFysTwvO5/McPXq0Z70IFQAiXdIYAPCbTwqOHz/e\ns16ECgCRLqm9BQC2nF5seXm5Z70IFQAiXVL7pmBjp+K+vj5yuVzPehEqAES6xN2b9hLoZS/ClgFg\nZmeBR4EY8F13/1aTMTPA3wAJYMHdZyKtUuQQ62Uvwl0DwMxiwGPAJ4CrwH+Z2TPufrFhzBjwd8Dv\nu/sVM5uMtEKRQ66XvQhb7QGcAS65+1sAZvY0cD9wsWHMZ4AfuPsVAHdfiLxKkUOsl70IWwXASeBy\nw/oV4APbxtwNJMzsJ8AI8Lfu/o/RlShyuPWyF2GrAGjn/MYJ4L3Ax4Eh4EUze8nd37zd4kRCEHUv\nwgsXLnDhwoW2HrtVAFwFphvWp9ncC2h0mc0Df3kgb2Y/Be4FFAAit+h2ehHOzMwwMzNTX3/kkUd2\nfJxWAfAKcLeZnQbeBj4NPLhtzL8Aj1UPGA6w+Rbh2+3+R0XkZt3qRbhrALh7yczOAc+z+THgk+5+\n0cweqt7+uLu/YWbPAa8DFeAJd//FLVckIl3rRWjdamNkZgenX5JIj/X19dUbitQajIyMjDA1NcWd\nd97JXXfdxenTp7nrrru48847d/36sJnh7k0TQjMBRfahbvUiVACI7EPd6kWoABA5QKLuRagAEDlA\nou5FqAAQOUCi7kWoABA5QKLuRagAEDlAou5FqAAQOUD20otwbW2t5f0pAEQOgWa9CBcWWn8zXwEg\ncgg060U4Ojra8t8pAEQOgWa9CNs5g5ACQOQQaNaLMB5vvXnry0AiAdjpy0Dd6UAoIvuSAkAkYAoA\nkYApAEQCpgAQCZgCQCRgCgCRgCkARAKmABAJmAJAJGAKAJGAKQBEAqYAEAmYAkAkYAoAkYApAEQC\npgAQCZgCQCRgCgCRgCkARAKmABAJmAJAJGAKAJGAKQBEAtYyAMzsrJm9YWZvmtlXdhn3fjMrmdkD\n0ZYoIp2yawCYWQx4DDgLvAt40Mzu2WHct4DngKYdSERk/2m1B3AGuOTub7l7EXgauL/JuC8B3wfm\nI65PRDqoVQCcBC43rF+pXldnZifZDIXz1avUA1DkgGgVAO1szI8CX/XNLqOG3gKIHBit+gdfBaYb\n1qfZ3Ato9D7gaTMDmAQ+aWZFd38msipFpCN2bQ9uZnHgf4GPA28D/wk86O4Xdxj/FPCv7v7DJrfp\nrYFIj+zUHnzXPQB3L5nZOeB5IAY86e4Xzeyh6u2PR16piHTNrnsAkT6Q9gBEemanPQDNBBQJmAJA\nJGAKAJGAKQBEAqYAEAmYAkAkYAoAkYApAEQCpgAQCZgCQCRgCgCRgCkARAKmABAJmAJAJGAKAJGA\nKQBEAqYAEAmYAkAkYAoAkYApAEQCpgAQCZgCQCRgCgCRgCkARAKmABAJmAJAJGAKAJGAKQBEAqYA\nEAmYAkAkYAoAkYApAEQCpgAQCZgCQCRgCgCRgCkARALWVgCY2Vkze8PM3jSzrzS5/bNm9pqZvW5m\nL5jZu6MvVUQi5+67LkAMuAScBhLAq8A928Z8CBitXj4LvNTkflyLFi29WXbavtvZAzgDXHL3t9y9\nCDwN3N84wN1fdPdsdfVl4FQb9ysiPdZOAJwELjesX6let5MvAM/eTlEi0h3xNsZ4u3dmZh8DPg98\n5JYrEpGuaScArgLTDevTbO4FbFE98PcEcNbdl6IpT0Q6qZ23AK8Ad5vZaTPrBz4NPNM4wMzuAH4I\n/JG7X4q+TBHphJZ7AO5eMrNzwPNsfiLwpLtfNLOHqrc/DnwdSAPnzQyg6O5nOle2iETBqh/Rdf6B\nzLrzQCJyE3e3ZtdrJqBIwBQAIgFTAIgETAEgEjAFgEjAFAAiAVMAiARMASASMAWASMAUACIBUwCI\nBEwBIBIwBYBIwBQAIgFTAIgETAEgEjAFgEjAFAAiAVMAiARMASASMAWASMAUACIBUwCIBEwBIBIw\nBYBIwBQAIgFTAIgETAEgEjAFgEjAFAAiAVMAiARMASASMAWASMAUACIBUwCIBEwBIBIwBYBIwMzd\ne12DiPSI9gBEAqYAEAlY5AFgZmfN7A0ze9PMvrLDmO9Ub3/NzN4TdQ23olXdZvbZar2vm9kLZvbu\nXtS5raaWv+vquPebWcnMHuhmfTtp8zkyY2Y/M7P/NrMLXS6xqTaeI5Nm9pyZvVqt+097UObeuHtk\nCxADLgGngQTwKnDPtjH3Ac9WL38AeCnKGjpY94eA0erls72uu52aG8b9GPg34A8OyO96DPgf4FR1\nffKA1P0XwF/WagYWgXiva99tiXoP4Axwyd3fcvci8DRw/7YxnwK+B+DuLwNjZjYVcR171bJud3/R\n3bPV1ZeBU12ucbt2ftcAXwK+D8x3s7hdtFP3Z4AfuPsVAHdf6HKNzbRT9zUgVb2cAhbdvdTFGvcs\n6gA4CVxuWL9Sva7VmF5vTO3U3egLwLMdrai1ljWb2Uk2n6Tnq1fth4982vld3w2Mm9lPzOwVM/vj\nrlW3s3bqfgL4HTN7G3gN+PMu1XbL4hHfX7tPMLvFf9cpbT++mX0M+Dzwkc6V05Z2an4U+Kq7u5kZ\nN//ee6GduhPAe4GPA0PAi2b2kru/2dHKdtdO3V8DXnX3GTN7B/AjM7vX3Vc7XNstizoArgLTDevT\nbCblbmNOVa/rpXbqpnrg7wngrLsvdam2nbRT8/uApze3fSaBT5pZ0d2f6U6JTbVT92Vgwd3zQN7M\nfgrcC/QyANqp+8PANwDc/Zdm9ivgncArXanwVkR8oCQO/JLNAyX9tD4I+EH2x0HAduq+g82DQB/s\ndb3t1rxt/FPAAwehbuC3gX9n88DbEPBz4F0HoO5vAw9XL0+xGRDjvf6d77ZEugfg7iUzOwc8X/3j\nPenuF83soertj7v7s2Z2n5ldAnLA56Ks4Va0UzfwdSANnK++ohbd/cw+r3nfafM58oaZPQe8DlSA\nJ9z9F72ruu3f9zeBp8zsNTaPr33Z3TM9K7oNmgosEjDNBBQJmAJAJGAKAJGAKQBEAqYAEAmYAkAk\nYAoAkYApAEQC9v/DDts6HUtcAQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113ef3e90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_nn_prediction(N): \n",
    "    # a function to plot the binary output of a network on the [0,1]x[0,1] space\n",
    "    x_list = np.arange(0.0,1.0,0.025)\n",
    "    y_list = np.arange(1.0,0.0,-0.025)    \n",
    "    z = [0.0 if N.activate([x,y])[0] <0.5 else 1.0  for y in y_list for x in x_list]\n",
    "    z = np.array(z)\n",
    "    grid = z.reshape((len(x_list), len(y_list)))\n",
    "    plt.imshow(grid, extent=(x_list.min(), x_list.max(), y_list.min(), y_list.max()),cmap=plt.get_cmap('Greys_r'))\n",
    "    plt.show()\n",
    "\n",
    "net.params[:] = [-30, 20, 20]\n",
    "plot_nn_prediction(net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br/>\n",
    "<br/>\n",
    "## Answer: It is impossible with only one neuron!\n",
    "<br/>\n",
    "<br/>  \n",
    "<br/>\n",
    "<br/> \n",
    "  \n",
    "## We need to use more than one neuron...."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"http://www.cs.nott.ac.uk/~gxk/courses/g5aiai/006neuralnetworks/images/ffnet.jpg\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-9.55095106  6.18397902 -2.81711572 -4.15033137 -4.13667621  6.54149648\n",
      "  6.45036843  6.6837421   6.4158751 ]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X</th>\n",
       "      <th>Y</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.083</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   X  Y  output\n",
       "0  0  0   0.075\n",
       "1  0  1   0.932\n",
       "2  1  0   0.932\n",
       "3  1  1   0.083"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Dtrain = SupervisedDataSet(2,1) # define a dataset in pybrain\n",
    "Dtrain.addSample([0,0],[0])\n",
    "Dtrain.addSample([0,1],[1])\n",
    "Dtrain.addSample([1,0],[1])\n",
    "Dtrain.addSample([1,1],[0])\n",
    "\n",
    "from pybrain.supervised.trainers import BackpropTrainer\n",
    "\n",
    "net = buildNetwork(2, 2, 1, hiddenclass=pybrain.SigmoidLayer, outclass=pybrain.SigmoidLayer)\n",
    "T = BackpropTrainer(net, learningrate=0.01, momentum=0.99)\n",
    "T.trainOnDataset(Dtrain, 1000)\n",
    "print net.params\n",
    "print_pred2(D, net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAYAAABccqhmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFb1JREFUeJzt3VtsW4d9x/Hv37rLuluyZF1seY7rxLEbtEOTrAU2dQ0w\nLygaoANWpN2GtcWWlxR76yUPTV7aIcDQZUWGIEuzrFiB5GF9WDYECRq0RociyRKguSy1XN9k636/\n2LrYuvz3IPKUUiSRlshzSJ7fByBEisfkX7T40+Hh4fmZuyMi8bQv6gFEJDoKAJEYUwCIxJgCQCTG\nFAAiMaYAEImxtAFgZv9qZqNm9sEOy/zQzC6Y2Xtm9onsjigiuZLJGsALwJntrjSzB4E73P048LfA\nM1maTURyLG0AuPv/ANM7LPIF4MeJZd8CGsysNTvjiUguZWMbQAfQn3J5AOjMwu2KSI5layOgbbqs\n/YtFCkBpFm5jEOhKudyZ+N4GZqZQEImIu2/+Iw1kZw3gZeCvAMzsfmDG3Ue3GaLgTo8//njkM4Qx\n94cffsiLL77IY489xuc//3mOHDmShV8NyXdp1wDM7EXgj4BmM+sHHgfKANz9WXd/xcweNLOLwDzw\n1VwOLHuzurrK8vIyt27dCr7eunWLoaEhJiYmmJ2dZXFxkZWVlahHlRCkDQB3fziDZR7NzjiSaysr\nK1y/fp25ubng69zcHNeuXePatWuMjY0xNzfH8vJy1KNKCLKxDaCo9fT0RD3Crmw398rKCjdu3GBi\nYoLx8XHGxsYYGxtjdHSUkZERxsfHuX79Ordu3Qp3YImEhXVAEDNzHXwkejMzM8Ff+/7+/uD89PQ0\ns7OzzMzMMDMzw+zsLPPz81GPK1my3UZArQHETHINYHx8nP7+fi5cuMD58+dZXFxkaWmJmzdvsrS0\npDWAmFAAFLHkFv7U80tLS8zOzjI+Ps7AwACXLl3i3LlzrK6uRjytREEBUKTcPfirvri4GJxGR0fp\n6+tjZGSEmZkZFhcX0Uuz+FIAFKlkAExPTwenmZkZRkZGGBgYYHh4mOnpaZaWlqIeVSKkAChS7s7C\nwgLT09MMDw8zNDTEyMgIo6OjjI+PMzExwfT0tNYAYk4BUKRS1wCGh4e5cuUKfX19jI6OcuPGDa5f\nv86NGzdYXFyMelSJkAKgSCUDYGpqiqGhIS5fvkxvby9jY2MsLy+zsrISnLQGEF8KgCKwurq64Qm9\nurrK4uLihp18kjv6TExMRD2u5BEFQBG4desWN27c2HCanZ3l8uXL9Pf3MzExwY0bN7R/v3yEAqAI\n3Lp1i7m5OSYmJoLT+Ph4sPFPASDbUQAUgeXlZebm5hgbG2NgYID+/n4GBwc3vAWoAJCtKACKQHIN\nYHR0lKtXr3Lx4kWuXLnCwsICS0tLLCws6CO+siUFQBFIBsDY2Bj9/f389re/5fz581se+EMklQKg\ngKytrXHz5s3gAzvJ88lP9o2OjjI1NcX8/Lw+zCMZUQAUkNXVVebn54OP7Sa/Dg8Pc/XqVUZHR5md\nndXuvZIxBUABSQbA5OQkIyMjwa69yff6x8bGmJmZ4ebNm1GPKgVCAVBA1tbWggAYHBzk6tWr9PX1\nMTU1xezsbHB4LwWAZEoBUEBS1wAGBwe5ePEivb29zM3NBQf3TJ5EMqEAyFNra2usra2xuroanJ+b\nm2NqairYyWdgYIC+vj4dukt2TQGQp27evMn8/DwLCwvB16mpKS5fvszQ0BBTU1MsLCywtrYW9ahS\nwBQAeWppaYmZmRmmpqaYnJwM/vIPDAwEATA/P68AkD1RAOSp5LH7RkZGGBwcZGhoiOHh4SAMJicn\ntQYge6YAyFM3b94MDuHV19fHpUuXGBwcDF4SJF8WKABkLxQAeWpzAJw/f55r164FGwVXV1eD8yK7\npQCI2HZdff39/QwPDzM2Nsbk5CTT09PMzc1FPa4UGQVAxNTVJ1FSAERMXX0SJQVAxJaXl7l+/XpQ\n1bVVV5/WACRXFAARU1efREkBECJ19Um+UQCERF19ko8UACFRV5/kIwVASNTVJ/lIARASdfVJPlIA\nhERdfZKP0gaAmZ0BngJKgB+5+5Obrm8GfgK0JW7vH9z937I/auFQV58Uih0DwMxKgKeBB4BB4G0z\ne9ndz6Us9ijwa3f/TiIMzpvZT9w9ti0U6uqTQpFuDeBe4KK79wGY2UvAQ0BqAAwDH0+crwMm4/zk\nB3X1SeFIFwAdQH/K5QHgvk3LPAf83MyGgFrgz7M3XmFSV58UinQBkMnWqMeAd929x8yOAT8zs3vc\n/frmBZ944ongfE9PDz09PbcxauFQV58UinQBMAh0pVzuYn0tINWnge8BuPslM7sCnADe2XxjqQFQ\nzNTVJ4UiXQC8Axw3s25gCPgS8PCmZXpZ30j4KzNrZf3Jfzm7Y+YndfVJodsxANx9xcweBV5j/W3A\n5939nJk9krj+WeD7wAtm9h6wD/imu0/leO68oK4+KXQW1mqomXmxrfIuLS1teE9/q66+5GlycjLq\ncSXG3N22+r72BNwDdfVJoVMA7IG6+qTQKQAypK4+KUYKgAypq0+KkQIgQ+rqk2KkAMiQuvqkGCkA\nMqSuPilGCoAMqatPipECYBN19UmcKAA2UVefxIkCYBN19UmcKAA2UVefxIkCYBN19UmcxDoA1NUn\ncRfbAFBXn4gCQF19EmuxDgB19UncxToA1NUncRf7AFBXn8RZLAJAXX0iW4tFAKirT2RrsQkAdfWJ\nfFQsAkBdfSJbi0UAqKtPZGuxCgB19YlsVFQBoK4+kdtTVAGgrj6R21OUATA5ObltV9/MzIyqukQS\niioA1NUncnuKKgDU1Sdyewo2ANTVJ7J3BRsA6uoT2buCDQB19YnsXUEHgLr6RPamYANAXX0ie1c0\nAaCuPpHbl/cBoK4+kdxJGwBmdgZ4CigBfuTuT26xTA/wj0AZMOHuPdkaUF19IrmzYwCYWQnwNPAA\nMAi8bWYvu/u5lGUagH8G/sTdB8ysOZsDqqtPJHfSrQHcC1x09z4AM3sJeAg4l7LMl4GfuvsAgLtn\n9aB66uoTyZ10AdAB9KdcHgDu27TMcaDMzH4B1AL/5O7/nq0B1dUnkjvpAiCTI2SUAZ8EPgdUA2+Y\n2ZvufmHzgk888URwvqenh56eno13pq4+kVClC4BBoCvlchfrawGp+lnf8LcILJrZL4F7gB0DYDN1\n9YmEL10AvAMcN7NuYAj4EvDwpmX+E3g6scGwgvWXCD+43UHU1ScSvh0DwN1XzOxR4DXW3wZ83t3P\nmdkjieufdfdeM3sVeB9YA55z99/c7iDq6hMJn4X1ZDIz3+m+VlZWuHz5Mn19fVy5ciU4v7mrL/lV\nISCSOXe3rb6fN3sCqqtPJHyRBIC6+kTyQyQBoK4+kfwQWQCoq08kepEEgLr6RPJDpGsA6uoTiVak\nAaCuPpFohRoAvb29AMFqv7r6RKIVagC8//77AIyPj6urTyQPhBoAH3zwAQAzMzPq6hPJA5EEwMLC\ngrr6RPJAqJ8FOHr0KLC+3//mrj4d0Uckd7b7LECoARDKHYnIR2wXAPvCHkRE8ocCQCTGFAAiMaYA\nEIkxBYBIjCkARGJMASASYwoAkRhTAIjEmAJAJMYUACIxpgAQiTEFgEiMKQBEYkwBIBJjCgCRGFMA\niMSYAkAkxhQAIjGmABCJMQWASIwpAERiLJJyUBHJroqKio+cysrKgN91cm5FASBSBCoqKqivr6eh\noYH6+nrq6+vZv38/sHMApH0JYGZnzKzXzC6Y2bd2WO5TZrZiZl/czQ8gIrtXWVlJQ0MDbW1tdHd3\nc9ddd3H69GlOnz6947/bcQ3AzEqAp4EHgEHgbTN72d3PbbHck8CrwJYNJCKSOxUVFTQ0NNDa2kp3\ndzdHjhyhpaUl7b9L9xLgXuCiu/cBmNlLwEPAuU3LfQP4D+BTtz25iOxZ8iVAcg3gxIkTdHR0pP13\n6QKgA+hPuTwA3Je6gJl1sB4Kf8x6AKgDUCRHzIySkhL27dsXnEpKSmhsbKS5uZm2tjY6Ojo4cuQI\nXV1daW8vXQBk8mR+Cvi2u7uZGXoJIJIzpaWl7N+/n+rqaqqrq4PzR48e5fDhw7S0tHDhwgVef/11\nampq0t7eju3AZnY/8IS7n0lc/g6w5u5Ppixzmd896ZuBBeBv3P3lTbelNQORPaqurqapqYkDBw5s\n+Nre3k5nZycdHR3BqampCVhfa9iuHTjdGsA7wHEz6waGgC8BD6cu4O6/lzxvZi8A/7X5yS8i2VFa\nWkptbS0tLS20t7cHT/YDBw4Ep7q6umAfgLS3t9OV7r5iZo8CrwElwPPufs7MHklc/+xefyARyVxp\naSk1NTU0NzfT1dXFsWPHOHbsGDU1Nezfvz94SVBeXp7R7e34EiCb9BJAZO9aW1u5++67ufvuuzl5\n8mRwvry8PNggmLqREPb2EkBEImBmlJWVUV5evuFrW1sb7e3ttLa20tLSQlNTE/X19ZSUlOzqfhQA\nInnIzKiurqa2tpa6ujrq6uqora0NdvQ5dOgQjY2NVFVVsf7m2+4oAETykJlRVVVFY2MjBw8e5ODB\ng7S0tNDa2kpbWxttbW00NDRQWVm5p/tRAIjkodQAOHToULBjT2tr64YP/WgNQKQIJQMg+R7/0aNH\n+djHPsbBgweprKwMPvJbWVmpABApZMknsJkF58vKyqipqaGpqYlDhw7R3d0dBEA2KQBEIlZWVkZV\nVRVVVVVUVlZSVVVFbW0td9xxB52dnTQ3N1NTU0NpafafrgoAkYiVl5dTV1dHY2NjcGpqaqKrq0sB\nIFLskgGQ3MLf3t5OW1sbLS0tNDc3KwBEilkyAFpaWjh8+DBHjx7lyJEj1NTUUFtbS01NjQJApFil\nrgEcPnyYEydOcPz4cUpLS4NTWVmZAkCkkJnZhid18nTw4EFaW1s37OTT2toaykwKAJGQlJSUsH//\n/mCVPrmK39HRQXd3d7CTT0VFRWgzKQBEQpIMgAMHDgQb95KH8Up+wEcBIFKk9u3bFwRAR0fHhrf5\nkm/9NTQ0KABEilHqGkBHRwfHjh3j+PHjNDQ0BDsCVVVVKQBEilFJSUlwTL9kAJw8eZLa2tpgN+DU\n3YHDoAAQyYGtuvoaGxvp7u6mvb2d5uZmamtrqayszPj4fbmgABDJga26+pI7+iSP2Lt///7gsF1R\nUQCI5EBqV1/y/f3W1tbg4B4HDhygurpaASBSjLbq6mtvbw8O71VXV6cAEClWW3X1dXV1UV5evuGk\nABApYNnu6gubAkBkDzLp6qutrY10S/9OFAAie5A8dNd2XX0HDx6krq4u46aesCkARPYg2119YVMA\niOxBtrv6wqYAENmDsrKyYA2gs7OTO+64Y9uuvnykABDJQFhdfWFTAIhkIKyuvrApAEQyEFZXX9gU\nACIZCKurL2wKAJEMhNXVFzYFgMgmUXb1hU0BILJJlF19YSv8n0Aky6Ls6gtbRnsnmNkZM+s1swtm\n9q0trv+Kmb1nZu+b2a/M7OPZH1UkHFs19Zw+fZrjx4/T1dVVVAGQ9icwsxLgaeABYBB428xedvdz\nKYtdBv7Q3WfN7AzwL8D9uRhYJNei7OoLWyY/wb3ARXfvAzCzl4CHgCAA3P2NlOXfAjqzOKNIqKLs\n6gtbJj9BB9CfcnkAuG+H5b8OvLKXoUTCkI9dfWHLJAA80xszs88CXwM+s+uJREKSj119YcskAAaB\n1GMZdbG+FrBBYsPfc8AZd5/OzngiuZOPXX3ZcPbsWc6ePZvRsua+8x94MysFzgOfA4aA/wUeTt0I\naGaHgZ8Df+Hub25zOxmvSYiEoaqqKnhrr7Ozc8uuvuTbgPX19VGPu2tmhrtvuXti2jUAd18xs0eB\n14AS4Hl3P2dmjySufxb4LtAIPJPYc2rZ3e/N1g8gkgv52NUXtrRrAFm7I60BSJ6pr6/n1KlTnDp1\nitOnT3P69GlOnTq1ZVdfIe3fv9me1gBEikGhdPWFTQEgsVAoXX1hUwBILBRKV1/YFAASC4XS1Rc2\nBYDEQqF09YVNASBFpdC7+sKmAJCiUuhdfWFTAEhRKfSuvrApAKSoFHpXX9gUAFJUCr2rL2wKACkq\nhd7VFzYFgBSkYu3qC5sCQApSsXb1hU0BIAWpWLv6wqYAkIJUrF19YVMASEEq1q6+sCkAJO/Fqasv\nbAoAyXtx6uoLmx4xyXtx6uoLmx4xyXupTT3J9/nb2tpoaWkJDuWtANgdPWKS9+LU1Rc2PWKS9+LU\n1Rc2PWKSN9TVFz4FgOQNdfWFTwEgeaNYu/rymQJA8sa+ffs2VHVt1dXX0NCgAMgiBYDkDXX1hU8B\nIHmjpKSE6upqmpqaggA4efLkll19kh0KAImEuvrygwJAIqGuvvygAJBIqKsvPygAJBLq6ssPenQl\nEpu7+u68807m5uY4duxYcBz/QnkJcPbs2ahH2LX8f3SlKJWXl1NTU0NjYyOtra10dXXR29vLoUOH\nOHDgALW1tVRUVCgAciz/H10RyRkFgEiMmbuHc0dm4dyRiHyEu2+591RoASAi+UcvAURiTAEgEmNZ\nDwAzO2NmvWZ2wcy+tc0yP0xc/56ZfSLbM+xGurnN7CuJed83s1+Z2cejmHPTTGkf68RynzKzFTP7\nYpjzbSfD35EeM/u1mf2fmZ0NecQtZfA70mxmr5rZu4m5/zqCMW+Pu2ftBJQAF4FuoAx4F7hr0zIP\nAq8kzt8HvJnNGXI49x8A9YnzZ6KeO5OZU5b7OfDfwJ8VyGPdAHwIdCYuNxfI3E8Af5+cGZgESqOe\nfadTttcA7gUuunufuy8DLwEPbVrmC8CPAdz9LaDBzKI+wFvaud39DXefTVx8C+gMecbNMnmsAb4B\n/AcwHuZwO8hk7i8DP3X3AQB3nwh5xq1kMvcwUJc4XwdMuvtKiDPetmwHQAfQn3J5IPG9dMtE/WTK\nZO5UXwdeyelE6aWd2cw6WP8lfSbxrXx4yyeTx/o40GRmvzCzd8zsL0ObbnuZzP0ccLeZDQHvAX8X\n0my7lu0PA2X6C7b5PcmofzEzvn8z+yzwNeAzuRsnI5nM/BTwbXd3Wz+KRj4cSSOTucuATwKfA6qB\nN8zsTXe/kNPJdpbJ3I8B77p7j5kdA35mZve4+/Ucz7Zr2Q6AQaAr5XIX60m50zKdie9FKZO5SWz4\new444+7TIc22nUxm/n3gpcQRdJqBPzWzZXd/OZwRt5TJ3P3AhLsvAotm9kvgHiDKAMhk7k8D3wNw\n90tmdgU4AbwTyoS7keUNJaXAJdY3lJSTfiPg/eTHRsBM5j7M+kag+6OeN9OZNy3/AvDFQpgbuBN4\nnfUNb9XAB8DJApj7B8DjifOtrAdEU9SP+U6nrK4BuPuKmT0KvJb4z3ve3c+Z2SOJ659191fM7EEz\nuwjMA1/N5gy7kcncwHeBRuCZxF/UZXe/N89nzjsZ/o70mtmrwPvAGvCcu/8muqkzfry/D7xgZu+x\nvn3tm+4+FdnQGdCuwCIxpj0BRWJMASASYwoAkRhTAIjEmAJAJMYUACIxpgAQiTEFgEiM/T/cu9x1\n4XMRhQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x113e88110>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot_nn_prediction(net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Little Red Riding Hood Neural Network\n",
    "\n",
    "<img src=\"http://themaleharem.com/wp-content/uploads/2014/06/Walter-crane-little-red-riding-hood-meets-the-wolf-in-the-woods.jpg\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"lrrh net.jpg\">"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input: dim(6, 4)\n",
      "[[ 1.  1.  0.  0.]\n",
      " [ 0.  1.  1.  0.]\n",
      " [ 0.  0.  0.  1.]]\n",
      "\n",
      "target: dim(6, 4)\n",
      "[[ 1.  0.  0.  0.]\n",
      " [ 0.  0.  1.  1.]\n",
      " [ 0.  1.  1.  0.]]\n",
      "\n",
      "\n",
      "   Big Ears  Big Teeth  Handsome  Wrinkled  Scream  Hug  Food  Kiss\n",
      "0         1          1         0         0       1    0     0     0\n",
      "1         0          1         1         0       0    0     1     1\n",
      "2         0          0         0         1       0    1     1     0\n"
     ]
    }
   ],
   "source": [
    "from pybrain.tools.validation import Validator\n",
    "\n",
    "validator =  Validator()\n",
    "Dlrrh = SupervisedDataSet(4,4) \n",
    "Dlrrh.addSample([1,1,0,0],[1,0,0,0])\n",
    "Dlrrh.addSample([0,1,1,0],[0,0,1,1])\n",
    "Dlrrh.addSample([0,0,0,1],[0,1,1,0])\n",
    "print Dlrrh\n",
    "df = pd.DataFrame(Dlrrh['input'],columns=['Big Ears', 'Big Teeth', 'Handsome', 'Wrinkled'])\n",
    "print df.join(pd.DataFrame(Dlrrh['target'],columns=['Scream', 'Hug', 'Food', 'Kiss']))\n",
    "net = buildNetwork(4, 3, 4, hiddenclass=pybrain.SigmoidLayer, outclass=pybrain.SigmoidLayer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11501eb10>]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEPCAYAAACDTflkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmUXWWZ7/Hvj6okJAwiASQkaEISgTDIZBIEmgPS3BKV\n4PW2kVYRtG2wO4L3tkrDtZta3bYNtlwcWGoUcCmg2I2A0SY3xCulURESDBBIAokQyMQUpiQ0ISHP\n/WPvIidFDbtOnX32GX6ftc6qs4d37+dsgcd32q8iAjMzs0rsUnQAZmbWuJxEzMysYk4iZmZWMScR\nMzOrmJOImZlVzEnEzMwqlmsSkdQhabmkFZIu7uX4DEn3S1os6V5Jp5YdWyXpgfTYPXnGaWZmlVFe\n80QktQEPA6cBa4GFwNkRsazsnN0iYnP6/Qjg1oiYlG4/BhwbEc/lEqCZmQ1ZnjWRqcDKiFgVEVuB\nm4AZ5Sd0J5DU7sCzPa6hHOMzM7MhyjOJjAVWl22vSfftRNJZkpYBc4ELyw4F8EtJiyR9Ksc4zcys\nQu05XjtTO1lE3AbcJukk4Hrg4PTQCRGxXtK+wHxJyyNiQU6xmplZBfJMImuBA8u2DySpjfQqIhZI\napc0OiI2RMT6dP8zkm4laR7bKYlI8ou/zMwqEBFV6S7IszlrETBZ0nhJw4GZwJzyEyRNlKT0+zEA\nEbFB0ihJe6T7dwNOB5b0dpOI8CeCyy67rPAY6uXjZ+Fn4WfR/6eacquJRMQ2SbOAeUAbcG1ELJN0\nfnp8NvBB4BxJW4FNwIfT4vsDt6T5pR24MSLuyCtWMzOrTJ7NWUTEXJIO8/J9s8u+fwX4Si/lHgWO\nyjM2MzMbOs9YbxKlUqnoEOqGn8UOfhY7+FnkI7fJhrUgKRo5fjOzIkgiGqBj3czMmpyTiJmZVcxJ\nxMzMKuYkYmZmFXMSMTOzijmJmJlZxZxEzMysYg2fRDZuLDoCM7PW1fBJ5Pe/LzoCM7PW1fBJZEmv\n7/Y1M7NaaPgk8sgjRUdgZta6Gj6JrFhRdARmZq2r4ZOIayJmZsVp+CTy3HPw8stFR2Fm1poaPomM\nGQPr1hUdhZlZa2r4JDJ2rJOImVlRGj6JHHCAk4iZWVGcRMzMrGJOImZmVjEnETMzq1jDJ5G3vAWe\nfLLoKMzMWlOuSURSh6TlklZIuriX4zMk3S9psaR7JZ2atWy3ffaBDRvy+gVmZtYfRUQ+F5bagIeB\n04C1wELg7IhYVnbObhGxOf1+BHBrREzKUjYtE6tXB9Omwdq1ufwMM7OmI4mIUDWulWdNZCqwMiJW\nRcRW4CZgRvkJ3QkktTvwbNay3UaPhmefhZxyoZmZ9SPPJDIWWF22vSbdtxNJZ0laBswFLhxMWYCR\nI2HYMNi8ubejZmaWp/Ycr52pbhARtwG3SToJuF7SIYO5SWdnJ21t8MUvwllnlSiVShWEambWvLq6\nuujq6srl2nn2iUwHOiOiI92+BNgeEVf0U+ZPJE1Zk7OUlRQRwbHHwuzZcNxxufwUM7Om0ih9IouA\nyZLGSxoOzATmlJ8gaaIkpd+PAYiIDVnKlttnn6RfxMzMaiu35qyI2CZpFjAPaAOujYhlks5Pj88G\nPgicI2krsAn4cH9l+7qXh/mamRUjt+asWuhuzrrwQpg4ES66qOiIzMzqX6M0Z9WMm7PMzIrhJGJm\nZhVzEjEzs4o1RRIZPdod62ZmRWiKJLL33vDcc0VHYWbWepoiiYwe7SRiZlaEpkgiromYmRWjKZLI\nbrvBq6/Cli1FR2Jm1lqaIolIro2YmRWhKZIIuF/EzKwITZNEXBMxM6u9pkoinitiZlZbTZVEXBMx\nM6stJxEzM6tY0yQRd6ybmdVe0yQR94mYmdVeUyUR10TMzGrLScTMzCrWNEnEfSJmZrXXNEnEfSJm\nZrXXVEnENREzs9pqmiSy++5+k6+ZWa01TRLpfpPv888XHYmZWevINYlI6pC0XNIKSRf3cvwjku6X\n9ICk30k6suzYqnT/Ykn3ZLmf+0XMzGqrPa8LS2oDrgZOA9YCCyXNiYhlZac9CvxZRLwoqQP4LjA9\nPRZAKSIy93S4X8TMrLbyrIlMBVZGxKqI2ArcBMwoPyEi7oqIF9PNu4FxPa6hwdzQScTMrLbyTCJj\ngdVl22vSfX35JHB72XYAv5S0SNKnstzQc0XMzGort+YskiSQiaRTgE8AJ5TtPiEi1kvaF5gvaXlE\nLOhZtrOz8/XvmzeX2LChVHHAZmbNqKuri66urlyurYjM/60f3IWl6UBnRHSk25cA2yPiih7nHQnc\nAnRExMo+rnUZsCkiruyxP8rj/5d/gc2b4ctfru5vMTNrJpKIiEF1F/Qlz+asRcBkSeMlDQdmAnPK\nT5D0VpIE8tHyBCJplKQ90u+7AacDSwa6oftEzMxqq9/mLEntwPyIOGWwF46IbZJmAfOANuDaiFgm\n6fz0+GzgH4E3A9+WBLA1IqYC+wO3pPvagRsj4o6B7uk+ETOz2uo3iaSJYLukvSLihcFePCLmAnN7\n7Jtd9v2vgL/qpdyjwFGDvZ/niZiZ1VaWjvXNwBJJ89PvABERF+YXVmXcnGVmVltZksgt6ae7B1sM\nYuRVLTmJmJnVVqbRWZJGAG9PN5enkwcL13N01ksvwdixsHFjgUGZmdW5ao7OGjCJSCoBPwAeT3e9\nFfh4RPy6GgEMRc8kEgHDhyfDfIcPLzAwM7M6Vs0kkqU56/8Ap0fEw+nN307yCpNjqhFANXW/yfe5\n52D//YuOxsys+WWZJ9LenUAAIuIR8p3pPiTuFzEzq50syeBeSdcAN5B0qn+EZCJhXXISMTOrnSxJ\n5AJgFtA9pHcB8K3cIhqi0aM9V8TMrFayzFi/PyIOAa7s79x64QmHZma102+fSERsAx6W9LYaxTNk\nY8bA+vVFR2Fm1hqyNGftDTyULlFbPmP9zPzCqty4cfDgg0VHYWbWGrIkkS/yxhUG63LGOiSTDefN\nKzoKM7PWkKVP5LsRcXCN4hmyceNgzZqiozAzaw1Z+kSWN1KfiJOImVntNF2fyH77wYsvwpYtMGJE\n0dGYmTW3LEnkH3rZV7d9IrvsAm97G/zpTzBlStHRmJk1tz6TiKRDImJ5RHRJ2jUiXik7dnxtwqvM\nIYfA8uVOImZmeeuvT+THZd9/3+NY3c5YBzj00CSJmJlZvrK8gBHeOMS3rh16KCxbVnQUZmbNL2sS\naShHHw133110FGZmza+/jvVxkr5BUgsZW/YdYGzukQ3BEUckI7RWrYLx44uOxsysefW5sqGkc+l9\nXXWRDPH9Qe7RDaDnyoblzjknqZHMmgU33wxz5sBxx8FFF0F73a6GYmaWv5ouj1vP+ksiDzwAp5wC\nw4Ylo7U+9jG44YbkBY0/+lGNAzUzqyMNk0QkdQBfA9qAayLiih7HPwJ8gaR2sxH4dEQ8kKVsek6f\nSQRg9WrYuhUOOijZ3rIF3vEO+OpX4X3vq8IPNDNrQA2RRCS1AQ8DpwFrgYXA2RGxrOyc44GlEfFi\nmjQ6I2J6lrJp+X6TSG9++lO46ir47W+H8OPMzBpYNZNInqOzpgIrI2JVRGwFbgJmlJ8QEXdFxIvp\n5t3AuKxlKzVjRvJurYULq3E1M7PWNmASkXSwpP8n6aF0+0hJX8xw7bHA6rLtNfQ/quuTwO0Vls2s\nvR3OPx+uu64aVzMza21Zxil9D/g88J10ewnJbPYvDVAuczuTpFOATwAnDLZsZ2fn699LpRKlUmnA\nMjNnwrRp8M1veqSWmTW/rq4uurq6crn2gH0ikhZFxHGSFkfE0em++yLiqAHKTSfp4+hIty8BtvfS\nuX4kcAvQERErB1l20H0i3Y46Cr71LXjXuyoqbmbWsGrdJ/KMpEllN/8fQJZVzBcBkyWNlzQcmAnM\nKT9B0ltJEshHuxNI1rJDdeqpcOed1byimVnryZJEZgGzgYMlrQP+J/DpgQqlC1rNAuYBS4GfRMQy\nSedLOj897R+BNwPflrQ4XbOkz7KD+2n9O+UUJxEzs6HqtzkrHWp7RUR8TtLuwC4R8VLNohvAUJqz\nXnwxWY/92Wdh112rHJiZWR2rWXNWRLwGnKjkv9ab6imBDNWb3gSTJ8P99xcdiZlZ48oyNuk+4GeS\n/gN4Od0XEXFLfmHVxrHHwr33JiO1zMxs8LIkkV2B54BTe+xviiSyaFHRUZiZNa4Bk0hEnFuDOApx\n7LEwe3bRUZiZNa4s80RGkswmnwKMJJ0IGBGfyD26AQylYx3glVdg771hwwYYObKKgZmZ1bFazxO5\nHngL0AF0AQcCm6px86LtuitMnOj12M3MKpUliUyKiH8ANqULUZ0BNE1X9GGHwdKlRUdhZtaYsiSR\nV9O/L0o6AtgL2De/kGpryhQnETOzSmVJIt+TtDfwRZJXjywFvpJrVDXkJGJmVrmmXR43q6VL4ayz\n4JFHqhSUmVmdq+nKhpIuK9t8/eSI+KdqBDAU1Ugir74Ke+4JL7zg15+YWWuo9eiszSSjsTYB20k6\n1sdX4+b1YPhwGD8e/vSnoiMxM2s8WSYbfrV8W9K/AXfkFlEBJk5MkshhhxUdiZlZY6lkjfXdqNJS\ntfWiO4mYmdngDFgTkbSkbHMXYD+g8P6Qapo40R3rZmaVyPICxveXfd8GPBURW3OKpxCTJsHcuUVH\nYWbWeLIkkZ5riOwh7ejUj4jnqhpRAdycZWZWmSxDfFcBbwWeT3e9GXiCZLhvRMRBeQbYn2oM8QXY\nsiVZpGrTJmjPklbNzBpYrYf4zgfeFxGjI2I08F7gjoiYUGQCqaYRI2C//WD16qIjMTNrLFmSyPER\ncXv3RkTMBd6VX0jFcJOWmdngZUki6yR9UdJ4SRMk/W9gbd6B1drEibByZdFRmJk1lixJ5GySYb23\nkiyJu1+6r6kcdBA89ljRUZiZNZYsM9Y3ABcCpG/zfSEitucdWK1NmAC33lp0FGZmjaXPmoikyyQd\nmn4fIelOYCXwpKQ/z3JxSR2SlktaIeniXo4fIukuSa9I+rsex1ZJekDSYkn3DO5nDd6ECa6JmJkN\nVn/NWTOB7oVjPw6IZDGqk4EvD3RhSW3A1STL6k4Bzu5OSmU2AJ8BvsobBVCKiKMjYupA9xsqJxEz\ns8HrL4lsKZuE0QHcFBGvRcQysk1SnAqsjIhV6Qz3m4AZ5SdExDMRsQjoawZ8VcYxZ7HffvBf/wUb\nN9bqjmZmja/fJCLpCEn7AiV2fnPvqAzXHguUz7xYw+Be3BjALyUtkvSpQZSriJS8Et61ETOz7Pqr\nUXwWuJmkCeuqiHgUQNJ7gT9muPZQp5KfEBHr0yQ2X9LyiFjQ86TOzs7Xv5dKJUqlUsU37G7SOvLI\nii9hZlZ3urq66OrqyuXauS2PK2k60BkRHen2JcD2iLiil3MvAzZFxJV9XKvX49V67Um3z3wmmS/y\n2c9W7ZJmZnWn1q89qdQiYHI6SXE4SUf9nD7O3enHSBolaY/0+27A6cCS3gpWkzvXzcwGJ7fXDUbE\nNkmzgHlAG3BtRCyTdH56fLak/YGFwJ7AdkkXkYzk2g+4JX1bcDtwY0TkvprihAmQU43PzKwp5dac\nVQvVbs5avBjOOQeW5F7nMTMrTjWbszLVRCSdAIwvOz8i4ofVCKCeTJgAq1ZBRDJay8zM+pdledwb\ngIOA+4DXyg41XRLZa69kPZENG2CffYqOxsys/mWpiRwLTKlqu1Ed6+5cdxIxMxtYltFZDwJj8g6k\nXniElplZdllqIvsCS9OXIG5J90VEnJlfWMVxEjEzyy5LEunMO4h6MmGCR2eZmWWVZT2RrhrEUTcm\nTIA5fU2JNDOznQzYJyLpeEkLJW2StFXSdkkv1SK4Irg5y8wsuywd61cDfwmsAHYFPgl8K8+gijR+\nPDzxBGxvurUbzcyqL9O7syJiBdCWrifyfZL1RZrSyJHw5jfDunVFR2JmVv+ydKxvljQCuF/SV4An\nqeFiUUXobtIaN67oSMzM6luWmsg56XmzgJeBccAH8wyqaO4XMTPLJsvorFWSRgH7R0Rn/iEVz0nE\nzCybLKOzzgQWk7zSHUlHS2rqQbDdL2I0M7P+ZWnO6gSmAc8DRMRikhcyNi3XRMzMssmSRLZGxAs9\n9jX1ANjx451EzMyyyJJEHpL0EaBd0mRJ3wR+n3NchTrwQHjySdi6tehIzMzqW5Yk8hngMJKXL/4Y\neAn4bJ5BFW3YMBgzJpl0aGZmfcsyOmszcGn6aRnd/SITJxYdiZlZ/eoziUj6ORD0PrGwaV8F382d\n62ZmA+uvJjIdWEPShHV3uq87oTT9KoeTJsEjjxQdhZlZfeuvT2QMSRPW4cDXgD8HnomIroj4dS2C\nK9KUKbBsWdFRmJnVtz6TSERsi4i5EXEOSa1kJfBrSbNqFl2BpkyBpUuLjsLMrL71OzpL0q6SPgjc\nAPwt8HXg1qwXl9QhabmkFZIu7uX4IZLukvSKpL8bTNm8HXQQrF8PmzfX+s5mZo2jv47160mG9t4O\n/FNEDGrRWEltJGuRnAasBRZKmhMR5Y1EG0iGEJ9VQdlctbfD5Mnw8MNwzDG1uquZWWPprybyEWAy\ncBHwe0kbyz5ZVjacCqyMiFURsRW4CZhRfkJEPBMRi4Ce0/oGLFsLbtIyM+tfnzWRiMi0YFU/xgKr\ny7bXkLyDK++yVXP44fDAA7W+q5lZ48iyKFWlhjIMOHPZzs7O17+XSiVKpdIQbruzd74TLr+8apcz\nMytEV1cXXV1duVxbEflM+ZA0HeiMiI50+xJge0Rc0cu5lwGbIuLKwZSVFHnFD7BhQzLp8Pnnoa0t\nt9uYmdWUJCKiKivUDrXJqj+LgMmSxksaDswE+lqHpOePGUzZ3IweDfvum3Sum5nZG+XWnBUR29I5\nJfOANuDaiFgm6fz0+GxJ+wMLgT2B7ZIuAqZExKbeyuYVa3+mToWFC5NOdjMz21luzVm1kHdzFsDV\nV8Mf/wjXXZfrbczMaqaazVlOIgNYsQJKJVizBlSVR25mVqxG6RNpCpMmwYgR8OCDRUdiZlZ/nEQG\nIEFHB9x+e9GRmJnVHyeRDGbOhBtugAZu+TMzy4WTSAYnnQSbNsHixUVHYmZWX5xEMthlFzjvPPjO\nd4qOxMysvnh0VkbPPgtvfzssWQJjx9bklmZmufDorALssw+cey5ceWXRkZiZ1Q/XRAZh7Vo44ohk\n7fV99qnZbc3Mqso1kYKMHQsf+AB8+9tFR2JmVh9cExmkBx+E00+H1av9Zl8za0yuiRTo8MPhgAPg\nV78qOhIzs+I5iVTgox+FG28sOgozs+K5OasC69bBYYfBM89Ae55rQ5qZ5cDNWQU74AAYPx7+8Iei\nIzEzK5aTSIXe8x6YO7foKMzMiuUkUiEnETMz94lUbNu2ZP31pUthzJhCQjAzq4j7ROpAezucfDJ0\ndRUdiZlZcZxEhuDkk+HXvy46CjOz4jiJDEGp5CRiZq3NSWQIjjwSnnwSnnqq6EjMzIrhJDIEbW1w\n4onwm98UHYmZWTFyTSKSOiQtl7RC0sV9nPON9Pj9ko4u279K0gOSFku6J884h6JUcue6mbWu3JKI\npDbgaqADmAKcLenQHuecAUyKiMnAXwPlL1kPoBQRR0fE1LziHConETNrZXnWRKYCKyNiVURsBW4C\nZvQ450zgBwARcTewl6S3lB2vyjjmPB11VLJY1dNPFx2JmVnt5ZlExgKry7bXpPuynhPALyUtkvSp\n3KIcorY2OOkkj9Iys9aU5ztos04l76u2cWJErJO0LzBf0vKIWNDzpM7Ozte/l0olSqXSYOMcsu4m\nrb/4i5rf2sxsQF1dXXTl1O6e22tPJE0HOiOiI92+BNgeEVeUnfMdoCsibkq3lwMnR8RTPa51GbAp\nIq7ssb+w156Uu/deOOcceOihoiMxMxtYo7z2ZBEwWdJ4ScOBmcCcHufMAc6B15POCxHxlKRRkvZI\n9+8GnA4syTHWIenuF/F8ETNrNbklkYjYBswC5gFLgZ9ExDJJ50s6Pz3nduBRSSuB2cDfpMX3BxZI\nug+4G/hFRNyRV6xD1dYG73633+prZq3Hb/Gtkh/+EG69NfmYmdWzajZnOYlUyYYNcNBByWtQRo4s\nOhozs741Sp9ISxk9GqZNg5/9rOhIzMxqx0mkij75Sfje94qOwsysdtycVUVbtsCBB8KCBXDwwUVH\nY2bWOzdn1akRI+DCC+Gf/7noSMzMasM1kSrbuBEmTYJ585L5I2Zm9cY1kTq2xx7wr/8Kn/gEbN1a\ndDRmZvlyEsnBeefBfvvBl75UdCRmZvlyc1ZO1q+Hd74Tvv1teP/7i47GzGwHTzZM1XMSAfjDH+DM\nM5Plcw85pOhozMwS7hNpENOnw+WXw3vf65czmllzynM9ESPpYF+9Gs44I1lzZI89io7IzKx63JxV\nAxFwwQXw2GPwi1/A8OFFR2Rmrcx9IqlGSSIA27bBBz+Y1ER++EPYxQ2JZlYQ94k0oPZ2+PGPk9rI\nhRfC9u1FR2RmNnROIjU0ahT853/CfffBued6MqKZNT4nkRrbay+44w549tlk/shzzxUdkZlZ5ZxE\nCjBqVLLuyGGHwXHHwaJFRUdkZlYZd6wX7N//HWbNSl6V0tnpVRHNLH/uWG8iH/oQLFkCjz+erEFy\n7bXJSC4zs0bgmkgduesuuPTSZATXpz+drJS4zz5FR2VmzcY1kSZ1/PFw551w882wfDlMnAjveU+y\n5O769UVHZ2b2RrnWRCR1AF8D2oBrIuKKXs75BvAe4GXg3IhYPIiyTVUT6WnjRpg7F376U5g/H0aP\nhhNPhGnTkk75KVOSfWZmg9EQM9YltQEPA6cBa4GFwNkRsazsnDOAWRFxhqRpwNcjYnqWsmn5pk4i\n5bZvh6VL4be/hYULYdkyeOihpCP+oINg+PAujjmmxLhxMGZMklz23jv5jB4Ne+4Jqso/MvWvq6uL\nUqlUdBh1wc9iBz+LHaqZRPJ8AeNUYGVErAKQdBMwAyhPBGcCPwCIiLsl7SVpf2BChrItZZdd4PDD\nk88FFyT7ImDdOli1Cq66qosDDijxxBNw993J/JPyz8svJ4lk991ht912fMq3d989GX48fHiyXnz3\np7/t7u/DhkFbWzIzv7297+/d221t+SU1/8diBz+LHfws8pFnEhkLrC7bXgNMy3DOWOCADGVbngRj\nxyaf+fPhc5/r+9ytW+HFF2Hz5h2fTZveuP3yy7BlS/L3+eeT792fV1/te3vbtuTz2msDf9+2LalZ\ndSeTvhJPW1uSPAfzaWtL3pp8552DL5vlIw3uU0mZal570SKYPbt61+7+5678b2/78vo7lLKPPw4L\nFtTmXrX8XZXcq5ryTCJZ25lapJGlWMOGJSO96mW0V8SOxNJXstm+Pfvntdd2fL/mmuS1MoMpn/Ue\nEYP/bN9enXMqKbd+Pdx7b3Wu3f2/W/nf3vbl9Xeo13j8cVixojb3quXvqvRe1ZJnn8h0oDMiOtLt\nS4Dt5R3kkr4DdEXETen2cuBkkuasfsum+1ujQ8TMrMoaoU9kETBZ0nhgHTATOLvHOXOAWcBNadJ5\nISKekrQhQ9mqPQQzM6tMbkkkIrZJmgXMIxmme21ELJN0fnp8dkTcLukMSSuBzcB5/ZXNK1YzM6tM\nQ89YNzOzYjXsjHVJHZKWS1oh6eKi48mTpAMl3SnpIUkPSrow3b+3pPmSHpF0h6S9yspckj6b5ZJO\nLy76fEhqk7RY0s/T7ZZ8Fumw+JslLZO0VNK0Fn4Wl6T/jiyR9CNJI1rlWUi6TtJTkpaU7Rv0b5d0\nbPr8Vkj6eqabR0TDfUiauFYC44FhwH3AoUXHlePv3R84Kv2+O8lEzEOBrwBfSPdfDFyefp+SPpNh\n6TNaCexS9O+o8jP5X8CNwJx0uyWfBck8q0+k39uBN7Xis0h/z6PAiHT7J8DHW+VZACcBRwNLyvYN\n5rd3t0rdA0xNv98OdAx070atibw+kTEitgLdkxGbUkQ8GRH3pd83kUy6HEvZZM3071np9xnAjyNi\nayQTNleSPLOmIGkccAZwDTuGiLfcs5D0JuCkiLgOkr7EiHiRFnwWwEvAVmCUpHZgFMmgnJZ4FhGx\nAHi+x+7B/PZpksYAe0TEPel5Pywr06dGTSJ9TVJseumItaOBu4G3RMRT6aGngLek3w8geSbdmu35\nXAV8Hihfqb4Vn8UE4BlJ35f0R0nfk7QbLfgsIuI54ErgCZLk8UJEzKcFn0WZwf72nvvXkuGZNGoS\nacnRAJJ2B34KXBQRG8uPRVL/7O+5NMUzk/Q+4OlIXtTZ6xDvVnkWJM1XxwDfiohjSEY4/n35Ca3y\nLCRNBD5L0jxzALC7pI+Wn9Mqz6I3GX57xRo1iawFDizbPpCdM2jTkTSMJIFcHxG3pbufSt81RloV\nfTrd3/P5jEv3NYN3AWdKegz4MXCqpOtpzWexBlgTEQvT7ZtJksqTLfgsjgN+HxEbImIbcAtwPK35\nLLoN5t+JNen+cT32D/hMGjWJvD6RUdJwksmIcwqOKTeSBFwLLI2Ir5UdmkPSeUj697ay/R+WNFzS\nBGAySYdZw4uISyPiwIiYAHwY+FVEfIzWfBZPAqslvT3ddRrwEPBzWuxZAMuB6ZJGpv++nAYspTWf\nRbdB/TuR/vP0UjrCT8DHysr0rehRBUMYjfAeklFKK4FLio4n5996Ikn7/33A4vTTAewN/BJ4BLgD\n2KuszKXps1kO/Leif0NOz+VkdozOaslnAbyDZKmE+0n+3/ebWvhZfIEkiS4h6Uge1irPgqRWvg54\nlaS/+LxKfjtwbPr8VgLfyHJvTzY0M7OKNWpzlpmZ1QEnETMzq5iTiJmZVcxJxMzMKuYkYmZmFXMS\nMTOzijmJmAGSNqV/3ybpDatoDvHal/bY/l01r29WJCcRs0T3hKkJwF8OpmD61tj+XLLTjSJOGMz1\nzeqZk4jZzi4HTkoXvLpI0i6S/k3SPZLul/TXAJJKkhZI+hnwYLrvNkmLlCwc9ql03+XAyPR616f7\nums9Sq+9RNIDkj5Udu0uSf+RLjZ1QwHPwSyT3NZYN2tQFwOfi4j3A6RJ44WImCppBPBbSXek5x4N\nHBYRj6c/A9C5AAABcElEQVTb50XE85JGAvdIujki/l7S30bE0WX36K71/HeS15YcCewLLJT0m/TY\nUSSLB60HfifphIhwM5jVHddEzHbW8/XypwPnSFoM/IHkfUST0mP3lCUQgIsk3QfcRfKW1MkD3OtE\n4EeReBr4NfBOkiRzT0Ssi+S9RPeRvOLcrO64JmI2sFmRLHD0OkklkvU7yrffDUyPiFck3QnsOsB1\ngzcmre5aypayfa/hf1etTrkmYrazjcAeZdvzgL/p7jyX9HZJo3optyfwfJpADgGmlx3b2kfn+wJg\nZtrvsi/wZySvI+91sS2zeuT/d2OW6K4B3A+8ljZLfR/4BklT0h/TNRaeBj6Qnl/+Cuz/C1wgaSnJ\nEgV3lR37LvCApHsjWfskACLiVknHp/cM4PMR8bSkQ3njKnR+3bbVJb8K3szMKubmLDMzq5iTiJmZ\nVcxJxMzMKuYkYmZmFXMSMTOzijmJmJlZxZxEzMysYk4iZmZWsf8PapX/hptKbKAAAAAASUVORK5C\nYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1145e1e10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "T = BackpropTrainer(net, learningrate=0.01, momentum=0.99)\n",
    "scores = []\n",
    "for i in xrange(1000):\n",
    "    T.trainOnDataset(Dlrrh, 1)\n",
    "    prediction = net.activateOnDataset(Dlrrh)\n",
    "    scores.append(validator.MSE(prediction, Dlrrh.getField('target')))\n",
    "plt.ylabel('Mean Square Error')\n",
    "plt.xlabel('Iteration')\n",
    "plt.plot(scores)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def lrrh_input(vals):\n",
    "    return pd.DataFrame(vals,index=['big ears', 'big teeth', 'handsome', 'wrinkled'], columns=['input'])\n",
    "\n",
    "def lrrh_output(vals):\n",
    "    return pd.DataFrame(vals,index=['scream', 'hug', 'offer food', 'kiss cheek'], columns=['output'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>big ears</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>big teeth</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>handsome</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>wrinkled</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           input\n",
       "big ears       1\n",
       "big teeth      1\n",
       "handsome       1\n",
       "wrinkled       0"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "in_vals = [1,1,1,0]\n",
    "lrrh_input(in_vals)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>scream</th>\n",
       "      <td>0.429028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hug</th>\n",
       "      <td>0.022506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>offer food</th>\n",
       "      <td>0.571803</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kiss cheek</th>\n",
       "      <td>0.522573</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              output\n",
       "scream      0.429028\n",
       "hug         0.022506\n",
       "offer food  0.571803\n",
       "kiss cheek  0.522573"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrrh_output(net.activate(in_vals))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}