11Lines.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A Neural Network in 11 lines of Python\n",
    "======================================\n",
    "\n",
    "From http://iamtrask.github.io/2015/07/12/basic-python-network/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simple 2 Layers NN\n",
    "----------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output After Training:\n",
      "[[ 0.00966449]\n",
      " [ 0.00786506]\n",
      " [ 0.99358898]\n",
      " [ 0.99211957]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEKCAYAAAAW8vJGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucnGV99/HPN9kkCyQBQSCAQDgVFB+pqBioLwnHRKLQ\nasFHoAKKSlsCIlYoBxMEbLD2QOKBRxQELVSkVYOLJByyplR8hFbAQ6J2IeGYcJAYQrJJNvvrH/c9\nmcPO7M7ubOaew/f9ek125z7M/ObayX3d11kRgZmZWc6YrAMwM7PG4ozBzMyKOGMwM7MizhjMzKyI\nMwYzMyvijMHMzIo4Y7CqSOqXtP8Iz50u6enRjqmK9z1Y0qOSNkv6VJn93ZI+OkrvNTVNo8z/Tw03\nvSUtkfSRbRlTFTGM+Ptloy/zL7GNLkkr0v9khY+v1TmGov/kEdEdEXvXM4bUZ4ClwPiI+Mcy+yN9\nIGkPSQslPZvGv89gL5ym87GjH3L9RcQxEXETgKSzJf3Htny/chlyRIyJiCe25fta9ZwxtJ4Ajk//\no+UeH88gDmXwnqX2BR6L6kZxbgHuBj5Q5WsHNXxGSWNHem4jk9RRxWEeVdvgnDG0CUkTJK2RdGjB\ntl0lrZf0+vT5xyT9Lj3uB5L2qPBaRXd8hXeZkh5JN/8uvfM+vbRqQ9Ib09dYK+mXkt5XsO+bkuZL\n+r6k1yT9dLAqBkknS/pV+lpLJB2Sbv8pcCxwo6Qtkv5osPSJiBci4gbgkcGOS1/7TpJM5970M15W\nsPs0Sc9IerFwu6S5ku6QdIukdcBZkvZMSylr0nQ/tyQdri54XpqGh0v6uaR1km6TdHvh8ekxs9M4\nnpN09iCfp1vSR9Pvxk3An6Sfa1O6f4KkL0p6StILkr4saUJBXM9I+pSk1cA3JO0o6a70vdekv++V\nHv9F4GiSv8vW0mxhKTM9/1ZJL0laKelySUr3nS3pQUmfl/QHSU9ImlnwWT6abtuQ/vy/Q/09bSBn\nDK1pwJ1sRGwE/g34UMHm04DuiHgprRaZB5wNTAVWAf9a4fW3VsGUeZ+3p78emJZWbisKTBoH3AV0\nA3sBlwC3l1y4zwD+GZgCPANcW/ZDJufcDlwK7AksAe6S1BER04AfA+dGxNiI+G2FzzJsEfHnwEry\nJbPPF+yeCbwZOAWYI+nggn2nAIuA1wPfJUnf1STp/WFgnqRjcm9DhTSWNB74HslnnwJ0AX9Wcvzu\nwN7A/sCFwJcl7VjpIyUfK34FnAP8Z/q5xqf75wFvBI4EDk1f88qC83cjySgPSt9rLHAbcAhwALAJ\n+BLJm3ya/N+lUml2AbBL+p7vSWM6p2D/O0jSbc/0db+RpstEku/NB4BJwPHAsgqf2QbhjKH1iPyd\nbO7xV+m+24DCO6jT022QXIy/HhH/GRFrgIuBIzVEXfsITAN2AK6KiFcjooskoyjMsL6Xtku8CtwM\n/HGF1/og8MOIuCsi1gFXA9sDRxUcU+8qrSsiYk1E/AR4HDisYN/PIuK2iOgFdiKJ86L0+IeAG0ky\niJxKsU8DOiLiCxGxLiL+BfivkmO2AJelafxdYD1wcOkLlVH0numd+seA2RHxbES8CFxF8d8L4DMR\nsTb9LL+PiNsj4uWIeBn4HEkpoeL7FLzfWJK/66cj4sWI+DXw98BfFBz2TERcHxGvAV8D9pC0G9BP\nkgntBiginoiIx6r4zFbCGUPrKdfG8JV0XzewvaQjJE0luWh9L923B7Bi64skF9qXSe7qR9OewNMl\n9f4r0u25+J8v2PcaMLHCa+1BcueenJi85lMUx1zv+uznCn5fR3HsTxX8vifw+zSdc1ZSXXrvCTxb\nsm1lyfOXIqJvkFiqtStJZvs/uRsN4CckpZGcF9MSKQCSdpB0s6RVkrYAPwd2zFUHpSr9XV4PjKPg\nu8jAdNmaxgXpNzEi1pOUFmYDKyX9m6SDhvFZLeWMoY1ExBbgDpK7vQ8Bd6V3XZD8Z9svd2xaLN+F\ngRcgGHix3rPMMZU8B+xdcpHYr8L7VPNaU3NPlHQV3XuErzVcI8lw+gt+fw7YWdKkgm1TSarOIEnj\nHQr2Fabx8wzMQKaOIJ5ySj/XS8AGYK+Sm43xBcf0l5xzMUl10ztILvKHk5QQcn/zwdLuJWAzBd9F\nitNl8OCTkuZ7Sb4HvwG+Xs15VswZQ2sarPokV51UWI0ESX31RyQdJWkn4IvATyPiqTKv8SjwQUk7\nSzoMKB0LsJriKpRC/5+kWuOzkiZLOgmYRb49YzhVP3cAsyS9V9Jk4DKgl+SONmeo19u6X1In0Jk+\n7UyfV7IaeEtJBle1iHg6jfMfJO0kaRpJOn47PeRR4H2S9kobZT9ZcPpDQJ+kSyRNknQGycV3NKwG\n9k3Tk4joJ6ni+rKSsRoTJB2Y/t0q2QHYCKwlydA+V+Y93qwyPbMKbl6+oKRzxJtIMppvlx5bStIU\nSR9Kq5UmAB24B9SIOGNoTaVtDHfndkTEz0iqFfYAflSw/X7gcuBW8lU7he0Rhf/B/il9jadJGv9u\nLtk/F/hq+t6nU9CQGhGbgPcBx5DcBX4BOLOgcbhco2ulhu7fkrSNXJe+1nHA+0qqUIa6MBTuX09y\nMQtgOcldeyXzgItILtCXVYi79H1K93+IfBXet4HLI+KBdN+3SNoNlpNU932L4jR8P8nfZxVJev6Q\npH693OcajvtJ2kaey/VKIukgsAx4AHgF+AHFJZTS9/onkpLC88C96fGFx8wnuRnoVfkxNrPT91kG\n3APcQvIdy73XYN+Pj5KUFF4A3g6cV/GTWkWq90I9km4h6bnxQkT8nwrHzAc+QnLhOT0ifl7HEM2a\njpJuwgsi4pasY7Hml0WJ4QbghEo70yLqNJI6xmuAr9YpLrOmIem4tEpnoqS/JunaeU/WcVlrqHvG\nkHbLWzvIIScD89Ouav8C7Cpp9/pEZ9Y0ppL0MnuBpDvpn0fE6iwDstZRzfD1etsLeLLg+QrgDSQN\nVmYGRMQ3SAd2mY22Rm18Lu3p4Z4FZmZ1klWJYbAL/bMk7QsPps+nUqFfuiRnGGZmwxQRg3azzqrE\nMFhQC4Hz0z7MZ5KMqqxYjRQRfkQwZ86czGNolEc7p8Xb2WVrf8455Pt2vp1dMo/NadEYj2rUvcQg\n6T6SWS9Jh9dfRDJvDBFxVUTcnfZMWkEyhcAZ9Y7RrFktZwc+yCa+w6tbt53GJJYXDaJuD06Lkat7\nxhARx1dxzPnA+XUIx6ylrGMSdwPvYDwvsp4utmc5O7COSUOe22qcFiPXiL2SbASmT5+edQgNo93T\nYh2TeIQ3AetZyfYk40Tbk9NiZOo+8nk0SYpmjt9stElvTn/bm2Sew9zMJRDxy4yiyobTojxJRIM2\nPpuZWYNyxmBmZkWcMZiZWRFnDGZmVsQZg5mZFXHGYGZmRZwxmJlZEWcMZmZWxBmDWUsRMLZk21gG\nn7fSrFgmGYOkmZKekrRG0iVl9u8o6S5Jr6bHnZ1BmGZNaBzlM4ZxGcTSCEozRGeQ1ah7xiBpLMm6\nz2cBhwGfkPTGksP+GlgF7Aq8D/hnSZ7XyawqE0gugLnHhGzDyYwYeIkbgzOHoWVRYjgC6ImIJRGx\nEvgmcErJMf0Uf7NXRURfXaM0a0qlpYWhtreyMZQvPbkGfShZ3IWXrun8JElmUehLwF3Aa8Bm4M/q\nE5pZKxhP/gI4Nn3ejvdVYymfMdhQssg6q5kOdSawHJgEvAv4qqTJ2zQqs5bQQfmqknZtY8ilR+7h\nGulqZJFKuTWdc/YHnik55mxgXkS8BjwsqQf4I+CR0hebO3fu1t+nT5/e9nPxm8F2wA5AZ/p8c/po\nN65WA+ju7qa7u3tY59R9PYa0EbkHOCf9uQSYFRHLCo75Cslf71PAG4G7gUMi4vclr+X1GMwKSEcB\nO6aPTqAXWAOsJeInWYZWd9KRJA3vk0gyyteAtcAmIh7KMrRMVbMeQxZLe/ZJOo+k0XkycF1ELJM0\nJ91/FXB1un818CLwydJMwczKqfT/vR174vSRZJCFpadNwPrMImoWXsHNrIVIxwI7kTQ450oMm4A1\nRDyQZWh1V7nEsLntSk+FGrLEYGbbUgCvA/YkqY3dQtKstybLoDLk9paRcIdesxYyblw/8GrJ1lcZ\nN25LFuFkrNLlrR2r1YbHJQazFtLRMYHNm/uA35GvStpCR0fn4Ce2pPEkJYZC25NUrdlgnDGYtZD+\n/vGUq0rq72/Hi+FmkkvcQeTTYjnOGIbmqiSzFrJly1iSacgKnZ1uby+dnRNJeib9DliR/tySbrfB\nuMRg1kI6OjbS17cIuLZg62V0dGzMKqTMdHSMoVzpqR3TYricMZi1kORiuAq4kvzFcBUdHe1XYth3\n37351a8GzhG1775vyCCa5uKqJLMWsssur6uwfac6R5K98ePLV6tNmOD74aE4hcxayLp1fSQXw3sL\ntp7NunVfySiiLG0CBlarRbgqaSjOGMxayMSJHbz88sCL4cSJ7VeVNHnyG4BjKa5Wm8nkye01Anwk\nnDGYtZCkxDCD0othO5YYJkzoA96dPvI6O+8te7zluY3BrIVMnNhBUn1yNTA3/XlPW5YYjjxyTzo6\nziva1tHxCaZN2yOjiJpHJiUGSTOBr5HMrvp3EXFdmWOmA18mWbvhwYg4sa5BmjUhlxjyHnroOfr6\nTqcwLfr6zuCnP3WJYSh1zxgkjQVuIGkhewJYImlhyXoMOwG3AB8BfgIcWu84zZrRlCk7lW1jmDJl\nx6xCyszGjR2Uq0rq7XUbw1CyKDEcAfRExBIASd8ETgGWFRxzOnBnRNyfPh+wcpuZDZR00RxYYpgw\n4TuZxpWFpI1hoM7OdpxQcHiyaGPYC3iy4PmT6bZCBwE7SnpK0mpJF9ctOrOmluuiWdzG0I5dNC+4\n4EQOOODyom0HHHAZs2efkFFEzSOLEkM1K+uMAw4jKV3sANwn6Z6I+FXpgV7z2SzPXTTzZs1KqpAW\nLLiS3t6xdHZuYfbsmVu3t4tmWfN5GnBtRByXPp8D9BY2QEu6BNg+Iuakz78B/Cgi7ix5La/gZlZg\nxowrWLz4mjLbr+See67OICJrNNWs4JZFVdIjwIGSjpW0L0kj9MKSY34AzJI0RdKBJLdAA0oLZlbs\nggtOZMqUTxVtmzLloratPunqWsqMGVcwffpcZsy4gq6upVmH1BTqXpUUEX2SzgO+SdJd9bqIWJaW\nHIiIqyJiuaR/BX4NbCTp0rqs4ouaWYE/UFyVtDbbcDLS1bWUCy9cRE9PvodWT0/S5tBu1UnDVfeq\npNHkqiSzYq5KynNalNeoVUlmto0kffcH6u1tv5HPTouRc8Zg1kLcdz/PaTFyzhjMWogbn/M8jmHk\nPLuqWctx4zN4HEMt3Phs1kLc4GpDqabx2SUGaxldXUuZP38xGzd2MGFCHxdccGLb3R26wdVGgzMG\nawldXUs599xbWLUqP9f+44/fwte/3l591t3gaqPBjc/WEq688lZWrSretmpVsr2duMHVRoPbGKwl\nbLfdTHp730bxOgSX09n5X2zYcE9WYWWiq2spCxbcW9DgekJblZoKuXpxILcxWNvo7e0gWYfgCpKv\ndbKSWW/v45nGlYVZs97d9hc/8JQYtXDGYC2ij2QdguISA2zOJpwM+S45MX/+4qJMAaCn51oWLLiy\nLdNjOBp2zef0uCNIlvY8LSL+vY4hWpMZO3YsW7YMLDGMHftotoHVme+S89xDa+Tq3vhcsubzYcAn\nJL2xzHEdwDzgR8Cg9WFmO+wwlqTEcA3JymXXAIvS7e2j8l3yvRlFlB330Bq5LHolbV3zOSJWkky/\nfUqZ4z4F3A68VMfYrEmtXx/k2xjmpj9npNvbh++S89xDa+SyqEoqt+bzEYUHSJoKnBARJ0g6iuqW\nA7U2NmZMP+XaGJLt7cN3yXmeEmPkGnXN5+uBv0l/F4NUJXnNZwPo7x9DuV5J/f3t1SvpggtOpKfn\n8qLqpOQueWaGUWXHPbRaa83nV4AdS049MyJuK3ktj2MwAHbb7c958cWDKS0x7Lrrb3jhhTsrndaS\nPI7BBlPNOIYsMoYOoAc4J/25BJhVaelOSTcDd5XrleSMwXImT34/r776SWAx+RLDiUyadD1r1/5b\ntsGZNZCGHOBWzZrP9Y6pmc2d+xW+9KUf09e3HR0dGzj//KOZO/evsg6r7jZt2ki5NoZNm3ozisis\neXlKjCY2d+5X+NznlhBxMLm7ZOk3fPazx7Rd5jBu3Pvo6/sbSksMHR1/z+bNd2UbnFkDaciqpNHU\n7hnD9tufyIYNewN7kL8YPs922z3D+vWLsg2uzjo7Z7Fx4xRK02LChFX09nZlG1ydeeRzntNioIas\nSrLRs2FD+a6JGza03zQQY8ZsBqaQDGzLuZwxY57JKKJseORzntNi5DztdtN7BfgdsCL9+Uqm0WRl\n7NjJlBvgNnbspCzDqjuPfM5zWoycSwxNbSPJn/Ag8tUnv023t5f+/s3ALSRVSTm3ENFepSePfM5z\nWoycSwxNbRxQekc8CRifQSzZithQdnt/f3v1SvLI5zynxcg5Y2hqYyhfldR+cw5GlP8qD9HG1nI8\nP1Ce02LkXJXU1FyVlLNpU/m7w0rbW5XnB8pzWoycu6s2MWk6sDNJldJ2wAaShWleIWJJhpHVn3Qs\nsB+l3VVhBRH3ZxmaWUOppruqq5Ka2maSzKDQBmBTBrGYWatwVVJTqzSldLuWol4B1lFcemo/HtRl\ntXLG0PQ2kPwZBaxnYAmiXbi9BTyoy0ZHJlVJkmZKekrSGkmXlNl/iqTHJL0m6WeS/jiLOBtfH9AJ\nbF/ysx3vlMeTpEdhD60+2q3rrgd12Who2DWfgXOB15PMwnpj3QJsKh1AL0lJofBnOxYEN1G+vaW9\nSgwe1GWjoSHXfI6IH0TEw5GMWroN2L3+YTaDPpIL3xaS9oYt6fP26qKZcHsLeFCXjY4sMoZyaz7v\nNcjxZwHttQRX1Sr9Z2/Xi8AGiktP7dfe4kFdNhoadc1nIGmLAE4Djt924VhrKNeeELRbe4sHddlo\nyCJjeJZkJFLO/sCAuZElHQ3MA46LShPhAHPnzt36+/Tp05k+ffpoxdkk+kgugLlHO1YjWaFZs97t\njMC26u7upru7e1jnNOSaz5LeAXwbmBERKwZ5rTYf+XxY+tuewESSPvzPARDxWEZRZSP5ykDSXyGX\nFi8BEPFwRlGZNZ6GHPkcEX1Abs3nx4Abc2s+59Z9Bv6apEP6E5L6JXkob1luY8jbRH48R+7RgUeB\nmw2f50pqYtKb09/2Jn+X/DQAEb/MKKpsSIenv+1OPi1WAxDx3xlFZdZ4vLSntZHNDCwA57rxmtlw\nOGOwFhEMrEJrxyo1s9p5dlUzMyvijMHMzIo4YzAzsyLOGMzMrIgzBjMzK+KMwczMijhjMDOzIs4Y\nzMysiDMGMzMr4ozBzMyKZJIxSJop6SlJayRdUuGY+ZLWSVom6a31jrH5vJB1AA3EaQEMew7+Vua0\nGJ66ZwySxgI3kCzZeRjwCUlvLDnmJGAayYI+1wBfrXeczWIie/F2+tmXlbydfiYOukpqa3NaFPPF\nMM9pMTxZlBiOAHoiYklErCRZl+GUkmNOBuZHxIsR8S/ArpJ2r3OcDW8iEziJHh5mMWezkodZzEn0\nMJEJWYdWd04Ls9GTRcawF/BkwfMn022DHbMCeMO2Dav5HMKzfIeeom3foYdDeDajiLLjtDAbPVks\n7fkB4D0RcW76/C+AIyJidsExdwHXRcSD6fP7gb+JkhVXJLXvKj1mZiPUiAv1PEvSdpCzP/BMhWMe\nTJ9PTbcVGerDmZnZ8GVRlfQIcKCkYyXtS9IIvbDkmIXA+ZJ2lXQm8GJErK53oGZm7ajuJYaI6JN0\nHkmj82SSKqNlkuak+6+KiLvTnkkrgKeAM+odp5lZu6p7G4OZmTW2phz5XM0AuXYg6RZJqyX9IutY\nsiZpN0kL0+/Es5IuyjqmrEgaL+knkl5Nvx9zs44pa5I6JP087djStiStkNSfPjZVOq7pMoZqBsi1\nkRuAE7IOokFsB9xO0lHhOOBiSW/ONKLs9AHnAbsAbwXeL+md2YaUuYuAXwDtXkUSwC4RMSYixlc6\nqOkyBqobINcWIuIhYG3WcTSCiFgZEbdHxJqIWA48DOyadVxZiIj+iHg8IjYB/cCW9GdbkjQVOB64\nCXBPxiqu+82YMVQzQM7amKS9gYOBh7KOJStp1Uk/8BxwT0Q8nHVMGboeuBiXFnJ6JD0v6YpKBzRj\nxuA/rlUk6XXAncDHI6I363iyEhF9ETGGZJzQuyQdmnVMWZB0KvA/EfFLXFoAeA+wG3AM8CFJZaui\nmzFjqGaAXDtxRpmSNAn4ITAvN2q+3UXECuA+YGbGoWTlKOCitPT0ADBL0uKMY8pMRCyPiE1pdet3\ngbeUO64ZM4ZqBsi1E98FAZI6Sb4Ht0bE97KOJ0uS9pH0Nkmdkg4GPgAsyzquLETERWlD6xiSu+Su\niDgx67iyIGlnSYen34tDgFOBR8sd23QZQ0Tkelx8E3gMuDEi2vJLL+k+4Ang0LT72QVZx5ShacDR\nwFcLuuOdnnVQGdkeuBH4PUlp4Y6IuDvbkBpGO5ewJwI3A68Ai4FbIuL+cgd6gJuZmRVpuhKDmZlt\nW84YzMysSM0ZQy3rN1c6V9IfS/ovSeslPSrpHbXGaWZm1akpY6hl/eYhzv0C8CVgR2BB+tzMzOqg\n1hLDSNdvnjLEuf0k3TDHpD+frjFOMzOrUq3rMZSbnuKIIY5ZkW7bc5BzPwncC3wdeB44ssY4zcys\nSrVmDNX2dR3uIKy/Aj5LMlvmmSR9b48b8KJe89nMbNiGWha51qqk4azfnDM1PWawcz8cETenc93c\nBFRsfI4IPyKYM2dO5jE0ysNp4XSICI4+eg7JvWsA+d+PPrq906UatWYMtazfPNi5z0k6W9J2wDnA\nb2qM08zazIQJfWW3d3ZuqXMkzaemjCEqTE8haU7BGs53k8yNvwK4nKSaqOK56Ut/jGRhjZeBvwQ+\nXkucZtZ+LrjgRA444PKibQcccBmzZ3ttq6E09ZQYkqKZ4x9N3d3dTJ8+PeswGoLTIuF0gK6upSxY\ncC+LFh3HjBn3M3v2Ccya9e6sw8qUJGKINgZnDGbW8iTwpSJRTcZQa68kM2swXV1LmT9/MRs3djBh\nQh8XXHBi298l2/A05JQY6b7Zkp6VtFbS3FrjNGsHXV1LufDCRSxefA0//vFcFi++hgsvXERX19Ks\nQ7Mm0pBTYkg6BjiXZGGNPYF/ryVOs3Yxf/5ienquLdrW03MtCxbcm1FE1owadUqMvwSuiIjfRsS6\niHi8xjjN2sLGjeVrh3t7x9Y5EmtmtWYM5abE2GuIY1ZQeUqM3LkHAUenVUy/kTRg1LOZDeS++zYa\nas0YRnNKjMJjxgGTSTKP2cCtw4zLrC25776Nhlp7JQ1nSowH0+dT02PGlZy7X8G5TwO3RcR6YLGk\nXkm7RMTLpQHMnTt36+/Tp09v+37b1t5yvY8WLLiSRYuuZsaMK5k9e6Z7JbWx7u5uuru7h3VOTeMY\nJHUAPSTTVvQAS4BZBSOYc43Pc4D3AjOA8yNi2mDnSvoEyfxIFwLvIhkVvU+Z9/c4BrMK3Hc/z2mR\nt83HMUREn6TctBaTgetyU2Kk+6+KiLvTzGEF8BRwxmDnpi99E0mG8GJ63odridPMzKrnkc9mLcp3\nyXlOi7xqSgw1D3AzM7PW4ozBzMyKNOyUGOn+T0vql7RzrXGamVl1GnJKjHT/VOBYksZnMzOrk0ad\nEgPgn4BPM/z1os3MrAYNOSWGpNOA30TEr2uMz8zMhqnWkc+jOSVGcqA0EbgYOEZS7jyXGszM6qSR\npsTInXsAyajndQX7XpC0d0Q8VxqAp8QwM6usZabEKHmPJ4G3RcTvy7y/B7iZVeBBXXlOi7xmnhKj\n6G1qidHMzIbHU2KYtSjfJec5LfI8JYaZmQ2bMwYzMyvijMHMzIo05FxJki6XtELSWknfl/S6WuM0\nM7PqNOpcSS8Cx5GMhF4LXFZLnGZmVr2GnCspIr4WET0R8Srw78DuNcZpZmZVasi5kkp8GLizxjjN\nzKxKDTdXUtFJ0tXAcxGxsNIxnhLDzKyyLKbEmAZcGxHHpc/nAL0RcV3BMTcA/xkR30qf9wBHkbQ5\nVDxX0meAd0TEqYO8vwe4mVXgQV15Tou8bT4lBvAIcKCkY0nmOzoLmFVyzEJgjqR7SOZKejEiVkt6\nudK56VQZ7wFm1hifmZkNU6POlXQJsA+wIZ15+76IOLGWWM3MrDqeK8msRbn6JM9pkee5kszMbNic\nMZiZWZFGnRJjZ0n3Sdog6V5JO9Uap5mZVadRp8S4FFgO7AH8Nn1uZmZ10JBTYqTnzIuINcB1wJ/W\nGKeZmVWpUafE2D0inkl/fxrPlWRmVjeNNCWGhvF6W3lKjJRGNOuItbCAEU5G03qi4N920zJTYkha\nDpwQEU9L2hdYFBGHlHl/j2NIuZ92ntPCrLJ6jGPYOiVGegE/i2QKjEILgfMl7SrpTNIpMYY4dyFw\nabpAzyXA92uM08zMqtSoU2LMA+4Angf+A6g4kZ6ZmY0uT4nRIlx9kue0MKvMU2KYmdmwOWMwM7Mi\ntY58rmrqiuFOfSHpTyT9NJ1G41eSjq8lTjMzq16tJYYhp64Y4dQX49PfdwP+Fvh2jXGamVmVah3H\nsBw4PiKekbQPsLh0vIGkI4FrCsYrfBbYFBHzqjk/PecFYK+I2Fyy3Y3PKTe45jktzCqrR+NzNVNX\nDDZtxpDnSzoFeLA0UzAzs21jyHEMku4FppTZdXmV71F671b11BeSDgfmAl7W08ysTobMGCLihEr7\nJK2WtHdEPE2yRvPqMoc9SzL9Rc5+6TaAiudLehPwLeDkiHixUgyeK8nMrLIs5kr6ArADcAVwLbA2\nIi4tOaYD6AHOSX8uAWalI6TLni9pf+BHwBkR8cgg7+82hpTr1fOcFmaV1aONYR5wMMnUFQelz5E0\nVdKTkEx9AeSmvngMuLFk6osB5wMfTp//TFK/pC2S9qwxVjMzq4KnxGgRvkvOc1qYVeYpMczMbNic\nMZiZWRHQ1LiFAAAHiElEQVRnDGZmVmTEGcO2miepYP++6VxJF480RjMzG75aSgzbap6knH8G7qoh\nPjMzG4FaMoaTgXkRsQa4DvjTMsccAfRExJKIWEnSZfWUoc6XdBrwa5KMw8zM6qiWjGGbzJMkaUfg\nQuDqGmIzM7MRGnRKjIzmSfo8SUmiV9KgfW3BU2KYmQ2mrlNipFNmnxART0vaF1hUZsrtacC1BVNu\nzwF6I+K6SudL+jlJe0ShKyLi82Vi8AC3lAd15TktzCrb1gPcFgKXSnodcAnw/TLHPAIcKOnY9OJ/\nVnpexfMj4q0RMSYixgBXAZ8ulymYmdm2UUvGsK3mSTIzswx5rqQW4eqTPKeFWWWeK8nMzIbNGYOZ\nmRVpyCkxJL1F0s/SfY9KGj/SOM3MbHgabkqMdMW3O4AvAjsDHwG21BCnmZkNQ63jGI6PiGck7QMs\nLjOO4UjgmoJxDJ8FNkXEvErnSzoJODci3l9FDG58TrnBNc9pYVbZtm583iZTYgB/BGyS9GtJL0v6\nhxpiNDOzYWrEKTE6gGnAccBrwF2SToqIu8sd7CkxzMwqa5UpMT4InBQRZ6XnXAlsiIgvlonBVUkp\nV5/kOS3MKmvKKTGAxcA0SQdKmkIyPfcvaojTzMyGoeGmxIiIV4DPAf8B/BK4PyIW1RCnmZkNg6fE\naBGuPslzWphV5ikxzMxs2JwxmJlZkYabEkPSOEm3SPqDpFWSBoyoNjOzbafhpsQATgV2Bd5AMp5h\ndjoy2mxQXV1LmTHjCgBmzLiCrq6lGUdk1pwGHeA2hJNJprRYI+k6km6mpZnDEUBPRCwBkPRN4BRg\n2SDn96fniiTjWgv8oYY4rQ10dS3lwgsX0dNzLQCLF19DT08yDnPWrHdnGZpZ02nEKTHuJMkM1gC/\nA/4xIpwx2KDmz1+8NVPI6em5lgUL7s0oIrPm1YhTYrwzjWsXkiqluyXdHxFPVPme1oY2biz/Ve7t\nHVvnSMya36AZQ0ScUGmfpNWS9o6Ip4F9gNVlDnsW2K/g+X7pNoBK558O/CAd6PaKpKXA24CyGYPn\nSjKACRP6ym7v7PSM7dbe6j1X0heAHYArgGuBtRFxackxHUAPcE76cwkwKyKWVTpf0meAdwEfJikx\n3Au8NyJ+WSYGD3BLtfugrtI2BoADDriM66+f6TYGswLVDHCrJWPYmWRBnXeRTF9xatqQPBVYEhH7\npce9B/h/wGTguoj4uyHO3wG4GZgBrAe+GBFlp952xpDX7hkDJJnDggX30ts7ls7OLcyefYIzBbMS\n2zRjaATOGPKcMZhZNTwlhpmZDZszBjMzK5LllBinSVomqU/S4SXn/G16/EpJJ440xnbg0b5mNtqy\nnBLjUWAWScNz4TlvAj4GvIWkN9PXJLlkU0auJ87ixdcA3SxefA0XXrio7TOH4XbNa1VOhzynxfDU\ncsE9GZgXEWuA64A/LXPM1ikxImIlyYI9pwBExG8rDFo7BbgpIp6KiAdIxi8cUUOcLat4tG834NG+\n4ItAjtMhz2kxPFlOiVHJnsCKYZ7Tljza18y2hXpPiTFS7ohZhkf7mtk2EREjepC0L+yd/r4vsLzM\nMdNI1mzOPZ8DXFJyzBLg8ILnlwJXFjx/AHhnhRjCDz/88MOP4T2Gur7XMu32QuBSSVcAlwDfL3PM\nI8CBko4lmRLjLJIG51KFgy0WAj+UdCtwQPr4WbkAhhqkYWZmw1dLG8M84GDgeeCg9DmSpkp6EiAi\n+oDzSBqdHwNujIhl6XFnSuoH3g08LOm/03N+DXwdeJxkaoyPe3izmVn9NPWUGGZmNvqacnxApUFz\n7SZdG3u1pF9kHUvWJO0maWH6nXhW0kVZx5QVSeMl/UTSq+n3Y27WMWVNUoekn0u6K+tYsiRphaT+\n9LGp0nFNlzEMMWiu3dwAVFwzo81sB9wOTAWOAy6W9OZMI8pOrgp3F+CtwPslvTPbkDJ3EfALksbX\ndhbALhExJiLGVzqo6TIGBhk0124i4iGSZVDbXkSsjIjbI2JNRCwHHiZZz6PtRER/RDweEZtI1lDf\nQn4t9baTLgVwPHATxR1d2tWQ1/1mzBhGMmjO2oikvUk6RjyUdSxZSatO+oHngHsi4uGsY8rQ9cDF\nuLSQ0yPp+bRHaVnNmDH4j2sVSXodcCdJb7berOPJSkT0RcQYYH/gXZIOzTqmLEg6FfifdAVIlxbg\nPcBuwDHAhySVrYpuxoyhdB3p/YFnKhzbDpxRpiRNAn5IMofXg1nH0wgiYgVwHzAz41CychRwUVp6\negCYJWlxxjFlJiKWR8SmtLr1uySTlQ7QjBnD1kFzkvYlaYRemHFMWfJdECCpk+R7cGtEfC/reLIk\naR9Jb5PUKelg4APAsqzjykJEXJQ2tI4huUvuioi2nMo/XSrh8PR7cQhwKsks1wM0XcYw2KC5diPp\nPpLZZw9Nu59dkHVMGZoGHA18taA73ulZB5WR7YEbgd+TlBbuiIi7sw2pYbRzCXsiyaDhV4DFwC0R\ncX+5Az3AzczMijRdicHMzLYtZwxmZlbEGYOZmRVxxmBmZkWcMZiZWRFnDGZmVsQZg5mZFXHGYGZm\nRf4XKcjyvGi2Xw4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f10ecbcd748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "# sigmoid function\n",
    "def nonlin(x,deriv=False):\n",
    "    if(deriv==True):\n",
    "        return x*(1-x)\n",
    "    return 1/(1+np.exp(-x))\n",
    "    \n",
    "# input dataset\n",
    "X = np.array([  [0,0,1],\n",
    "                [0,1,1],\n",
    "                [1,0,1],\n",
    "                [1,1,1] ])\n",
    "    \n",
    "# output dataset            \n",
    "y = np.array([[0,0,1,1]]).T\n",
    "\n",
    "# seed random numbers to make calculation\n",
    "# deterministic (just a good practice)\n",
    "np.random.seed(1)\n",
    "\n",
    "# initialize weights randomly with mean 0\n",
    "syn0 = 2*np.random.random((3,1)) - 1\n",
    "\n",
    "plt.subplot(211)\n",
    "\n",
    "for iter in range(10000):\n",
    "\n",
    "    # forward propagation\n",
    "    l0 = X\n",
    "    l1 = nonlin(np.dot(l0,syn0))\n",
    "\n",
    "    # how much did we miss?\n",
    "    l1_error = y - l1\n",
    "\n",
    "    # multiply how much we missed by the \n",
    "    # slope of the sigmoid at the values in l1\n",
    "    l1_delta = l1_error * nonlin(l1,True)\n",
    "\n",
    "    # update weights\n",
    "    syn0 += np.dot(l0.T,l1_delta)\n",
    "    \n",
    "    plt.plot(range(1,5), l1, 'ob')\n",
    "    plt.hold(True)\n",
    "\n",
    "\n",
    "print(\"Output After Training:\")\n",
    "print(l1)\n",
    "\n",
    "plt.plot(range(1,5), y, 'or')\n",
    "plt.xlim([0, 5])\n",
    "plt.title(\"Evolution of l1 through iterations\")\n",
    "plt.subplot(212)\n",
    "plt.stem(range(1,5), y-l1)\n",
    "plt.xlim([0,5])\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It works pretty well, let's now see how this simple implementation converges with respect to the number of iterations for the training.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OO implementation\n",
    "---------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TwoLayersNN:\n",
    "    \n",
    "    def __init__(self, in_data, out_data):\n",
    "        \n",
    "        self.in_data = in_data\n",
    "        self.out_data = out_data\n",
    "        \n",
    "        self._init_synapses()\n",
    "        \n",
    "    def _init_synapses(self):\n",
    "        \n",
    "        self.syn0 = 2*np.random.random((len(self.in_data[0]), len(self.out_data[0]))) - 1\n",
    "        self.trained = False\n",
    "        \n",
    "        \n",
    "    def NL(self, x, deriv=False):\n",
    "        \n",
    "        if(deriv==True):\n",
    "            return x*(1-x)\n",
    "        return 1/(1+np.exp(-x))\n",
    "                \n",
    "    def train(self, iter_nb=10000):\n",
    "        \n",
    "        if self.trained:\n",
    "            self._init_synapses()\n",
    "        \n",
    "        for iter in range(iter_nb):\n",
    "            \n",
    "            l0 = self.in_data\n",
    "            l1 = self.NL(np.dot(l0, self.syn0))\n",
    "            \n",
    "            l1_error = self.out_data - l1\n",
    "            l1_delta = l1_error*self.NL(l1, True)\n",
    "            \n",
    "            self.syn0 += np.dot(l0.T, l1_delta)\n",
    "            \n",
    "        self.trained = True\n",
    "        return self.syn0\n",
    "            \n",
    "    def test(self, in_data, batch=True):\n",
    "        \n",
    "        if not batch:\n",
    "            in_data = [in_data]\n",
    "            \n",
    "        return self.NL(np.dot(in_data, self.syn0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convergence Study\n",
    "---------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on 1 iterations\n",
      "Training on 201 iterations\n",
      "Training on 401 iterations\n",
      "Training on 601 iterations\n",
      "Training on 801 iterations\n",
      "Training on 1001 iterations\n",
      "Training on 1201 iterations\n",
      "Training on 1401 iterations\n",
      "Training on 1601 iterations\n",
      "Training on 1801 iterations\n",
      "Training on 2001 iterations\n",
      "Training on 2201 iterations\n",
      "Training on 2401 iterations\n",
      "Training on 2601 iterations\n",
      "Training on 2801 iterations\n",
      "Training on 3001 iterations\n",
      "Training on 3201 iterations\n",
      "Training on 3401 iterations\n",
      "Training on 3601 iterations\n",
      "Training on 3801 iterations\n",
      "Training on 4001 iterations\n",
      "Training on 4201 iterations\n",
      "Training on 4401 iterations\n",
      "Training on 4601 iterations\n",
      "Training on 4801 iterations\n",
      "Training on 5001 iterations\n",
      "Training on 5201 iterations\n",
      "Training on 5401 iterations\n",
      "Training on 5601 iterations\n",
      "Training on 5801 iterations\n",
      "Training on 6001 iterations\n",
      "Training on 6201 iterations\n",
      "Training on 6401 iterations\n",
      "Training on 6601 iterations\n",
      "Training on 6801 iterations\n",
      "Training on 7001 iterations\n",
      "Training on 7201 iterations\n",
      "Training on 7401 iterations\n",
      "Training on 7601 iterations\n",
      "Training on 7801 iterations\n",
      "Training on 8001 iterations\n",
      "Training on 8201 iterations\n",
      "Training on 8401 iterations\n",
      "Training on 8601 iterations\n",
      "Training on 8801 iterations\n",
      "Training on 9001 iterations\n",
      "Training on 9201 iterations\n",
      "Training on 9401 iterations\n",
      "Training on 9601 iterations\n",
      "Training on 9801 iterations\n",
      "Training on 10001 iterations\n",
      "Training on 10201 iterations\n",
      "Training on 10401 iterations\n",
      "Training on 10601 iterations\n",
      "Training on 10801 iterations\n",
      "Training on 11001 iterations\n",
      "Training on 11201 iterations\n",
      "Training on 11401 iterations\n",
      "Training on 11601 iterations\n",
      "Training on 11801 iterations\n",
      "Training on 12001 iterations\n",
      "Training on 12201 iterations\n",
      "Training on 12401 iterations\n",
      "Training on 12601 iterations\n",
      "Training on 12801 iterations\n",
      "Training on 13001 iterations\n",
      "Training on 13201 iterations\n",
      "Training on 13401 iterations\n",
      "Training on 13601 iterations\n",
      "Training on 13801 iterations\n",
      "Training on 14001 iterations\n",
      "Training on 14201 iterations\n",
      "Training on 14401 iterations\n",
      "Training on 14601 iterations\n",
      "Training on 14801 iterations\n",
      "Training on 15001 iterations\n",
      "Training on 15201 iterations\n",
      "Training on 15401 iterations\n",
      "Training on 15601 iterations\n",
      "Training on 15801 iterations\n",
      "Training on 16001 iterations\n",
      "Training on 16201 iterations\n",
      "Training on 16401 iterations\n",
      "Training on 16601 iterations\n",
      "Training on 16801 iterations\n",
      "Training on 17001 iterations\n",
      "Training on 17201 iterations\n",
      "Training on 17401 iterations\n",
      "Training on 17601 iterations\n",
      "Training on 17801 iterations\n",
      "Training on 18001 iterations\n",
      "Training on 18201 iterations\n",
      "Training on 18401 iterations\n",
      "Training on 18601 iterations\n",
      "Training on 18801 iterations\n",
      "Training on 19001 iterations\n",
      "Training on 19201 iterations\n",
      "Training on 19401 iterations\n",
      "Training on 19601 iterations\n",
      "Training on 19801 iterations\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "# input dataset\n",
    "X = np.array([  [0,0,1],\n",
    "                [0,1,1],\n",
    "                [1,0,1],\n",
    "                [1,1,1] ])\n",
    "    \n",
    "# output dataset            \n",
    "y = np.array([[0,0,1,1]]).T\n",
    "\n",
    "iterations_numbers = list(range(1, 20000, 200))\n",
    "results = []\n",
    "\n",
    "\n",
    "NN = TwoLayersNN(X, y)\n",
    "\n",
    "for it in iterations_numbers:\n",
    "    print('Training on {} iterations'.format(it))\n",
    "    NN.train(it)\n",
    "    guesses = NN.test(X)\n",
    "    results.append(np.mean(np.power(np.abs(y - guesses), 2)))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEWCAYAAACe8xtsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8VGXZ//HPBZinUB7h10HqcYNa+CiKmngq24IgHtC0\nTK0IBFLQB1BLqVcoe3tKzITQ8JzbYyqlJZSRJlOWJ8pASOJngJmHTDBCSVPkev6412YNu32Y81oz\n832/XvNirzUza137eg1z7fu+17pvc3dERETy0S3pAEREpPqoeIiISN5UPEREJG8qHiIikjcVDxER\nyZuKh4iI5E3FQ0RE8qbiISIieeuRdAAdMbPtgTnAv4GMu9+VcEgiIhJJc8vjROBedz8dOC7pYERE\nJFbR4mFmt5rZq2a2tM3+EWb2gpmtM7Op0e6+wF+jn9+rZJwiItK5Src8rgOGZe8ws+7R/tHAPsAZ\nZrYH8CLw0ehlaW4hiYjUnYp+Kbv748D6NrsHAyvdfaG7/wVoAY4H7gM+a2ZzgAcqGaeIiHQuDQPm\nfYHVWdurgcHu/i9gbFdvNjNNCywiUgB3t0Lfm4buoKK//N29ZI/p06eX9PWdPd/ec13ta/t8Z88p\nF7Wbi3y2lQvlor3tYiVRPNpG/RLQL2u7P2G8I2dNTU1kMpkiwwoaGxtL+vrOnm/vua72tX0+e/v5\n55/vNJZ8KRedx1LM64vNRT7bykW8rVw0kslkaGpq6jSOXFgpKlBeJzRrAOa5+8BouwewEjgt+nch\ncIy7L8/xeF7p3yGtxowZQ0tLS9JhpIJyEVMuYspFzMzwaum2MrOHgVXAnma2ycwmu/tGYAJhoHwJ\ncGOuhaNVKVse1WzMmDFJh5AaykVMuYgpF1Rvy6PU1PIQEclfVbU8pLzU+oopFzHlIqZclE5NFA91\nW4mI5EbdVhF1W4mI5E/dViIiUnE1UTzUbRUoBzHlIqZcxJSL0nVbpWF6kqKVIhEiIvWgsbGRxsZG\nmpubizqOxjxEROqQxjxQt5WISK50tVVELY9YJpPJe66dWqVcxJSLmHIRU8tDREQqTi0PEZE6pJaH\niIhUXE0UDw2YB8pBTLmIKRcx5UL3eWxB93mIiORG93lENOYhIpI/jXmIiEjFqXjUEPXnxpSLmHIR\nUy5KpyaKhwbMRURyozvMIxrzEBHJn8Y8RESk4mqieIwaBStXJh1F8tR1F1MuYspFTLkonZooHrvv\nDgceCBMnwksvJR2NiEjtq5kxj7VrYcYMuPlmGDsWpk6FPn2Sjk5EJJ005hHp3RuuuAKWLoUNG2DA\nAGhuhvXrk45MRKT21EzxaLXzzjBnDjz1VBgH2X13uOoqeOutpCMrP/XnxpSLmHIRUy5KpyaKR3v3\nefTvD7fdBr/8JTz6KHzsY3DDDfDuu8nEKCKSBrrPI5LrfR5PPQXf/CasXg0XXQSnnALdaqJ0iojk\nr9gxj7opHq0eeSQUkQ0b4JJLYORIsILTJyJSnTRgnqchQ+Cxx+DSS2HaNDj44FBQaoH6c2PKRUy5\niCkXpVN3xQNCS2PkSFi8GKZMgdNPh2HDQteWiIh0re66rdrz7rtwyy1hLOSAA+Dii2GvvUoUoIhI\nCqnbqgS22iq0Pp57Dj71KRg6VFOeiIh0RsUjy7bbwrnnhiJSjVOeqD83plzElIuYclE6Kh7t2GEH\nuPBCWLECevaEvfeG886DNWuSjkxEJB1SPeZhZv2AbwI7uvtJHbym7Ot5vPxyuKz33nth0iQ455xQ\nYEREqlVNj3m4+2p3H590HPU85YmISHsqUjzM7FYze9XMlrbZP8LMXjCzdWY2tRKxFCPtU56oPzem\nXMSUi5hyUTqVanlcBwzL3mFm3aP9o4F9gDPMbA8zG2VmM81s5wrFlre99oL774cf/QjmzoU99oC7\n7oJNm5KOTESkMio25mFmDcA8dx8YbR8MXOLuQ6PtC4F33P3yrPfsBFwGDAVucvcZ7RzXR48eTUND\nAwC9evVi0KBBNDY2AvFfGuXcfvppmDu3kQ0b4OSTMxxyCBx+eOXOr21ta1vbXW1nMhlaWloAaGho\noLm5uTrmtmqneHwOGNE6pmFmo4DB7j4pz+OWfcA8F+4wf36YN2u77eCyy8JUKCIiaVTNA+Yl+8Zv\nb0r2SkvDlCdJ5yBNlIuYchFTLko3JXsli0fbYvES0C9ruz/wYiEHbmpq2txMS1q3bnDqqbB8OZx0\nEpx4IpxwAixblnRkIiKhC6uq1vNop9uqB7ASOC36dyFwjLsvz/O4qei26shbb8G114b11YcPh6Ym\n2HXXpKMSkXpXFd1WZvYwsArY08w2mdlkd98ITABagCXAjfkWjlZp6LbqSLVPeSIitaVU3VapvsM8\nF2lvebS1dm1ohdx8M4wdC1OnQp8+pTl2JpNJTfdd0pSLmHIRUy5iVdHyKLc0tzza6t0brrgCli4N\nqxkOGADNzbB+fdKRiUg9UMsjUm0tj7ZWrQrjIAsWhFbIxImhq0tEpJzU8qhyaZ/yRESkPTVRPKqp\n26ojpZjypNpzUErKRUy5iCkX1XmfR9mk6T6PYg0eDA89FFofV18NgwbBAw+EO9hFRIpVdfd5lEu1\nj3l0RlOeiEi5FDvmoeJRBTZtgnvugQsugH794NJLQwtFRKRQGjCnNsY8OpPrlCe1nIN8KRcx5SKm\nXGjMYwu1NObRma22ChMuPvccfOpTMHQojBoVVjcUEcmFxjwi9dBt1ZH162HWLJg9O7RIpk2Dvn2T\njkpEqoG6rerYDjvAhRfCihXQsyfsvTecdx6sWZN0ZCJS62qieNT6mEdXWqc8uf76jKY8idTz56Et\n5SKmXGjMYwv1MubRlT59YM6csADVypVhFt/vfCdMCy8iAhrz2Kyexzy6smxZuLx30aLQvXXaaWHQ\nXUREYx7SodYpT+67r/ApT0RE2qPiUUM66s+txylP1LcdUy5iykXpqHjUkSFD4LHHwh3q06bBwQfD\nI48kHZWIVKOaGPOYPn06jY2NGjTPg6Y8EalPmUyGTCZDc3Oz5raq9t8hSe++C7fcAhddBAccABdf\nHMZKRKS2acBcNiukP7dWpzxR33ZMuYgpF6Wj4iFAWPr23HNDEdl9dzjwwLAk7ksvJR2ZiKSRuq2k\nXWvXwowZcPPNMHZsWF+9T5+koxKRUlG3lZRF65QnS5eiKU9E5D+oeNSQcvTn7rxzdU55or7tmHIR\nUy5KR8VDctK/P9x2G/zyl/Db34YicsMN4WotEak/NTHmofs8Ku+pp8La6qtXh8t8TzklrHgoIulW\nkfs8zKwB+DbwQeCHwNWto9Nm9v/d/WOFnrhUNGCerEceCUVkwwa45BIYORKs4I+jiFRKuQfMvw/8\nGZgM9APuN7Ntoud2K/SkUh5J9OemdcoT9W3HlIuYclE6XRWPwcDF7r7Y3c8B5gO/MLP/Kn9oUi3M\nQotj8WKYMiXcdHjEEfDkk0lHJiLl0lW31QrgRHf/Y9a+U4CvAwPdvXv5Q+ycuq3SR1OeiKRfubut\nvg0Myt7h7ncDFwC/LvSkUttqdcoTEYl1Wjzc/SZ3v7Od/fPc/fDyhSWFSFt/bpJTnqQtF0lSLmLK\nRenkdHGlmfUtdyBSu3bYISyDu2IF9OwJe+8N550Ha9YkHZmIFKrL+zzMbCBwo7sfVJmQtjj38cAx\nwA7Aze7+UDuv0ZhHlXn55XBZ7733wqRJcM45ocCISOWUdczDzA4H7gG+VOgJiuHuP3H304EJwMlJ\nxCClV61TnohIrKtuq3nASe7+52JOYma3mtmrZra0zf4RZvaCma0zs6mdHGIacE0xMdSDauvPLeeU\nJ9WWi3JSLmLKRel0VTzuBKaZFX3P8HXAsOwdZtY92j8a2Ac4w8z2MLNRZjbTzHa2YAbwoLsvLjIG\nSam99oL77guPuXNhjz3grrvCUrkikk65jHlMA/Zw9y8WdaIw1ck8dx8YbR8MXOLuQ6PtC4F33P3y\nrPdMBr4MLAIWu/v17RxXYx41RlOeiJRfsWMePbp6gbtfYmajCz1BJ/oCq7O2VxPuaM8+92xgdlcH\nGjNmDA0NDQD06tWLQYMGbZ4ksbWZqu3q2e7WDR57rJH582HKlAxf/zpcc00jQ4akIz5ta7satzOZ\nDC0tLQCbvy+LUbFZddtpeXwWOMrdx0fbo4DB7j4pz+Oq5RHJZDKbPzS1YtMmuOeecKnvLruEObQO\nPLDr99ViLgqlXMSUi1g1rSTY9hv+JcJki636Ay8WcuCmpqbNFVZqS7ducOqp8OyzcPLJ8LnPwQkn\nwLJlSUcmUp0ymQxNTU1FH6erua2e5z+/9NszM+pi6uxYDWzZ8ugBrAROi/5dCBzj7stzCTzruGp5\n1JG33oJrrw3rqw8fDk1NsOuuSUclUn3K2vJw9wZ375fDo6vC8TCwCtjTzDaZ2WR330i4f6MFWEK4\nETGvwtFKLY/6keSUJyK1oCItj2qglkesHvtz164NrZCbboJx42DqVOjTpz5z0RHlIqZcxBIb8zCz\npwt9r0ip9O4NV1wRxkA2bIABA6C5OfwsIuXT1ZjHfh09BSxy98RXrdYa5pJt1aowDrJgAZx/Ppx5\nZujqEpEgU6E1zDcCj3bw9GFaDErSatmycHnvU0+Ff087LawzIiJBubutngPGufvhbR+E1oekiC4a\niK1Zk9GUJxF9LmLKRel0VTym0/Fd6F8ucSwF09VW0pHBg+Ghh8KEi1dfDYMGwQMPgBqrUq90tVVE\n3VaSK3eYPz/Mm7XddnDZZTBkSNJRiSSj2G6rrsY8vtrJe93dryr0xKWi4iH5KnTKE5FaUu4xj57A\n+zt49Cz0pKWmbqtAOYh1lot6m/JEn4uYcqFuq83U8ojpBqhYPrmo9SlP9LmIKRexsnZbVQMVDymV\n9eth1iyYPRtOOgmmTYO+fZOOSqQ8qmlWXZFU22GHMA6yYgX07AkDB8J558GaNUlHJpI+Kh41RP25\nsWJy0d6UJzNmwHvvlS6+StLnIqZclE7excPMVpUjkGJowFzKYeedYc4cePJJ+PnP4YgjNHuvVL/E\nBszNbFMa5rRqpTEPqYT33gv3hXzve2EG32OPTToikeJUfMDczH7j7p8s9ISlpuIhlfSb38AXvxgu\n7Z0xA7beOumIRApT8QHzNBUO2ZK67mLlysUnPwl/+AP85S9w8MFhUaq00+ciplyUTmq6n0SqxU47\nhQkXx4+HQw6B229POiKRyquJ+zy0nockZckSOOUUOOCAMB7SMzXzLoi0ryLreWx+kdlAd19a6EnK\nSWMekrQNG2DKFPj1r+Huu2G/jpZQE0mRSo15XGdmvzOzM81sx0JPJuWl/txYJXOx/fbhCqyLLoIj\njwx3qafp7xl9LmLKRenkVDzc/VDC+h27AcvM7AdmNryskYlUmVNOCfeE/OAHMHIkvPZa0hGJlE9e\nYx5mtjVwCjATeAXYBjjf3X9UnvByikndVpIq77wDF1wAd94Jd9wBGoqTNKrIfR5mNggYC4wA5gE3\nuPsKMxsMPOzuOxQaQLFUPCStFiwIa6ePHx/mzOrR0ZqcIgmo1JjHLOAJYKC7f9XdVwC4+1PARYWe\nXEpL/bmxNOTiyCPh6afh8cfh8MPhhReSiSMNuUgL5aJ0ch3zaHT3u9z93+08d2XpwxKpDR/6UGiB\nHHtsuJz3/vuTjkikNLpahnZ6jsfJuPuvShNSfnSfh1SLJ54IKxgefTRceSVsu23SEUk9qsh9HmY2\nBshlQGGxuy8pNIhiaMxDqsm6dXDGGbB8ebgn5H/+J+mIpF5pJUEVj820xGYszblwh+9/H77+dbj0\nUvjKV8AK/i/ctTTnotKUi5hWEhSpMmYwbly4I/1734PPfx7+8Y+koxLJj1oeIgl6+22YOhV+8pNw\nX8ihhyYdkdSLsrc8zKybmR0W3SAoIiW0zTbw3e/C1VfDZz8LF19cvcvdSn3psni4+ybgp0AZe2Wl\nFHQNe6zacjFyJPz+97BwIQwZAi++WLpjV1suykm5KJ1cxzxmAFPMTPfIipRJ377w0EMwYgTsvz/8\n+MdJRyTSsVynJ3kT2JZw2e5b0W5PclqSVhrzkFr0xBPwhS/AUUfpnhApj4pcbeXu73f37u7ew917\nRo+yFw4zG2Bm15rZXDObUO7ziaTFQQeF5W7XroXBg+GPf0w6IpEt5Xyprpkdb2bfMbMrzWxkOYNq\n5e5/cveJwMmArkPpgvpzY7WQix13DNO7n3tumJl3zpzC1gmphVyUinJROjkVDzO7HJgOLAdWANPN\n7Fu5nsTMbjWzV81saZv9I8zsBTNbZ2ZTO3jvSGA+8LNczydSK8zCzLy//W24sfAzn4E1a5KOSiT3\nMY/XCDPq/i3a/hCwzN375HQSs4OBDcCd7j4w2tcdWAmcBqwCFgLHAJ8A9gO+7e4vZx1jvrsf286x\nNeYhdeGdd2DaNLjrLrj1Vhg6NOmIpJoVO+aR69VTG4APAX+Ltj8MvJnrSdz9cTNraLN7MLDS3RcC\nmFkLcLy7Xw7cHu37NHAisDXhcmGRuvW+98EVV8Dw4TB6NHzxi+G+kPe9L+nIpB7lWjy+AfzczH5B\nuN/jCOCcIs/dF1idtb2aUFA2i2bq7XK23jFjxtDQ0ABAr169GDRo0Ob5a1r7OOthO7s/Nw3xJLnd\nui8t8ZRyu0cP+MMfGhk3DgYOzDBtGowa1fHrFy9ezNlnn52a+JPcnjVrVl1/P7S0tABs/r4sRpfd\nVmbWDRgFPEzoUnJgkbu/kteJQstjXla31WeBo9x9fLQ9Chjs7pPyPK66rSIZTfq2WT3kwj0Mojc1\nwYwZYWykvQkW6yEXuVIuYpVahnYt8FF3/1fBJzLbBZifVTwOAi5196HR9nTgbXefkedxtZ6H1LVl\ny8I9IR/7GFx/PfTunXREkmaZSqznsflF4UqovsBM4J+t+9399ZxP9J8tjx7EA+YriQbM3X157uGr\n5SECYYLFb34T7rknXJU1fHjSEUnaVWpK9onASOAR4PfR43e5nsTMHiZcUbWnmW0ys8nuvhGYALQA\nS4Ab8y0crZqamrbo665XykGs3nKxzTbwne+Eq7DGjYMpU+CtaC6IestFZ5SLkIOmpqaij5PPmMc9\n7v520WcsMbU8YurPjdVzLl5/HSZMCHel33knrFtXv7loq54/F21VzZhHuah4iPwnd7jjjnB3+te+\nFh7duycdlaRJpbqtZgCXm1k/M9up9VHoSUtN3VYiWzKDUaNg0SL4+c/hsMPgz39OOipJg4p1WwGY\n2fOES3S34O79io6gSGp5xNQkjykXsUceybB0aSMXXwwXXQQTJ5Z3zfQ00+ciVqlZdRvcvV/bR6En\nFZHK6dYtDKD/5jfQ0gJHHlnaxaakPnVaPMxsdtbPZ7Z57q5yBZUvdVsF+osqplzEWnMxYAA89ljo\nwtp/f7jvvmTjSoI+FxXqtjKzTe7ere3P7W0nRd1WIvl78slwY+GwYXDVVbDddklHJJVWqQFzqQJq\nfcWUi1h7uTjwQHj6aVi/Hg44AJ55pvJxJUGfi9KpieKhbiuR/O24Y7gP5Pzzw/TuV19d2GJTUl0q\n1m1FmI4dYPusnwG2V7eVSG147rkwxXufPnDLLfDBDyYdkZRbWbut3L1b1prl2T/3TEPhEJHS2H33\nsFrhvvvCoEHwU62eI11QAagh6rqLKRexXHOx1VZw6aVhcsWzzoKxY+Ef/yhvbJWmz0Xp1ETx0JiH\nSOkcdhgsXQrbbw977gk/+lHSEUkpVfQO8zTTmIdI+fz2tzB+PAwcCNdeq7VCaoku1RWRsjn0UPjD\nH+CjH4V99oEFC5KOSNJCxaOGqOsuplzEis1F61oht90GX/kKTJ4cFp+qRvpclI6Kh4jkZMiQcDPh\nq6+GGwuXLUs6IkmSxjxEJC/uYYLF88+HCy+E//3f+p2lt5ppzANdbSVSSWZw2mlhksU77oCjj4ZX\nXkk6KsmVrraKqOUR01oFMeUiVs5cvPsuXHIJXHcdzJ4Nn/98ulsh+lzE1PIQkcRstRU0N8O8edDU\nFIrHa68lHZVUgloeIlISb78NF1wQJlu84QY49tikI5LOFNvyUPEQkZL69a9hzJhwddbMmdCzZ9IR\nSXvUbSWb6aKBmHIRq3QuDjsMliwJP++zTygmaaHPRemoeIhIyfXsCTfdFAbRTz0Vzj0X3nor6aik\nlNRtJSJltXYtnHlmmGzxjjtgv/2SjkhA3VaA7vMQSbPeveHuu2HaNBgxAi6+OFziK8nQfR4RtTxi\nuoY9plzE0pSLF18Ms/SuWQO33hqmfK+kNOUiaWp5iEjV+MhH4MEH4YwzoLERvv1teO+9pKOSQqjl\nISKJWL06XNK7aVNohfTvn3RE9UUtDxGpSv36wcKFcMIJcOCB4eos/R1YPVQ8aoguGogpF7E056Jb\nt3AZ78KFMGcOHHcc/P3v5TtfmnNRbVQ8RCRxe+0FTzwR/t17b5g7N+mIpCsa8xCRVHniiTAWsu++\n8L3vwU47JR1RbdKYh4jUlIMOCuumf/CDoRXy4INJRyTtSX3xMLPtzWyRmR2TdCxpp/7cmHIRq8Zc\nbLstzJoFt98OEyaES3vfeKP441ZjLtIq9cUDOB+4J+kgRKTyDj88rJu+cWOYZPHRR5OOSFpVZMzD\nzG4FRgB/d/eBWftHADcAOwDfcvcZbd43DNgJ2AZY4+4/befYGvMQqQMPPAATJ4YFpy67LLROpHDV\nMuZxHTAse4eZdY/2jwb2Ac4wsz3MbJSZzTSznYFPAwcBXwC+YpbmBS5FpJyOOy5Mrvjyy/CJT8Dv\nf590RPWtRyVO4u6Pm1lDm92DgZXuvhDAzFqA4939cuD26DXToudGA6911MQYM2YMDQ3h8L169WLQ\noEGb569p7eOsh+3s/tw0xJPkduu+tMST5PbixYs5++yzUxNPMdvPPJNhwgR4+eVGjj4ahg/P8OUv\nw7Bhub1/1qxZdf390NLSArD5+7IYFbtUNyoe81q7rczsc8AIdx8fbY8CBrv7pDyPq26rSEaTvm2m\nXMRqNRevvAKnnw5//WuY6n2vvbp+T63mohDV0m3VnpJ942tK9kD/KWLKRaxWc/HhD4dxkEmTwsD6\npZeGgfXO1Gou8pGptinZzWwXYH5Wy+Mg4FJ3HxptTwfebjtonsNx1fIQqXMvvADjxsGGDXDLLfDx\njycdUfpVU8ujbZC/A3YzsyFRYRkNPFDBeGqOWl8x5SJWD7n47/+GBQvCkreHHgrf/W6YrbeteshF\npVSkeJjZw8AqYE8z22Rmk919IzABaAGWADe6+/JCjq9uKxHp1i10YT3xBNx7b+jKWrUq6ajSp+q6\nrcpF3VYi0tZ778HMmTBjRrgnZPx40IX+W6qmbquyUctDRLJ17w5f+xpkMnD99WHt9JdeSjqqdFDL\nI6KWR0yXIcaUi1i952LjxtD6uOYaGDs2w7e+1ahWCGp5iIh0qkcPuPDCMKA+d25YubCcC07Vi5oo\nHuq2Cur5r8u2lIuYchHsuy88+2wjAwaEqd5/+MOkI0qGuq0i6rYSkXw9+SSMGgX77x+6s3r3Tjqi\nylO3lWym1ldMuYgpF7HWXBx4ICxZEu5SHzgQ5s1LNq5qpOIhInVp223hqqvgnntgyhQYOxbWrUs6\nqupRE91W06dPp7GxUX27IlKQN9+E886Dn/4UbroJhg9POqLyyWQyZDIZmpubi+q2qoniUe2/g4ik\nw0MPhTmyjjoKrrwSevZMOqLy0ZiHbKa+7ZhyEVMuYl3lYtiwsOBU67K3v/pVZeKqRioeIiJZdtwR\nbr4ZZs+GL3whjIds2JB0VOlTE91WGvMQkXJ4/fUw2eKiRdDSAoccknRExdOYR0RjHiJSbvfdB2ed\nBV/+MjQ3wzbbJB1R8TTmIZupbzumXMSUi1ihuTjxxHBfyMqVsN9+oSVS71Q8RERy8IEPhLmxLrgA\njj0Wpk2Dd95JOqrkqNtKRCRPr7wC55wDl18ODQ1JR1OYYrutVDxEROqQxjzQrLqtlIOYchFTLmLK\nRelm1e1RfCjJK0UiRETqQettDc3NzUUdR91WIiJ1SN1WIiJScSoeNUT9uTHlIqZcxJSL0lHxEBGR\nvGnMQ0SkDmnMQ0REKq4miofu8wiUg5hyEVMuYsqF7vPYgu7zEBHJje7ziGjMQ0QkfxrzEBGRilPx\nqCHqz40pFzHlIqZclI6Kh4iI5E1jHiIidUhjHiIiUnGpLh5m1mhmj5rZtWb26aTjSTv158aUi5hy\nEVMuSifVxQPYBLwBbA28mHAsqbd48eKkQ0gN5SKmXMSUi9KpSPEws1vN7FUzW9pm/wgze8HM1pnZ\n1Hbe+qi7Hw18HSjujpY6sG7duqRDSA3lIqZcxJSL0qlUy+M6YFj2DjPrHu0fDewDnGFme5jZKDOb\naWY7Z42EryO0Psou32ZtV6/v7Pn2nutqX9vny9kMVy4KP3a5c5HvdikpF4Ufu5ZyUZHi4e6PA+vb\n7B4MrHT3he7+F6AFON7db3f3c9z9ZTM7wcyuA24Drq5ErGn7MLTd19mH4fnnn+80lnwpF53HUszr\nK/kloVzE28pF5+fPR8Uu1TWzBmCeuw+Mtj8HjHD38dH2KGCwu0/K87i6TldEpADFXKqb5MSIJfnS\nL+aXFxGRwlTyaqu2xeIloF/Wdn90RZWISFWoZMujbQvhd8BuZjYEWEkYOD+mgvGIiEiBKnWp7sPA\nKmBPM9tkZpPdfSMwgTBQvgS40d2XVyIeEREpTtXPbSUiIpWX9jvMRUQkhWqueJjZ9tEd7TeY2ReS\njidJZtbPzG4ys7lJx5I0Mzs++kzcbWbDun5H7TKzAdF8cXPNbELS8SQp+r5YZGZ1P96a71yCNVc8\ngBOBe939dOC4pINJkruvbr2Ppt65+0+iz8QE4OSk40mSu//J3ScS8nBo0vEk7HzgnqSDSIm85hKs\niuKR59xYfYG/Rj+/V9FAK6CIecJqToG5mAZcU7koKyPfXJjZSGA+8LNKx1pO+eQhaoE+C7yWRKyV\nkOfnIr+5BN099Q/gYGBvYGnWvu7A88DhwC6Eq7n2AL4EHBO95gdJx55kLrKen5t03EnngnCp+Axg\naNJxJ52p93TWAAAFzUlEQVSLNu+bn3TsCX4mLgFmAguAHxNdQFRLjwK/L96Xy3dGkneY58zdH4+m\nN8m2eW4sADNrAY4HZgPXRH2YD1QwzIrIJxdm9ipwGTDIzKa6+4xKxlpueX4ujgCGAjuY2W7ufn0F\nQy27PD8XHyB0724N/LSCYZZdPnlw92nR9mjgNY++OWtJnp+LAcCRQC9ymEuwKopHB/oCq7O2VxPm\nxvoXMDaZkBLTUS5eJ/Tx15OOcjGJCk2umSId5eJXwK+SCSkR7eahdcPdb614RMnq6HNxOXB/rgep\nijGPDtTcXwlFUC5iykVMuQiUhy2VJB/VVDw0N1ZMuYgpFzHlIlAetlSWfFRTt5XmxoopFzHlIqZc\nBMrDlsqTj6SvBsjxioGHCdcgtz4mR/uPAl4grDT4jaTjVC6UC+VCeUjTo5z50NxWIiKSt2oa8xAR\nkZRQ8RARkbypeIiISN5UPEREJG8qHiIikjcVDxERyZuKh4iI5E3FQ1LHzDaZ2ZVZ218zs+klOnaL\nmX22FMfq4jwnmdnfo5mNs/c3tK6tYGb7mNlRJTznjmY2sb1ziZSaioek0TvACWbWO9ou5Z2sBR/L\nzPKZzmccMM7dP7jFyd2fd/eB0ea+wNEljOG/gDM7OJdISal4SBq9C9wAnNP2ibYtBzN7M/q30cx+\nYWY/MbMNZvYtM/ti9Nf/cjPrn3WYwWb2FzP7a+va1WbW3cy+Ha269pKZjWtz3PuBf7UTz6lmttrM\nXjOzy6N9NwHDgZ+Y2b1tXt9gZkvNbGvg+8BZUUvrTDPbzsxuMbPXzWylmR0bvWeMmd1lZr8ClpvZ\ntma2wMzeMLNVZta63PLdwJ7R8e41s13MbFl0jG2iY//DzJ41s8asY99hZgvNbL2ZzYj294j2vxH9\nbvU2tb90oZomRpT6Mgd4xsyuaLO/bcshe/sQwqJPZwHPEFaH609YenYSoRgZ8Elgf2Af4I5osZzR\nhGWL+xP+gp9nZg9Fx/0UoYXwpewTm9nOhJXoRgB/A35mZo+7+3gz2xX4qrs/3d4v5+7/NrPTgP3d\nfXJ0vMuAp4GJQAPwYzN7OHrLsOh3ew7YCJwN/Bn4OGENhgcIa5LPb21tRL9Xa37OAnaKjntw9Hvv\nGj03NDr+euB3ZjYb+BBh5tWdCSvPbdGCElHLQ1LJ3d8AbgMm5/G2J939CXd/Efgj8H13fxN4iPCl\nCeHL9GZ3X+PuvyTMKjqA0FI4H3iDMGHc3kBrl88id1/o7hvanO8A4Bfuvtjd/0YoeIdlPd92NtO2\nrM1rhgPfJbRwngV2B3aLnvuZuy/xsNhZD+BiwtrbzwD9o9UBOzvfocB33f2f7v5zwjKkH4vy8aC7\nL3P3FwjFaxdgBaGYjgc+7O4ruvhdpM6oeEiazSKMHWyftW8jUYs56v/fOuu5f7V5Xev2e2zZys7+\nknXiv86PdPdu0aO7u7cu0frPDuLzNscytmwJFTK+8vE2MSyLjrM+6zVfis7Vz927EYrdNjkcu6Pi\nkl0U3wV6uPub7n4Y8GvgzNbuLJFWKh6SWu7+D+BeQgFp/SJ+HjjazLYndN10z/OwBpxmZv8vWs9g\nV+BPwALgG2b2ETN7v5mNMLOeXRxrEXCEme1rZh8CziC/5V3fBPqaWev/wwXARWbWx8x6mdmJWTFn\n2wZ4C/i3mY0BPhLt3wD0NrPt2jnXo8CU6LhHEloXf2rn2ABmZh81s/2BZcCDhFaRyGYqHpJG2X+x\nfwfok7V9I6Gb6W+EVseGDt7X9nie9fNvCAvitADj3f0d4CbgceApwqpqE7Pe1+5x3f0VwjjK/YQv\n2YfcfV4ev99CwrjCv83sTEJX1D+B5YTxjM+0Ez/AnYSi9xrwiej1uPtrhJbC2migPvt9c4A1hOI7\nE/iSu7/bwe/nQE9CrtcRutKm5fB7SR3Reh4iIpI3tTxERCRvKh4iIpI3FQ8REcmbioeIiORNxUNE\nRPKm4iEiInlT8RARkbz9HzlzZSeDdKFEAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f10e63ba978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.loglog(iterations_numbers, results)\n",
    "plt.xlabel('Number of iterations')\n",
    "plt.ylabel('Error |y - l1|^2')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### So what ?\n",
    "\n",
    "There's clearly 2 zones on the convergence diagram. An **interesting track** could be to **figure out why**.\n",
    "\n",
    "Another track leads to an analysis of the position of the bend : does it move ? Is it stable with respect to iterations ? to error order of magnitude ?\n",
    "\n",
    "Finnaly, it's possible to train a network and stop when overall $\\mathbb{L}_2$ error is below a *goal error*.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stability of the slope singular point\n",
    "\n",
    "To be tested on different re-trained NN but also on a single n-times re-trained NN."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Let's re-use the above show code into a function\n",
    "# so one can re-run the same experiment multiple times\n",
    "\n",
    "\n",
    "def slope_singularity_exp(iterations_numbers, NN=None):\n",
    "    \"\"\"\n",
    "    Re-train the same NN for different numbers of training iterations.\n",
    "    Store and return the mean L2 distance between guesses and expected output\n",
    "    \n",
    "    If a NN instance is passed as a parameter, this network will be used \n",
    "    instead of creating a new one.\n",
    "    \"\"\"\n",
    "\n",
    "    if not NN:\n",
    "        NN = TwoLayersNN(X, y)\n",
    "\n",
    "    results = []\n",
    "    for it in iterations_numbers:\n",
    "        NN.train(it)\n",
    "        guesses = NN.test(X)\n",
    "        results.append(np.mean(np.power(np.abs(y - guesses), 2)))\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Different NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running experiment no.0\n",
      "Running experiment no.1\n",
      "Running experiment no.2\n",
      "Running experiment no.3\n",
      "Running experiment no.4\n"
     ]
    }
   ],
   "source": [
    "iterations_numbers = list(range(1, 20000, 250))\n",
    "\n",
    "global_results = []\n",
    "\n",
    "for i in range(5):\n",
    "    print(\"Running experiment no.{}\".format(i))\n",
    "    global_results.append(slope_singularity_exp(iterations_numbers))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEWCAYAAACe8xtsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVNX7wPHPA+4r4q6oLOZSaq6UlUW5pFlZ2bftq2ml\naZZLVpplqa2almZlWln4q8z2TNstMS0Ll1xyF0RFUXFBEUW28/tjBi7yVWFgmDsMz/v14iX3zsy9\nzzyvcR7OOfeeI8YYlFJKKVf42R2AUkqpkkeLh1JKKZdp8VBKKeUyLR5KKaVcpsVDKaWUy7R4KKWU\ncpkWD6WUUi7T4qGUUsplZewO4HxEpDIwCzgDRBlj5tscklJKKSdvbnncBnxmjHkQuNnuYJRSSlk8\nWjxEZJ6IHBSRjXn29xSRPSKSJCJjnbsbAnudv2d6Mk6llFIX5umWx2yge+4dIuLv3D8AuBQYIiIt\ngXigkfNp3txCUkqpUsejX8rGmJXAiTy7w4EYY8xSY8xuIBLoA3wF9BWRWcC3noxTKaXUhXnDgHlD\nYFeu7V1AuDHmFHB/fi8WEZ0WWCmlCsEYI4V9rTd0BxX5y98Y47afCRMmuPX5F3r8XI/lty/v4xd6\nTHPhu7lwZVtzobk413ZR2VE88ka9DwjJtR2KY7yjwCZOnEhUVFQRw3KIiIhw6/Mv9Pi5HstvX97H\nc2/HxcVdMBZXaS4uHEtRnl/UXLiyrbmwtjUXEURFRTFx4sQLxlEQ4o4K5NIJRYKBRcaY1s7tMkAM\ncJ/z36VAb2PMlgIez3j6PXirgQMHEhkZaXcYXkFzYdFcWDQXFhHBlJRuKxFZAsQCl4hIloiMMMZk\nAENxDJSvB94taOHI5s6WR0k2cOBAu0PwGpoLi+bCormg5LY83E1bHkop5boS1fJQxUtbXxbNhUVz\nYdFcuI9PFA/ttlJKqYLRbisn7bZSSinXabeVUkopj/OJ4qHdVg6aA4vmwqK5sGgu3Ndt5Q3TkxSZ\nOxKhlFKlQUREBBEREUyaNKlIx9ExD6WUKoV0zAPttlJKqYLSq62ctOVhiYqKcnmuHV+lubBoLiya\nC4u2PJRSSnmctjyUUqoU0paHUkopj/OJ4qED5g6aA4vmwqK5sGgu9D6Ps+h9HkopVTB6n4eTjnko\npZTrdMwD2PzfzaTuSbU7DKWUKjV8onhUDKvI6nariR0fS8bJDLvDsY3251o0FxbNhUVz4T4+UTzm\n+c3j1NunOLP7DNHNokmYm4DJ1K4spZTKS+8wd8o75nFi1QliRseQkZxB09eaUuO6GjZGp5RS3qmo\nYx4+VzwAjDEkfplI7JhYKreqTNi0MCo1q2RThEop5X10wBxg6FBISMjZFBHq3F6HTps7Ub1LddZe\nsZYdo3aQfjTdxiCLn/bnWjQXFs2FRXPhPr5RPKpWhVat4KmnICkpZ7d/BX8aP9GY8C3hmDRDdIto\n9s7YS1Zalo3BKqVUyecT3Vajf3yC8SH9qDHldVi0CMaMgUcegQoVznpuyqYUYh6P4XTMacKmhlHz\n5pqIFLrVppRSJZZ2WwFvbA+j4Ve3MqFfY07+vBhWrIBmzeCDDyAzM+d5lS+pTJsf2tB0ZlNin4pl\nfdf1JK9LtjFypZQqmXyiePRp3ZW0Nh8w5d8a1P/xLl55/EpSP4p0FI82bWDhQsjVwqrZsyYd13ek\n9h212dBzA1sf2MqZhDP2vQE30f5ci+bCormwaC7cxyeKxyVRH/GxfypdL76e020+4Nm1WTRY+SBv\nTb2T9JdfhGeegauucrRInPzK+NFwaEMu23YZZWuVZVWrVcQ9H0fmqcwLnEkppUo2vc/DKe+lujtP\nnebBFXEsSztEuV2/Ue30AqZcN5b+m/zwnzARWreGl15y/JvL6djTxD4Zy4m/ThD6cih17q6D+Ol4\niFLKN+l9HueZGHHTyRQGrYgjOj2RsnHfUefMj0zvOo5blx3Cb/JkuP56eO45aNLkrNcd/+M4Ox/d\nCUDYa2EEXBXgkfehlFKepAPm53FJlcqs7HkJf3fpQIuw/uxrOpN+y/+ghfmQn75/A9O4MbRvD48+\nCocP57yu+pXVaf9Xe4JGBrHlni1sumMTp2NP2/hOCk77cy2aC4vmwqK5cB+fLR7ZOgZUZd0NbYjq\n0p7g0MHENH6FW5Z/R7s6S/njx3chPR1atIDnn4eTJwEQP6Huf+sSvjWcym0qs6bTGmLGxJBxvPRO\nuqiUUrn5bLfV+fx88BhDVu1iz6kjlE14l3bljzG79UNc+tYXsHSpY3B98GAoWzbnNWf2n2HXM7s4\n8t0RgicGU39QffzK+HzdVUr5MB3zKMRiUMYYvt5/lOH/xHIg+QhlD77JVVXL8HaTe7ho2vuwcye8\n8ALccQf4WUUieV0yMaNjSDuYRtirYdTsWdPdb0cppTxCxzwKQUS4rWFN4nt3ZN4VHah60TMsLTeI\nVhtnc8t/A4mf+iy89hp07Ag//5xzj0jVtlW59NdLCX05lJ0jdrKh1wZSNqXY/G4s2p9r0VxYNBcW\nzYX7lMrikU1E6NekNgduCGfWFZ2o1OxFFmXeQVjsbPqNbkPiqMEwfDh06warVuW8ptbNtej0bycC\neway7tp1bB+2nbTENJvfjVJKeY5Xd1uJSAjwNFDdGPOf8zzHbWuYp2dl8cb2g0yIjeXUgf2UOTqZ\n+5t0ZkpCMNWmTIcrr3R0ZzVrZr3maDpxz8Vx6ONDNBrTiKARQfiVL9U1WSlVApSKMQ8R+dwTxSPb\nmawsXtm8n5f3xnEmfjdlj73CqKY9eHZTJSrNnAV9+8KECVC/fs5rTm07RcyYGFI2phD6Sii1+9bW\nSReVUl6rRIx5iMg8ETkoIhvz7O8pIntEJElExnoiloIo7+fHM62CONyjM0906gKt32Bq4sXULPcl\nL88ZTFql8v8zBXyl5pVovbA1zd9tzu4XdrPu6nWcWHXCo3Frf65Fc2HRXFg0F+7jqf6V2UD33DtE\nxN+5fwBwKTBERFqKSH8RmS4iDTwU23lV8vdnctvGHOp+BQ+3v5bMS2fxzKFGBNZdwttzHiYjYb+j\nC2vaNEhNBaBG1xp0XNORevfV498+/7Kl/xZS96ba/E6UUsq9PNZtJSLBwCJjTGvndmfgBWNMV+f2\ns0CaMWZyrtcEAi8BXYH3jDFTznFcM2DAAIKDgwEICAigbdu2REREANZfGu7YPpqeTr93P+enk4eQ\nqoYqx+cy/FRHrv1tB9ft2QuTJhHVuDH4+xMREUFGcgafDfuMIwuPcOOoG2k0phErVq9wWzy6rdu6\nrdsF3Y6KiiIyMhKA4OBgJk2aVDLGPM5RPG4HehpjBjm3+wPhxpjhLh7X7WMe+TmYlsao6D18nhwP\nu/6kRvKnzKt7C70+WIocPuKYePHmm8E55pG6J5XYp2JJWppEyAsh1Lu3HuKv4yFKKfuUiDGP83Db\nN/7EiRNzKqwn1C1Xjk+uakrctZfT++K+HGszg5tPZRF0QworHrkFkz0F/PLlAFRoXIGLP7qYVl+1\nIuG9BNZ0XMOxpcfcHpcnc+DtNBcWzYVFc+G+Kdk9WTzyFot9QEiu7VAgvjAHnjhxYk4zzZOCKlRg\nYURztl1zOdc2v4eDradxbVYqzfpVYcMtV2D694cbb4SNjusEql1WjXYr2tF4XGO23b+Njbds5NSO\nUx6PWylVekVERJSs9TzO0W1VBogB7nP+uxTobYzZ4uJxPd5tdT6bT6bw4Mo4/jpzAIn7kktOb+aL\nk21oOvtT6NnzrCngM1Mz2ff6PvZM3UO9/vVo8mwTytYoe8HjK6WUu5SIbisRWQLEApeISJaIjDDG\nZABDgUhgPfCuq4Ujm6e7rc7n4iqVWdH9EqKvDqfNRUPZ2Hw8LWufIOLxtuwPLHfWFPD+FfxpPLYx\n4ZvDyUrNIrp5NPEz48lKz7L7bSilfJi7uq1KxE2CF+JNLY+8/jh2nMF/xbItNRGJf4+bTp1i7vZq\nBC78GUaOdBSSKlUAOPnvSWIeiyE1LpWwaWHUvLGmyzcZRkVF2dJ95400FxbNhUVzYSkRLY/i5i0t\nj7yurFGdzb3a8VOXTjQJG8O3YQ9T99JkHnisCyfXr4KLLoJZsyA9nSqtqtDmxzY0ndGU2LGxrO++\nnpPrT9r9FpRSPkZbHk7e3PLIzRjDooNHGbY2hoTkg/glvMGjydV4btlhKsTFnzUFfFZGFgnvJBA3\nKY6aN9Uk5IUQytcrb/dbUEr5kFIxt9WFlJTikc0Yw4J9hxm5YSeHkw7gf/B1XjxWm0e/j6FsFjB5\nMnTvDiKkJ6Wz58U9JLyfQKPRjQgaHYR/RX+734JSygdotxXe2211LiLC3UG1Seh1Oe9ecQVVmr/I\nkxf9h8p3lef97mFkPvJwzhTwZQPKEjY1jA7RHUj+J5noFtEcnH+Q8xXLkpIDT9BcWDQXFs2Fdlvl\nKGktj7wysrJ4O/YgT+3YScqh3ZQ/+Drz4xvR58u/8LviSnjxxZwp4JOWJxEzOgb8oelrTal+RfWz\njqWDgRbNhUVzYdFcWLTbqoQXj2xnsrJ4bVsCz++OITVhB9X3v81Xu+oTsfAv5LbbHFPAN2iAyTIc\n/Pggu57aRbUrqhE6JZSKwRXtDl8pVcJo8fCR4pHtVGYmL23ex7R9u0jbt4n68XNZtLU27X5ehQwZ\nAmPGQEAAmSmZ7H11L/Gvx9PgwQY0HteYMtXK2B2+UqqE0DEPStaYR34q+fvzQuvGJHa7klHtbyDx\n8lfp2LUHLYcGs33TSoxzCnh//3SCnw2m08ZOpB1II7p5NF+M/oKsDL3JELRvOzfNhUVzoWMeOXyt\n5ZHX0fR0xq3bw/tH95C550/Ct3/Dl2v8abBzPzJpEtx7L/j7k7w2mQX3L6BtZlvCXg0jsEeg3aHb\nSvu2LZoLi+bCot1WPl48sh1KS+OxNbv55EQ8Wbt/pdfGn/nw7zMEpmTkTAFvgMMLDxP7RCwVm1Uk\nbFoYlVtWtjt0pZQX0uJRSopHtvjUVEas3s23J/eRFfcdA9as4M3lR6hcs67jHpEuXchKy2LfW/vY\n89Ieat9Zm+CJwZSrVc7u0JVSXkTHPPCtMY/8BFWowFdXNWf7NZ3p0eIBPrztJaqP7MHY0HR+7nsr\n3Hgjfts20ejRRoRvDUf8hFUtV7Fn2h6yzpSe8ZDS8nkoCM2FRXOhYx45SlvLI68tKSkM+TuWP08f\ngJ9f5eV9R3j09zjK9OzlmAI+OJiUrSnEPhFLypYUwl4Jo9attVyedLGk0b5ti+bCormwaLdVKS8e\n2dYln2TwXztZe+YgFWI/4M2ft3Pvn7vwHzAQnn4aatXi6JKjxDwWQ5nqZQh7LYxqHavZHbZSyiZa\nPLR4nGVl0gkG/72DLakHqbbjXeb9EMuN6/biN+pRePRRTMXKJHyQQNwzcdToUYOQF0OoEFTB7rCV\nUh6mYx4qR1RUFJ0DqvHv9R345aorCGw1jlsfeYGgxy9jxQ8LyGoahsyeRYN7axG+PZzyQeVZfelq\ndk3YRWZKpt3hu5X2bVs0FxbNhfto8fBR19UMYOf1nVh45RX4tXuWa0a/QIshrfn3nelktWxBme++\nIPT5YDqu7cjpHaf5u/nfHJh3AJOlrTilVP58ottqwoQJRERE6EDYeRhj+OLAYR5Zt53EE/tov+ZN\nvvw6jkZVAvGb8gp0787xv08QMzqGrDNZNH2tKQHXBNgdtlKqGERFRREVFcWkSZN0zKOkvwdPyTSG\nD+MPMXrTdpKO7ua6P2ex4JsYAptdjN+UVzAdO5L4WSIxY2Oo2r4qoa+EUqlpJbvDVkoVAx3zUDny\n68/1F2Fgo7oc6nElb3buyt83vkzdyS9yT3AKyb16YP7zH+q0SyJ8SzhVw6uy9vK17HxsJ+nH0j3z\nBtxI+7YtmguL5sJ9tHiUQmX8/BgWUp/DPbow+bJefHvHKwS++jyP+O/idHhH/EY+RJN7yxK+KZzM\nk5lEt4gm/s14stJLz02GSqkL024rxanMTF7Zto/Je2Ng3xrGfP0hTy/fRbmHHkbGjuXk3jLEjI7h\nTPwZwqaFEXhDoM/fZKiUryvW+zxEpC3wLlAX+AJ42hhz2vlYmjHG9gmTtHi4T3JGBs9tjuf1hFjK\n7fmDlxYsYMg/+yn/1NOYhx/m6G+niHk8hvJB5Ql7LYwqravYHbJSqpCKe8zjHeADoA2wG1giIjWd\nj+nKQ16mqP25VcuUYWqbYA5c14V7O9zDY2NnUv/FYXzy+TucCQuh5qFv6fhPO2rdUov13daz7cFt\nnDlwxj3Bu5n2bVs0FxbNhfvkVzwuAT4wxiQZY14Hngd+FZHg4g5M2SewbFlmtQ9jX0QXbuowgP7P\nvU3Y2Hv45bXnSG3TkoZBqwnf0gn/qv6sarWK3S/vJvO0b91kqJS6sPy6rdYDg4wxq3LtuwZHi+Qi\nY4ztA+56n0fx23fmDCPXxPBN8n5C1n/Jgjnf0jKwHpVem8Gp+h2IHRtL8ppkQieHUufOOjoeopQX\n88h9HiLSB8AYszDP/k7AMGPMfYU9sbvomIfn7Dp9mmGrd/JLSgJtVn3CZ28vpn6rNlR+dQZJRxux\nc/RO/Mr5ETY9jOqXV7c7XKXUBRTrmIcxZmHewuHcv8obCoc6W3H354ZUrMgPXVrzb5crqdplBC3m\nfsQNHeuyP6IL5d4eTIfPa9JgaAM23b6JzXdvJnV3arHGcyHat23RXFg0F+5ToG4nEWld3IGokqNF\n5cosi2jHmi5XcqrraBr/33zuqp/F0XZtqL5qEpf90ZSKzSuyuv1qYp+KJSM5w+6QlVJulu99HiLS\nFZhkjLnKMyG5Rrut7Pf38RMMit7CttMHufP7d3ljfhQVHh2J/Hc0sS8d5NhPxwh+Lpj699dH/HU8\nRClvUNz3efQDHgN6GGMSC3uS4qTFw3ssO5bE/as2s+9kAkO+nsPExX9TedIkzoTfy84xu8lIyiDs\ntTACuwXaHapSpV5x3+fxPtDXWwuHOpvd/bnX1AhgZ/fOfHX1tXzZ7xmC3pnD5KVfk3pLC1o9+C/B\nzzZh+5DtbLxpIylbU4o1Frtz4U00FxbNhfvkVzwmAHNFpKInglEln4hwQ61A9va4ig+v7sZbQ57j\notem8s7/zSJ9bDgdph8j4JrqrOuyjh3Dd5B+pORNuqiUKtiYR3+gvzGmh2dCOuvcfYDeQDVgrjHm\nl3M8R7utvFiWMXy8/xAjNm6m7MHtvP7OHK5MyaDeK+8S920NEj9NpPG4xjR8pCF+5Wy/bUipUsMj\na5iLSDdjzJLCnqSoRCQAmGaMGXSOx7R4lAAZWVm8t/cgT2zZQo29G3ln1js0rx5I7TFvE/tWFqe3\nnyb0lVBq3VJLbzJUygM8sp5HUQuHiMwTkYMisjHP/p4iskdEkkRk7AUOMR54sygxlAbe3J9bxs+P\noU3qc6RHBCOv68ttk2fS8z83su6xW6lcbjwXPRdI3LNxrLt2Hclrk4t8Pm/OhadpLiyaC/fxVD/B\nbKB77h0i4u/cPwC4FBgiIi1FpL+ITBeRBuIwBfjBGLPOQ7GqYlTOz4/HwoI40i2C/3a7k24z3ub2\nLh2IHd+dGh1nU/eWqmzsvZEtA7dwZr93TrqolMr/Ut04oCB9QtONMTMveCLHZIqLjDGtndudgReM\nMV2d288CacaYybleMwK4F1gFrDPGzDnHcbXbqgRLzsjgha17mLEvhvC1v/HGnI+p0ue/+JW7j/3v\nHyJoZBCNHmuEf2V/u0NVyqcUtdvqgtOqG2OCC3vgAmgI7Mq1vQsIz3P+mcAFixLAwIEDCQ4OBiAg\nIIC2bdvmTJKY3UzVbe/cXrNiBb2AJ6+LYHzdJnTyD6T9pmhm/96VeoNGs3TpDk69cYq+r/albr+6\nLPt9mVfFr9u6XVK2o6KiiIyMBMj5viwKj60keI6WR1+gV/YguPOqrnBjzHAXj6stD6eoqKicD01J\nlZiWxuMbdrLg8F5u/W0hj3+xmOp3vcSJ31piMgxNpzcloEtAvsfxhVy4i+bCormweGTA/DwnPuri\nS/J+w+8DQnJthwLxhYll4sSJORVWlWy1y5VjXseLib02gozbhnLVW+8wNTmK1GN3UTXiEFv6beHf\n2//ldMxpu0NVqkSKiopi4sSJRT5OfmMefc+x2wACfO7Keh7naHmUAWKA+5z/LgV6G2O2FPSYzuNo\ny8OHOaaB38Ky5ASGfbWA3tGbqNf5dY58KdS/rz6Nn25M2YCydoepVIlTrGMewCfAfP631SDn2Hde\nIrIEuM75exYwyhgzU0SGApE4bgKc4mrhyDZx4kQidDEon+SYBr49206dYkjFmszuk8gTC97g8sZH\nOLXrRaKbHyD42WDqD6mPXxm9yVCp/EQ5F4MqqvxaHuuAu8/1pS4iWd6ykqC2PBxKQ3/uhpMnGbRy\nHdtTEhn/YSQt91amfrmRZBwVwqaFEdgrEBEpFbkoKM2FRXNhKe6Wx0jg5Hkeu7qwJ1WqsNpUqUJ0\n96uIPnGCQeWrk5BymAmREwnOaIkZ9R8qvF6JsFfD7A5TKZ/nsautiouuYV66LTuWxOA/o0lJTuSZ\nD+bR6Gg3AnZfQa3bahMyKYRydcvZHaJSXiW726q41zB/4wKvNcaYEYU9sbtot5UyxvDTkaMMXRmN\n/7EDPPn+pwQdv4Oqe5vR6PFGBI0Kwr+C3mSoVG7FvRjUQM49MC44ise8wp7YXbR4WEp7f64xhm8S\nD/PIX9GUW/4r4//YSaNjA6mW2oDQKaHU/k/tUjnpYmn/XOSmubAU9x3mkec4YRNjzO7CnrA46NVW\nChz/GW6tU5s+N93AM2mpPN3lOhrtjeWhuV+RMro/NWbUpun0plS7rJrdoSplG49cbXXOF4hkGmO8\npg9AWx7qfDKyspi7J57xG9bTattWBkTuoXHCrdTrVY/Ql0Op0LiC3SEqZRuPrOeR54RecYluNi0e\nKj9pWVm8sT2GF3ds46o1G7jjwzM0TLyWJiMa03hsY8pUze+iQ6V8jx3Tk7xU2JMVF52exEFzYMmd\ni3J+fjzW4iL23dCLjrfcxCOvd+KDB/4gOnIxK0L+IGFuAibTd/8A0c+FRXPhoelJcp4k8hUwF8e6\nGllFPqsbacvDooOBlgvlIjkjg+dXrWX2kUMM/GID1y5qTM3aIbSedTE1rqvh2UA9QD8XFs2FxVPL\n0PYABuGYMn0B8IExZlthT+pOWjxUYR1LT2f8ipV8dPIEI9/bzGW/NqfGZQ1pO+tiKjWvZHd4ShUr\nj455iEgtoD/wDLAVeBf4yBiTXtgAikqLhyqqxLQ0xi5ZysIz6YybHkPr6ObU6teINlOaUbamTrqo\nfJPHxjxEpDaOGXAHAj8CM4AbgV8Ke3LlXtqfa3ElF7XLleP9G65nQ69urB4VzOD3MlgV/Qe/Bf3G\n5hdjyUrzqp5al+nnwqK5cJ8CXWYiIt8ATYEPgOuMMUec+xcCh4svvILR+zyUOzSsUIEFt/Uh7tQp\nRpVZzK7DaTw+7U92TttKm1ntaXJX3VJ5k6HyLR69z0NErjXGLC3y2YqBdlup4rItKYmRi78jY3s1\nhr2VTnrVMlw7/2rqXJH/SoZKeTuP3+fhbbR4qOK24eBBRn33Ew3/qsHdH/lzqGUGt33ZnWrBFe0O\nTalCK9YxDxGJE5FdBfixfYJEpf25ubkzF23q1uW3++9l+LhWTJuayKaGfvzRYhlzen1P6vEMt52n\nuOjnwqK5cJ/85rYK9lAcSnm98JAQfns4hGURm3ihyz/0+DKQn4N+4PST9bjz6U52h6eUR/lEt5Wu\n56E8zRjDT9HRzPkqjoFzarCzZSLDf7ubchW9ZuYepc7JI+t5lAQ65qHslJ6RwdjXP6TxRw2peeAY\nl3zShfYRDewOS6l82TG3lfJS2p9r8VQuypYpw2uP3UfrNyqzomsl9t+4kTeGfu+RcxeUfi4smgv3\ncbl4iEhVEWkjesG7Ujm6XnUlU2ddw6IH46n3VTlevfQDjhxKtTsspYpNQe/z+BV4DNgELAeqAb8a\nY4YXb3j5024r5W3mvv85h96rTnBsChVfuYhb7m1ld0hK/Q9PTYx4CqgBXA0EAYuAHcYY26cg1eKh\nvNHeXbuY9tRKui2uwz/dExj/RT/8/LSxrryHp8Y8jgCXAHfgmNfKtokQ1flpf67F7lw0Cglh+kd3\nsmFYLCGrGvBG6/9j25YjtsRidy68iebCfQpaPMYAkcABY0wC0BlYXFxBKeUL/Pz9eXrKgzR9vxzJ\ntQJZf000M19aYndYSrmFT1yqq/d5KG+XmpzMS4M/peMPofx92U6e+uZ+KlfS5W+V53n0Pg8RqQg8\nAFwMVAQMgDHm/sKe2F10zEOVJJ++8gkpc2pxuvxRms5sx/XdmtkdkiqlPDXm8SHQHBgKRAEtgf2F\nPakqHtqfa/HWXNw55m56fxdKSi1D8l17GDf8Y4r7bx9vzYUdNBfuU9Di0QPHuEcm8BHQF7izuIJS\nypfVbRHGE7/dzsHesYR/VJ+nu73HnoRku8NSyiUF7baKBdoBq3GsJpgELNdLdZUqmrWfRvHP+BTS\nyxgyx1Tj4fuutjskVUp4qttqPFALeBxYACwFnizsSZVSDu3vjGDAqi5k1jpE/ccyeGjAHNIzSvay\nt6p0KFDxMMbMN8bEGGMWGmOCjDG1jTFzijs45Rrtz7WUpFyUCajGw8vvx39gAt0Xh/FCtwVErd7t\ntuOXpFwUN82F++jEiEp5iT6v/ZcuPzWh0aGy7O67nSfHfWJ3SEqdl0/c51HS34NSuZnMTObe/gF1\nfgvlj847GT7vLoLqVrM7LOVjin1uKxHxA64C/jbGnCnsiQpDRFoAI3GMt/xqjJl9judo8VA+afX8\n3/l3/ElOV8hAHq3K0MHX2h2S8iHFPmBujMkCvgM8PqubMWarMeYhHJcFX+np85c02p9r8YVcdLzn\nav67PoIzdY9Rb4zh8dvfJC3d9cF0X8iFu2gu3KegYx5TgJEiUqj5FERknogcFJGNefb3FJE9IpIk\nImPP89qbcMyj5V0r7CjlAWWrVmLU0gGkj0yi04oWvHH5J0T9vs3usJQq8H0eJ7GmJTnt3G2MMQXq\niBWRzkDHL1adAAAVn0lEQVQK8LExprVznz8Qg+O+kVgcl//2BjoC7YGpxpj9uY6x2Bhz4zmOrd1W\nqlQ4sH0fC/qtpMHu6mztG88zb92HLsmmCssj63m4g4gEA4tyFY/OwAvGmK7O7WeBNGPM5FyvuQa4\nDSgPrDfGvH2O42rxUKWGycri9cHzCf6yAZsv3ka/T++mcaMAu8NSJVBRi0eBu6FEpA+OxaAMsMwY\ns6iwJ3VqCOzKtb0LCM/9BGPMMmBZfgcaOHAgwcHBAAQEBNC2bducGXaz+zhLw3bu/lxviMfO7ex9\n3hKPu7aX/f47bfsHUf72qiQ8HszbnWaRcasfU99+8ryvX7duHaNGjfKK+O3enjFjRqn+foiMjATI\n+b4sioJ2W03GMb/VLBwD50OAX4wx4wp8ov9tefQFehljBjm3+wPhri5tqy0PS1RUVM6HprQrDblI\nTT3D1Ls+59KlDfjn8q08uWgI5cv5/8/zSkMuCkpzYfHUMrSJQGtjzAHndj3gX2NMLRcCbQIszlU8\nLgdezNVtNQFINcZMcekN6HoeqpRb8Ob3nJlWhlOVkmn2SnO63qhrpqvzi/Lweh5xwC3GmHXO7XbA\n18aY4AKf6H9bHmWwBsxjcA6YG2O2uPQGtOWhFEcPHeW927/joo0N2XTzPsbP6293SMrLeWpixHHA\njyLyfyLyIY7LZgs8MaKILMFxRdUlIpIlIiOMMRk41geJBNYD77paOLJNnDjxrL7u0kpzYCltuQis\nE8iY3/uTMHAvzX+oz9TLPmP75kNA6cvFhWguHDmYOHFikY9T0DvM+wNLcFxGa4BVzrXMbactD4v2\n51pKcy62rt3KLw+spdaBOhwalMalXSuV2lzkVZo/F3l5aszjCNDIGHOqsCcqLlo8lPpfJiuLV++Y\nyUVLWrO+02FGfnEz1atXtDss5UU81W31CjBZREJEJDD7p7AndTfttlLqbOLnx+NfjKLC7Ewa7i7L\nFx1+57PIP+0OS3kBj3VbQc6Aed4nGmNMaJEjKCJteVi0SW7RXFh+/OEHNk7dQbM1rVjXfR9PLbiH\nsmX+95Le0kA/F5Zib3k4xzwmAC2NMSG5fmwvHEqp/FWoWJEnfhvBiSeO0PTv2rzdcTF/rSjUtSlK\n5fCJMQ+9z0OpgjkQd4j5dy0meEcw2+6KZ9xb99odkvIwT9/nMQYIAqYDx7P3G2OOFvbE7qLdVkq5\nbuagSBosbERc2EHu+qQbQSF17A5JeZinBsyHATcBvwFrcv0oL6IXDVg0F5Zz5WLEewMJ+6oR/ull\n+aPzev7vma89H5gN9HPhPgUqHsaY4DzjHSHGmJDiDq6g9GorpVzXrkszRkb3ZfO1cVR9szqvd36f\n1OOn83+hKtE8crWViMw0xoxw/j7MGDMr12PzjTH3FDmCItJuK6WK7otPf2ff5GTqHoQ6T1fhuoev\nsTskVcyK9SZBEckyxvjl/f1c23bR4qGUe5w4fZrn+8+nyy8h7Oq8k+GLH8CvlF7SWxp4asxDlQDa\ndWfRXFgKmotqFSsy9YsHiJt4goCtwcxpu5A1v2wo3uA8TD8X7uMTxUPHPJRynxGP3sJ1f3TgePUU\ndt2dyNQhc+0OSbmRp8Y8snCsPQ5QOdfvAJW120op3/baw/MJ+aQu2y4+yB1zuxDavJHdISk3KTFr\nmBcXLR5KFa/1f8ewYtAaqh+rwYmBxxj2wh12h6TcQMc8VA7turNoLixFzcWll4UxbMN/iL9iDw3f\nqMVL10aSdCTZPcF5mH4u3EeLh1IqXyLCk589QLUPqhB4sAbftv+DT6b/YndYykbabaWUckl6ejqT\n+75Hm2XN2dYxnlGL7qZcpbJ2h6VcpN1W6NVWSnlS2bJleebbh0iddprq+wL5sM3PfPfhH3aHpQrI\no+t5eDNteVh0rQKL5sJSnLlIPpnCzFvm02p1GOu6b2P8Jw/i78U3FurnwqItD6WUbapWqczTSwaT\n9MQpWiwPY+Zl37D8l3V2h6U8QFseSim3OLz/OPNuW0jIjiC29tnDU+8PtDskdQF6n4cWD6W8ypsj\n5lPn0zrsa3SU7nM60qqDLjrqjbTbSuXQiwYsmguLp3PxyMx7aLekOVl+mWy7fhczhs7z6PkvRD8X\n7qPFQynldhe1bsRj0Xez47Z9NPo8iFcv+5B9u47YHZZyI+22UkoVq+jlm/hz1Fbq7a9K8pAUBk+8\n1e6QFNptBeh9Hkp5s/AulzAi+hZiuuymzvTqvNg1khNJJ+0Oq9TS+zyctOVh0WvYLZoLizfl4qdP\nVrJnYiLl0spRdiTcM6qnR8/vTbmwm7Y8lFIlxvV3d2bgxhvYe/FeKk8ox8s93iUtNc3usFQhaMtD\nKWWLz9/5lRNT04AM6j1bg979r7I7pFJF7/PQ4qFUiXUy+TQzbvuE1tHBbL4mjie+vJcyZcvYHVap\noN1WKodeNGDRXFi8ORdVqlZk/C/3c/jJEzTY0JC5bRYTtXhtsZ3Pm3NR0mjxUErZ7oFxt9D778s4\nVieJpH5JvHj3XLRDwbtpt5VSyqvMHvcZAZE1OFTvOFfObE2HLs3tDskn6ZiHFg+lfM7umAN83m8J\nodvrs/3WvTz53kC7Q/I5Pj/mISKVRWSViPS2OxZvp/25Fs2FpSTmoklYPR5f2Y/4u/YRtjCIV8MX\nsG3j7iIftyTmwlt5ffEAxgCf2h2EUsrzRrx1Lxf/GIp/JmzouoPXh39sd0jKySPdViIyD+gJHDLG\ntM61vyfwDlANeNkYMyXP67oDgUAF4LAx5rtzHFu7rZQqBaY88AFNv2nMrmYHuGv+9QSF1LI7pBKt\nRIx5iEhnIAX4OLt4iIg/EAPcB8QCS4HeQEegPTAVGAZUBi4GTgO35q0UWjyUKj2if9/EXyO2UPtQ\ndU4OOMbgl++wO6QSq6jFwyN34xhjVopIcJ7d4UCMMWYpgIhEAn2MMZOBD53PGe98bACQeL4qMXDg\nQIKDHYcPCAigbdu2OfPXZPdxlobt3P253hCPndvZ+7wlHju3161bx6hRo7wmnqJsn8pKpPVrgfzx\nbiyt3g7lwe/G0ntiF/rcdmOBXj9jxoxS/f0QGRkJkPN9WRQeu9rKWTwW5Wp53A70NMYMcm73B8KN\nMcNdPK62PJyidNK3HJoLi6/m4tdv17B9XDyVT1Ygc+gZ7ht3c76v8dVcFEZJvtrKbd/4OiW7g/6n\nsGguLL6ai643d2Dwut7Et91D4OQqvHTd+yQfP3XB1/hqLlwRVdKmZBeRJsDiXC2Py4EXjTFdndsT\ngNS8g+YFOK62PJQq5b6f/wfxzx2l3JkylB/tz93De9gdktcrSS2PvEGuBpqKyHXOwjIA+NaD8fgc\nbX1ZNBeW0pCLG+65kvs2XM++lvFUfroML/Scy5lzTPVeGnLhKR4pHiKyBMcVVZeISJaIjDDGZABD\ngUhgPfCuMWZLYY6v3VZKqbLlyvH094NJn2qoH1Ofj1r9zLfzltkdltcpcd1WxUW7rZRSeaWcTGXG\nrR/T+u8QNl4dy9ivB+pU73mUpG6rYqMtD6VUbpWrVODpXx7g+LOnCdoUxNw23/PzF3/ZHZZX0JaH\nk7Y8LHoZokVzYSntuUg6coLZfb+k5T9N+Lbt98xZMllbIWjLQymlLiigZjWejLqPxEeTqL+pLnM6\nLOL3H9bZHVaJ5xMtjwkTJhAREVGq/7pSSuXvYPwR5t31HRdtCmJrr1jGzR9kd0geFxUVRVRUFJMm\nTfL+ua2Kk3ZbKaVc9dbYT6n5f4Ecqnucy2e0JDziErtD8jjttlI59KIBi+bCormwZOfi4Sl30nl5\na9Iqnia+7wGm3DvX3sBKIC0eSqlSqUnTejy+sj97+h8k5KcmTO/0KRtWxdgdVonhE91WOuahlCqK\nHRvj+XbQcoJj6rCrbzyPzxlgd0jFRsc8nHTMQynlLq8NmUfjL4PYHXaQm967imatG9sdUrHRMQ+V\nQ/u2LZoLi+bCkl8uRs8ZQPOfQvDL9Gd91+3M0GVvz0uLh1JK5dK6QyiPrr6TXb330Hh+PaZd/hFx\nOw/YHZbX8YluKx3zUEoVh9XLt/DniH+pcyCAxH6HGT71brtDKjId83DSMQ+lVHGbfM/7NP8hmB0X\nx9Pvkxto0LiW3SEVmY55qBzat23RXFg0F5bC5uLJ+fdT99PaVDpZhajOa5k9/gv3BlYCafFQSqkC\nuKJHax5a04fdV+yh3puBTL46ksSEY3aHZRvttlJKKRdFfbuaf56L4eoZl9LhqhZ2h1MoRe220uKh\nlFKlkI55oItBZdMcWDQXFs2FRXPhvsWgfGJFFHckQimlSoPs2xomTZpUpONot5VSSpVC2m2llFLK\n47R4+BDtz7VoLiyaC4vmwn20eCillHKZjnkopVQppGMeSimlPM4niofe5+GgObBoLiyaC4vmQu/z\nOIve56GUUgWj93k46ZiHUkq5Tsc8lFJKeZwWDx+i/bkWzYVFc2HRXLiPFg+llFIu0zEPpZQqhXTM\nQymllMd5dfEQkQgRWS4ib4vINXbH4+20P9eiubBoLiyaC/fx6uIBZAHJQHkg3uZYvN66devsDsFr\naC4smguL5sJ9PFI8RGSeiBwUkY159vcUkT0ikiQiY8/x0uXGmBuAJ4Gi3dFSCiQlJdkdgtfQXFg0\nFxbNhft4quUxG+iee4eI+Dv3DwAuBYaISEsR6S8i00WkQa6R8CQcrY9i52qzNr/nX+jxcz2W3768\njxdnM1xzUfhjF3cuXN12J81F4Y/tS7nwSPEwxqwETuTZHQ7EGGOWGmN2A5FAH2PMh8aYR40x+0Xk\nVhGZDfwf8IYnYvW2D0PefRf6MMTFxV0wFldpLi4cS1Ge78kvCc2Fta25uPD5XeGxS3VFJBhYZIxp\n7dy+HehpjBnk3O4PhBtjhrt4XL1OVymlCqEol+raOTGiW770i/LmlVJKFY4nr7bKWyz2ASG5tkPR\nK6qUUqpE8GTLI28LYTXQVESuA2JwDJz39mA8SimlCslTl+ouAWKBS0QkS0RGGGMygKE4BsrXA+8a\nY7Z4Ih6llFJFU+LntlJKKeV53n6HuVJKKS/kc8VDRCo772h/R0TusTseO4lIiIi8JyKf2x2L3USk\nj/MzsUBEuuf/Ct8lIi2c88V9LiJD7Y7HTs7vi1UiUurHW12dS9DnigdwG/CZMeZB4Ga7g7GTMWZX\n9n00pZ0xZqHzMzEUuNPueOxkjNlqjHkIRx6utDsem40BPrU7CC/h0lyCJaJ4uDg3VkNgr/P3TI8G\n6gFFmCfM5xQyF+OBNz0XpWe4mgsRuQlYDHzv6ViLkyt5cLZANwOJdsTqCS5+LlybS9AY4/U/QGeg\nDbAx1z5/IA64FmiC42qulkA/oLfzOZ/YHbuducj1+Od2x213LnBcKj4F6Gp33HbnIs/rFtsdu42f\niReA6cBPwDc4LyDypZ9Cfl+UK8h3hp13mBeYMWalc3qT3HLmxgIQkUigDzATeNPZh/mtB8P0CFdy\nISIHgZeAtiIy1hgzxZOxFjcXPxfdgK5ANRFpaoyZ48FQi52Ln4s6OLp3ywPfeTDMYudKHowx453b\nA4BE4/zm9CUufi5aANcDARRgLsESUTzOoyGwK9f2LhxzY50C7rcnJNucLxdHcfTxlybny8VwPDS5\nphc5Xy6WAcvsCckW58xD9oYxZp7HI7LX+T4Xk4GvC3qQEjHmcR4+91dCEWguLJoLi+bCQfNwNrfk\noyQVD50by6K5sGguLJoLB83D2YolHyWp20rnxrJoLiyaC4vmwkHzcLbiyYfdVwMU8IqBJTiuQc7+\nGeHc3wvYg2OlwXF2x6m50FxoLjQP3vRTnPnQua2UUkq5rCSNeSillPISWjyUUkq5TIuHUkopl2nx\nUEop5TItHkoppVymxUMppZTLtHgopZRymRYP5XVEJEtEpuXaflxEJrjp2JEi0tcdx8rnPP8RkUPO\nmY1z7w/OXltBRC4VkV5uPGd1EXnoXOdSyt20eChvlAbcKiI1ndvuvJO10McSEVem83kAeMAYU/es\nkxsTZ4xp7dxsB9zgxhhqAMPOcy6l3EqLh/JG6cA7wKN5H8jbchCRk85/I0TkZxFZKCIpIvKyiPzX\n+df/FhEJzXWYcBHZLSJ7s9euFhF/EZnqXHVtn4g8kOe4XwOnzhHP3SKyS0QSRWSyc997QA9goYh8\nluf5wSKyUUTKA+8DDztbWsNEpJKIfCAiR0UkRkRudL5moIjMF5FlwBYRqSgiP4lIsojEikj2cssL\ngEucx/tMRJqIyL/OY1RwHvuYiGwWkYhcx/5IRJaKyAkRmeLcX8a5P9n53krb1P4qHyVpYkRVuswC\nNojIK3n252055N6+AseiTw8DG3CsDheKY+nZ4TiKkQBXAR2AS4GPnIvlDMCxbHEojr/gF4nIL87j\ndsHRQuiX+8Qi0gDHSnQ9gQPA9yKy0hgzSETCgMeMMWvP9eaMMWdE5D6ggzFmhPN4LwFrgYeAYOAb\nEVnifEl353vbAWQAo4CdQHMcazB8i2NN8sXZrQ3n+8rOz8NAoPO4nZ3vO8z5WFfn8U8Aq0VkJlAP\nx8yrDXCsPHdWC0opbXkor2SMSQb+Dxjhwsv+Nsb8ZYyJBzYB7xtjTgK/4PjSBMeX6VxjzGFjzK84\nZhVtgaOlMAZIxjFhXBsgu8tnlTFmqTEmJc/5OgE/G2PWGWMO4Ch4V+d6PO9spnlJnuf0AF7H0cLZ\nDFwENHU+9r0xZr1xLHZWBngex9rbG4BQ5+qAFzrflcDrxpjjxpgfcSxD2syZjx+MMf8aY/bgKF5N\ngG04iukgoL4xZls+70WVMlo8lDebgWPsoHKufRk4W8zO/v/yuR47led52duZnN3Kzv0la7D+Or/e\nGOPn/PE3xmQv0Xr8PPGZPMcSzm4JFWZ8pXmeGP51HudEruf0c54rxBjjh6PYVSjAsc9XXHIXxXSg\njDHmpDHmauB3YFh2d5ZS2bR4KK9ljDkGfIajgGR/EccBN4hIZRxdN/4uHlaA+0SktnM9gzBgK/AT\nME5EgkSkioj0FJGq+RxrFdBNRNqJSD1gCK4t73oSaCgi2f8PfwKeE5FaIhIgIrflijm3CsBp4IyI\nDASCnPtTgJoiUukc51oOjHQe93ocrYut5zg2gIhIIxHpAPwL/ICjVaRUDi0eyhvl/ov9VaBWru13\ncXQzHcDR6kg5z+vyHs/k+n0FjgVxIoFBxpg04D1gJRCNY1W1h3K97pzHNcYk4BhH+RrHl+wvxphF\nLry/pTjGFc6IyDAcXVHHgS04xjNuOUf8AB/jKHqJQEfn8zHGJOJoKRxxDtTnft0s4DCO4jsd6GeM\nST/P+zNAVRy5TsLRlTa+AO9LlSK6nodSSimXactDKaWUy7R4KKWUcpkWD6WUUi7T4qGUUsplWjyU\nUkq5TIuHUkopl2nxUEop5bL/B2JT4KpNCZIRAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f10e635e2b0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.hold(True)\n",
    "for res in global_results:\n",
    "    plt.loglog(iterations_numbers, res)\n",
    "\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"Number of iterations\")\n",
    "plt.ylabel(\"Error as |y-l1|^2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems that different NN of the same type exhibit a bend at the same point. What's interesting in this point is that we now can infer the explanation for this bend point can be found in the underlying maths.\n",
    "\n",
    "At low numbers of training iterations, the results are slightly dispersed. This issue may be unseen using only one single NN."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Single NN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running experiment no.0\n",
      "Running experiment no.1\n",
      "Running experiment no.2\n",
      "Running experiment no.3\n",
      "Running experiment no.4\n"
     ]
    }
   ],
   "source": [
    "NN = TwoLayersNN(X,y)\n",
    "iterations_numbers = list(range(1, 20000, 250))\n",
    "\n",
    "global_results = []\n",
    "\n",
    "for i in range(5):\n",
    "    print(\"Running experiment no.{}\".format(i))\n",
    "    global_results.append(slope_singularity_exp(iterations_numbers, NN))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY8AAAEWCAYAAACe8xtsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4FFX3wPHvSaOH0GsgQXqNNMVGFBEU1Ff52V4LKKiI\ngii+FkAJIgg2sCEKaAREBRvSlKLBhlKDdIEkhBogEBJKSLu/P3bDhBggm2x2Nsn5PE8eMrOzM2fO\ns+zJvXfmjhhjUEoppVzhY3cASimlih8tHkoppVymxUMppZTLtHgopZRymRYPpZRSLtPioZRSymVa\nPJRSSrlMi4dSSimX+dkdwPmISAVgMnAGiDLGzLY5JKWUUk7e3PK4HZhjjHkEuMXuYJRSSlk8WjxE\n5FMRSRCRjbnW9xSReBFJEpHnnKvrAXucv2d6Mk6llFIX5umWxxSge84VIuLrXN8XaAc8KiItgL1A\nsHMzb24hKaVUqePRL2VjzEogOdfqzsAuY8zPxpjdQCRwK/AN0EdEJgPfezJOpZRSF+YNA+b1gNgc\ny7FAZ2PMKeChi71ZRHRaYKWUKgBjjBT0vd7QHVToL39jjNt+Ro0a5dbtL/R6Xq9dbF3u1y/0muai\n5ObClWXNheYir+XCsqN45I56HxCaY7kRjvGOfIuIiCAqKqqQYTmEh4e7dfsLvZ7Xaxdbl/v1nMtx\ncXEXjMVVmosLx1KY7QubC1eWNRfWsuYinKioKCIiIi4YR36IOyqQSwcUCQHmG2PaOJf9gF3Ag85/\nfwZ6GWO25nN/xtPn4K369etHZGSk3WF4Bc2FRXNh0VxYRARTXLqtRGQZEAO0EpEsERlijMkABuIY\nKN8ATM1v4cjmzpZHcdavXz+7Q/AamguL5sKiuaD4tjzcTVseSinlumLV8lBFS1tfFs2FRXNh0Vy4\nT4koHtptpZRS+aPdVk7abaWUUq7TbiullFIeVyKKh3ZbOWgOLJoLi+bCorlwX7eVN0xPUmjuSIRS\nSpUG4eHhhIeHM3r06ELtR8c8lFKqFNIxD7TbSiml8kuvtnLSloclKirK5bl2SirNhUVzYdFcWLTl\noZRSyuO05aGUUqWQtjyUUkp5XIkoHjpg7qA5sGguLJoLi+ZC7/M4h97noZRS+aP3eTjpmIdSSrlO\nxzwA5s8HLSBKKeUxJaJ4nHj8OUz37rBhg92h2Er7cy2aC4vmwqK5cJ8SUTwanezDkH9ak37dDfDw\nw3DwoN0hKaWUV9I7zJ1ExKSlGaZOhUkRSbxdayw37PsE32eehqeegnLl7A5RKaW8jo55AP7+MGgQ\nrN4RxC+9X6dz1l9snrmWrGbN4fPPdTxEKaXcrEQUj2yVK8Orr8K3f1/C+I5f0+fkTA4//yamyxWw\ncqXd4RU57c+1aC4smguL5sJ9SkTx+LvX36REp5xdbtAAZs6EkUuu4c6QVYzY+xipN9+Buece2L3b\nxkiVUqpkKBFjHnve2UP8uHiCwoMIeTmE8k3Kn33dGFiwAF4adpKnMl7nv0ffxW/Qo/D88xAYaGPk\nSillHx3zAOoPrk/nHZ2p0KYC669Yz/ZHtpO6NxUAEbj5Zli1uQInnomgo//f/DZnP5lNmsHUqZCZ\naXP0SilV/JSI4gHgV9GPhsMb0nl7Z/yr+bOm3Rp2DttJ2uE0wBpUX7GzHovujKR76gLixswks92l\nsGyZzdG7h/bnWjQXFs2FRXPhPiWieOScGNG/qj+NXm1Ep02dyErNYlXzVcRGxJKRnAE4BtXHjYPI\njR14qesKHtkXQfI9j5LV+2bYts3Gs1BKqaKn93k4XWxuq9Mxp4mLiOPoj0dp8FwD6j5WF99yvmdf\nX7sWXnj6DNdvf48nT48n4IF7kIhRUK2aJ8JXSilb6JgHcOB0ynlfK9eoHC1mtCDspzCO/3qcVU1X\nsX/qfrLSswDo0AF+jCpDi6nDCK+xhXnfZZHRuDlMnAhpaZ46BaWUKlZKRPGo93sU3X6ZTdzJY+fd\npkKrCrT+tjWtvmrFoS8OsbrVahK+SMBkmbOD6r9srcGB4e9xne8KNry1jPRmreC774rNTYban2vR\nXFg0FxbNhfuUiOKxoFl94k4eo9HK37kq6jP+ST5y3m0DLwskbHkYTT9oyt6Je1nTfg2JCxMxxuDv\nD489BgtiWjLngYXcnfg+CY+MJKPrdbB+vQfPSCmlvFuJGvP4af8mBv29gn/8GtDRN4npl3anTVDt\n877XGMOReUeIHRGLXxU/Go1rRNA1QWdfj4+Hl4ZnUHP+dCIYRdn/3IjPq2Ohbt0iPy+llCpKhR3z\nKFHFI9vKhO08Gr2cTb7BtPVJYmrYtXSqWv+8+zCZhoTZCcSNiqN8s/KEjgul0qWVzr6+bh28+GQy\nt20dxwNpU/Ef9iTyv2egfPnz7lMppbyZDpjnoUutZvzdYxBrw1rhm5HCZWvX02r5DH49HJfn9uIr\n1L6/Np23dabazdXY2Gsjm+/czKntpwBo3x4W/BJI7cjx3FRjDSsmbybtkuYwaxZkZXnwzC5M+3Mt\nmguL5sKiuXCfElk8sl1avRFrbxjEpo5hBGadpGv0Bpoum8Gygzvz3N4nwId6g+px2Y7LqNi+Iuuv\nWs+2AdtIjU9FBHr3hh+2h7I14kv6nPmcmKHvkNb+cvjtNw+fmVJK2curu61EJBQYAVQ2xtxxnm3y\n/Qzzncf303/NfH7NqkVDknm7ZSduqdfivNunH0tnzxt72D9lP7UfqE2DFxoQUDMAgORkeG18Foff\n+ZwJvi9Q4brL8H9zAjRq5PJ5KqWUp5WKMQ8RmeuO4pEtPuUwA9Z8x7L0qtTlJG81D+POBm3Pu/2Z\ng2eIHxdPwmcJ1Hu8HsHDgvGr7AfAnj3w8vOnaPz9WzxpJhIwsD8+L45w3MqulFJeqliMeYjIpyKS\nICIbc63vKSLxIpIkIs95IhaABpVqsOTah9l/1bVcWiade7b+Q+0lM4mMXZfn9mVql6HJO03osLYD\nqfGp/NXkL+JfjyfzdCbBwTD1s/J0XzGS+9pt4ttpiaSGNMNM/gAyMjx1SoD25+akubBoLiyaC/fx\n1JjHFKB7zhUi4utc3xdoBzwqIi1E5H4RmSgiRX49bO3yVZnftT+HrunOVeUyGLB9B9WXzOKDnX+R\nV2umXEg5WkS2ICwqjOQ/k/mr8V/sm7KPrPQs2reHub/Vocys6fw3aDEbRszhdPMw+PHHoj4NpZTy\nOI91W4lICDDfGNPGudwFeMUY0825/BKQZowZn+M9VYFxQDdgmjFmQh77NX379iUkJASAoKAgwsLC\nCA8PB6y/NPKznJx2kj4fvsJPp30JDGvJyAYNaH8gHRHJe/vVycx5bA5pB9Lo83ofat5dkxW/rCAz\nE3b805U/h3/PLWcep2LTutwwOxJatnQpHl3WZV3WZXctR0VFERkZCUBISAijR48uHmMeeRSP/wN6\nGmMGOJfvBzobYwa7uF+Xxzwu5lR6KkPWfM2MpCzK+vrxbL06DG/ZFR/JO8/Hfj5GzAsxZJ3KInRs\nKNV6V0NESE6GN19NI/2dyYxgLP733EHAq6OhRg23xquUUq4qFmMe5+G2b/ycU7K7Q3n/skzrci8p\n3e/igSBhzO7dVFo6h+Ebl5GRx30dVa6tQvuV7Ql9JZTY4bGsv3I9x6KOERgIo18N4LFtQ/lf723M\n+Nyf06EtyJzwOpw547Z4s7kzB8Wd5sKiubBoLtw3Jbsni0fuYrEPCM2x3AjYW5AdR0REnG2muVMZ\nvwDe63w3J3r8l0FVfZm0J55Ky77i6egfSM9VRESE6rdUp2N0R+o9Xo/t/bezoccGktckExwMk7+s\nRvtf32Zg699Z8cqvnGzYAjP3q2Iz6aJSqmQIDw8vXs/zyKPbyg/YBTzo/PdnoJcxZquL+3V7t9X5\nZGZlMnrDfN46cJh0vyo8VK0cE8N6UNbX71/bZqVlceDjA+wes5vALoGEjgmlQosKGAOLFsHXg5Yz\n4ujT1GocSMWpE6FjR4+cg1JKQTHpthKRZUAM0EpEskRkiDEmAxgIRAIbgKmuFo5s7u62Oh9fH19e\nvvQ/JN/Yn4g65Zh9cC+BP33Pg6vncTIj/ZxtfQJ8qDfQcbd6YOdAortGs+3BbZyJT6VXL/hoVzeW\nv7aOl2L6ceyaWzjZ5wHYW6CGl1JK5Zu7uq2KxU2CF+LJlkduxhje3rKEl+NiSA6owx2Vsvig/U0E\nBZT917YZxzPY88Ye9k3eR617a9FwREMCagWQnAyTxqRQ8b3xDJQp+A55gjIvPgsVKrgcT1RUVJF0\n3xVHmguL5sKiubAUi5ZHUfNUyyM3EWFoqx4k3jSQyQ0CWZ54kGorfuT2P+ZyOPXkOdv6VfYjdEwo\nnbd2Bh9Y1XIVMSNjKJeVzkuvV+LOHWN58aZ1LHx7JyfqNyNzeqRXTbqolCoZtOXhZGfLIy8zd/7G\nc/9sIME/mB7lTzOt/U3ULVfpX9ul7k4l7uU4EucnEjwsmHqD6+Fb3pf162H6w3/Sf8vTNKqTSuD0\niUh4VxvORClVkpWKua0uxNuKR7avYv/k6a1r2BvQkGsDTjC9w42EVAj613Ynt50k7sU4jv9+nIYj\nG1JnQB3E34dFCw3LB87h2cTnKNulPUEfvQaNG9twJkqpkki7rbCv2+pC/i/0cuJveoIFTesSdzKR\nRit/4+oVs/knJfGc7So0r0Crua1oM78NR74/wqrmq0iYdZCbboTX4u5iwWtbeX91Z1JaX07KI8Pg\n2Pmf0+5tObCT5sKiubBoLrTb6ixvbXnk9tP+jc5H5IbQ0S+J6WHdaRNU61/bJa1IImZ4DBnHMwh9\nJZTqt1bnxAnh/ZcSqPPBi/TxnYfv6Bcp9+Sj4O9/znt1MNCiubBoLiyaC4t2WxWT4pHtj4PbeHTD\ncjb7NqCdz3Gmhl1Lx6r1ztnGGMPRRUeJGR6DTzkfGo1rRJXrqrB3L0wZ9DfdfxxG2yp7CfzoDXxv\nvgnOM22KUkqdjxaPYlY8sq07sosB634gWhrQwieJD9tew1U1Gp6zjckyHPryEHEvxVE2pCyhY0MJ\n7BzI+nWGuQ8uYsC2YVRqEUz1GW8hbdvYdCZKqeJIxzzwzjGPi2lf/RLW3fA4Gzu0JTArhWuio2m6\nbOY5j8gVH6HWPbXotKUTNe6owabbN7Hp9k00LXOKsdG92D53Ix8l3MqxjtdzuM+jRH3zjY1n5F2K\n2+ehKGkuLJoLHfM4q7i2PHLbeXyf4xG5mbVoKCm82/Iyetdrds42macz2T95P/GvxVO1Z1VCIkLw\nDy7HrHePkTryFRqkT+OKp18gKGIolP33jYqlifZtWzQXFs2FRbutSkjxyLY75RAPr/6WZRnVqSen\neKv5pdwR3PqcbTKSM9jz5h72vbePmvfUpOHIhqRVKMO053fQZOpzXFl+HWUmTqB8vzt1PEQplSct\nHiWseGQ7eOooA1Z9zeIzQdSQVMY3aU2/0EvP2SbtcBrxr8Zz8NOD1H2kLsHPBpNw0p9ZA6K4afnT\n1AwuS/WZE/G78jKbzkIp5a10zIPiOeZxMbXLV2VB+MMcuuYGriqbzoDt26mx5DOm7Fx99hG5ATUC\naPxWYzpGdyT9SDpTQqaQOWM3//v6arL+WsOMso+QGH47e7v+F7M73uYz8qyS9nkoDM2FRXOhYx5n\nldSWR27Hz5xg0OqvmHMigIo+8GLDEJ5q2gXJ0S31w8wfCF4YzPEVx2kwogF1BtRl+aJT7Hz0de5N\neo9T9w+kztvPQ6V/T5dS0mjftkVzYdFcWLTbqpQUj2wn008zZM3XzEwylPX159n6dRne4upzHpGb\nsj6F2BGxnNp6ipCIEKrdXYs5E/cSMHoE18sysiLGUG1YP/D1te9ElFK20uJRyopHttSMMzyz9hum\nJp7Bz68cQ+tUZ3Sra/HzsXoik35NInZ4LOmJ6YS+EkqZ66sz++k1tPv0aRpWTaHytLeocPN1Np6F\nUsouWjxKafHIlp6Zzgvrv+P9Q8lkbt7DE927ML7N9QQ4WxXGGI7+cJTY4bGInxA6LpRTzYOY1/cb\nev/yLFktW1N/9uv4t252kSMVL9o9YdFcWDQXFh0wL+X8ff15o+MdpPToy12Vffho324qLf+OQesW\nkZqZgYhQ7cZqdFjbgeBngtnx+A6O9P2b+8Z1J+n3LSw9fTUnwq4k5uYnMYlH7T4dpVQxUSJaHqNG\njSI8PFz/ogCyTBbjNy5kwp59nA6oyb1Bfrwb1pOK/gGO1zOyOBh5kN2jd1OxfUVCxoSyastpDg8a\nRY8TX3Fi8HCCXx0EAQE2n4lSqihERUURFRXF6NGjtduquJ9DUTDGMGnLj4yJiyE5oC53BBqmtL+J\nyv5lAMhMzWT/B/uJHx9Ple5VaPBSKD98HkPVV5+hRcAu/Ca+QY2HbtabDJUqobTbSp2V8xp2EeGp\nVj1JvOkx3g+uxLIj+6kW9QO3r/yaI2dO4VvWl+Cngrls52WUb1qe6CvW0jbBn3Z/f8fS3u+Q+OgL\n7ArpRsqv0fadUCHo9fwWzYVFc+E+WjxKOBHh0ebdONzrcT4JqcLKoweo9esyev0+lwOpJ/Cr5EfI\nSyF03tYZn/I+bO2ymiuDm1B+7Rp+r3cHp8N7suWK/qTHH7D7VJRSXkS7rUqhuTErGbZtDXsDQriu\nzEmmd7iRhuUrA5C6N5XdY3Zz+OvD1B9an+TOgWx9ZDw99kznwD1P0fyjYUj5cjafgVKqsIr0Ul0R\nCQOmArWAr4ARxpjTztfSjDG2j6pq8Si4hfFrGbzpD+ICQrnSP4WP2/egSaWqAJzaeYq4l+I49tMx\nGjzfkF3lz3Dmf88TduYvTo54lUYj7gEfbbgqVVwV9ZjHR8AnQFtgN7BMRKo5X/Mr6EFV0XC1P7dX\ngw7E3DSYpS0acOjUEZr99QeXRX3GpuOHKN+4PC1nt6Tdj+1IWn6MMmMP0eqNd/jric9IHjOJnTW7\ncPDr34vmRNxA+7YtmguL5sJ9LlY8WgGfGGOSjDFvA2OA5SISUtSBKc/pVq8t228czG+tL+Fk6lHa\nrv6LS3+exZqj+6nYriJt5reh5eyWHJ51kNoLy1B12gI2XPUEmXfezYYWd5G8IdbuU1BKedjFuq02\nAAOMMatzrOuKo0XSxBhje7+F3ufhfmsO7+CRdT8S7dOAFj7JfNjuaq6q3hBjDMeWHCNmeAwAlZ+o\ny+Z3pnHNhreJvf5h2n4+HP9qgTZHr5S6EI/c5yEitwIYY+blWt8JGGSMebCgB3YXHfMoOpsS4xiw\ndiGrpD5NJJnJba6kW61GmCzD4W8OEzsyloDaAch/Akl4/RXaJyxm3yOjafd2f8RfezWV8mY6t5UW\nj7OKat6eHUn76L/me37Lqk1DnxO827Izves2Iysji4QZCcSNjqNi24ocb51KmXf+RzUSyZjwJk2f\nuMHtseSXzmFk0VxYNBcWj9wkKCJtCnoAVfw1CarHL9c/RuzlV3CJz0lu2bSJ4KWz+PrAVuo8VIfL\n/rmMKtdXQT5Jo/wtk9l121j8hj5OdL2b2Ld8q93hK6WKwEVbHiLSDRhtjLnKMyG5Rlsenrf/5BEe\nWf0Ni89UoabPGSY0ac0DIWFknMhg76S97J20l6De1dgbu5yOv47mn0vvovXcCAIbVbc7dKWUU5G2\nPETkPuAN4LaCHkCVPHUrVGdB+CMkXNOdKwJSeWjbNmosmc20hGgajmjIZf9cRrlaAVTcFMb+B37i\n9IkypDVuwW+3vUn6iTN2h6+UcoOLdVt9DPQxxhz2RDCqcDx9DXv1ckF83XUAidf2pnu5Mzzxz3aq\nLf2Cdw+vodH4RnTa2Imy5cpQJvE24u5ejN8vv7G/aitWPf8NJqtoW4t6Pb9Fc2HRXLjPxYrHKGC6\niOh8FOq8KpepyOyrHySp223cVjGd52N2UHnpl7x5fBWNJzehw18dKO9Tnky/Z9jX/UMqT3yFDVXD\n2Txjrd2hK6UKKD9jHvcD9xtjPH7pjPNS4V5AIDDdGLM0j210zMPLnE5P5Zl13zAt8Qx+fhV4qm51\nIlqGk7r5FLEjYzmx/gTp9eNp+ufT/BNyPY0+H0f9y+rZHbZSpYpHLtUVkeuNMcsKepDCEpEg4A1j\nzIA8XtPi4aXSMtJ4Yf23TD6UAv5BDKpVmVdbX8fpVSeIHR7L6fjTpPmvovX2May7cjCd5zxD5boV\n7A5bqVLBI5fqFrZwiMinIpIgIhtzre8pIvEikiQiz11gFyOB9woTQ2ngbf25AX4BvNnpLlJ69mNY\ndeHDvTFU+uk7hpf9g2ZLW9NsSjMqVrqKf5p+T8WdJ0ip35yl988g/UxWoY/tbbmwk+bCorlwH09N\nLzIF6J5zhYj4Otf3BdoBj4pICxG5X0QmikhdcZgALDbGFM+nEin8fP14pX0fkm96mFE1/Zm5L47K\nP8/nmSq/0+yPNjR+tQm+Ve8gvsksan67kG2VO/Pr2F/QBqVS3uti05PEAfn5LzzRGPPOBQ/kmExx\nvjGmjXO5C/CKMaabc/klIM0YMz7He4YADwCrgWhjzId57Fe7rYoZYwyTNv/AmLgYUsrU445AeL9d\nT858mUTsqFgyfY9TN34seyrXpMrU12h72yV2h6xUiVPYbqsLTkBkjAkp6I7zoR6QczrWWKBzruO/\nA1ywKAH069ePkJAQAIKCgggLCzs7BUF2M1WXvWv5qfAbGdrKMGzGm0xfs4c5yT7c2jSD+98vR/qS\nE8iX4/H1P8DO269nYXBn7l3wIQ3aBnlN/Lqsy8VtOSoqisjISICz35eF4bG5rfJoefQBbsweBHde\n1dXZGDPYxf1qy8MpqhjP2/PpP1E8v2Mjh8o05MZyaXzYrDsZHyWx5414xGc7IYlvsfr6xwn//FGC\nql980sXinAt301xYNBcWjwyYn+fAR118S+5v+H1AaI7lRsDegsQSERFxtsKq4qlv03AO9BrMF42q\nsyFpP8HRv9D/6j+p83dL6jx8HTGVPqDuyhT217qC7x9bTHqa/sGgVEFERUURERFR6P1cbMyjTx6r\nDSDAXFee55FHy8MP2AU86Pz3Z6CXMcalmfS05VEyzd+9mic3ryQuoBFXBZxgat1rkYnHORi5j8pp\nCznuv520ca/S7cnWSIH/dlKq9CrqZ5inAbP5d6tBcNw46JvPIJcB1+VYNdQY846I3Ah8iOMmwAnG\nmFddCd65b30YVAm2bG80gzb+wk7/RnT2S+bDKl0JeCOJw1/vp1bq52yrWY4Gn47i0h417Q5VqWIh\nykMPg4oG7smrNSAiWd7yJEFteTiU5P7c3w5uZmD0T2zxCyHMN4UpZa6g3NijHFueQJ20T1nTpj1X\nzBlCw2ZlgZKdC1dpLiyaC0tRj3k8CZw4z2vXFPSgSrnqqtqt2NRzMKvaNoO0RC4/+Tf3DNuCfN+Y\nlE5DqL+1CSda9OOzW+aQdEz/mFCqqJWIJwlqt1XpszExlgFrF7Jagmnik8LkY+2p+EICmbH7EZaz\n58lHuW38Zfj72x2pUt7FU91W717gvcYYM6SgB3YX7bYq3XYk7aX/mnn8llWXEDnJ+7tbU3nEIfwO\nx3Ko7A783n6SHgOCdVBdqVyKuttqLbAmj5+1zh/lRUrj5cpNgurzy/WPE3N5Fxr5pNCrwS7u/vQw\nyx7OpLJPN2o88hVf1HmNNVHn630t+Urj5+J8NBfuc7E7zCNzrxORhsaY3UUWUQFERERot1UpFxJY\nm2XdHuPAySMMWPU1Iy9JoO6cyrwbFUa9SaeQa9/nkzZ1ufbb/xJySb4uElSqRMrutiosl8c8RCQz\nv5foeoJ2W6m8HDmVxCOr5/L96YrUToOJ3wRSayb4mY1s7XUZfWZeS1CQ3VEqZR+PPM8j1wG94hLd\nbFo81IUknUlh0Kq5zDkRQO3T/rw5zZe6i8tyxmcHCU/25I7xLQgIsDtKpTzPjulJxhX0YEVFpydx\n0BxYsnMRVKYSs69+iOPdbueG2qnc/8RJ7v38FImdatFg4k4WBn7IgimHS/T07/q5sGguPDQ9ydmN\nRL4BpuN4rkbhn9TjRtrysOgNUJbz5SI1PZVha79iWmIawUcCGTchlTpbKxFf9RRN5t5G5+vKej7Y\nIqafC4vmwuKpx9DeAAzAMWX6F8AnxpjtBT2oO2nxUAWRlpHGC+u+YfLhFBrtqcrL409RY3clYhqX\npevC7oQ29ZphPaWKhEfHPESkOnA/8CKwDZgKzDLGpBc0gMLS4qEKIz0zndEb5jFx/xGa76zNixNO\nUDmhAruvqset33aiSjW9QUSVTB4b8xCRGjhmwO0H/ABMAnoDSwt6cOVe2p9ryW8u/H39eaX9/5HS\n6xFuvxb6fpTMyLE+VNsSy6qa85h97w7OnCnef5zo58KiuXCfiz9VBxCR74DGwCfAdcaYROf6ecCR\nogsvf/Q+D1VYPuLDiHb/YXhbw8SQRTzU4hBha0N58r2NLKm0DXm2E73G1NY71VWx59H7PETkWmPM\nz4U+WhHQbitVFIwxTNm6lFc3beHSP5vw6NQsTuFPvXeuoMuDgXaHp1Shefw+D2+jxUMVJWMMM3as\n4LU/19BmZUsemCEkBZaj44xONO1ewe7wlCqwIh3zEJE4EYnNx4/tEyQq7c/NyV25EBH6Ng1n8wPP\ncPuwSox97W+W9khl562/M7PFnyRsOO2W4xQl/VxYNBfuc7G5rUI8FIdSXu/OxldzZ+OrWbB1Be83\n/4WG0Z2pdvlvnP5PMLd/1gzx0QERVXqUiG4rfZ6HssPPa5cw9cvVhM/rRNkzPvwn6hoCQ3SuE+Xd\nPPI8j+JAxzyUnUxWFi9PjCBx5eVc/6Mfdd8Io+Oj+jx15f3smNtKeSntz7V4Khfi48OoYS9zz//8\nmPtADPEv/M2s7n+RecZ7ZvHRz4VFc+E+LhcPEakkIm1F9Ip3pbJ1uewGpr5+Lz/3X8LBpGTmhCzh\nUPQpu8NSqsjk9z6P5cAwYDPwKxAILDfGDC7a8C5Ou62Ut5n52fusXFSTm+ZVoexToVw/5hK7Q1Lq\nXzw1MeIpoApwDVAfmA/sMMZUKeiB3UWLh/JGe/dsZ+yYL7lq4RWk1Yb7o8Lxq5SvCR2U8ghPjXkk\nAq2AO3ETRfLcAAAYbUlEQVTMa2XbRIjq/LQ/12J3LuoHN2PylJHEPPwTcZXT+S50OTuXJtkSi925\n8CaaC/fJb/F4FogEDhpjDgBdgAVFFZRSJYH4+PBixDh6RGTxyw1xbOmzlrkPrUZbyqokKBGX6up9\nHsrbnT6VzItDx9B0RQ/8JY27fupG+bpl7A5LlUIevc9DRMoB/YGWQDnAABhjHirogd1FxzxUcfLx\nlEnEzKtPx5WVqT2pKZf3a2h3SKqU8tSYx0ygGTAQiAJaAPsLelBVNLQ/1+KtuXho4FAentyS32/4\nk/3DdvFxr+VkpRftPSHemgs7aC7cJ7/F4wYc4x6ZwCygD3BXUQWlVEnWMLQlE74Yzub7fyR5fzqf\nNf6RhC0pdoellEvy220VA1wKrMHxNMEk4Fe9VFepwlm+cB5R7yXQ6Y9LYFhVbnnpUrtDUqWEp+7z\n+C/wF9AaeB8oA4w0xnxY0AO7ixYPVdydPJHEuIfG0eq3HiSHnKH/khvwr6j3hKii5ZExD2PMbGPM\nLmPMPGNMfWNMDW8oHOpc2p9rKU65qFAxiLFzXuP44+s5kZnK141/YsOSvW7bf3HKRVHTXLiPToyo\nlJd4bMQz3DK9KVsvXUfcHf8wre8SvSdEea0ScZ9HcT8HpXLKysrklUdHE7z8atLKn+L/FnejWnBF\nu8NSJUyRd1uJiI+IXCMiHr+jSUSai8gHIjJXRAZ6+vhK2cHHx5eXpr5MvXfhWJUUoi79k3lv/ml3\nWEqd46LFwxiTBSwEPD4FuzFmmzHmMRyXBV/p6eMXN9qfaykJubihV3eeXHIba69dQ9Yrp5jY7Qsy\n0zJd3k9JyIW7aC7cJ79jHhOAJ0WkQJeAiMinIpIgIhtzre8pIvEikiQiz53nvTfjmEdrUUGOrVRx\nVq5cBcbNfZ6EUfsovz+QT1suInrpLrvDUirfl+qewJqW5LRztTHGBObrICJdgJPAZ8aYNs51vsAu\nHPeNxAA/A72AjkB74HVjzP4c+1hgjOmdx751zEOVCnF79/BpvyW0WxPCgdsO89gnd9sdkirGPHKf\nhzuISAgwP0fx6AK8Yozp5lx+CUgzxozP8Z6uwO047ivZYIz5II/9avFQpYYxhjFDP6TF7CbsC9nP\n/V/dTLWGQXaHpYqhwhaPfHdDicitOB4GZYAVxpj5BT2oUz0gNsdyLNA55wbGmBXAiovtqF+/foSE\nhAAQFBREWFjY2Rl2s/s4S8Nyzv5cb4jHzuXsdd4Sj7uWV6xYwTW3Ncf0qcSBISd4s9OHlLnFMGra\n8+d9f3R0NEOHDvWK+O1enjRpUqn+foiMjAQ4+31ZGPntthqPY36ryTgGzh8FlhpjXsj3gf7d8ugD\n3GiMGeBcvh/o7OqjbbXlYYmKijr7oSntSkMuzmSk8cqdM+m4PJRdHXcwZNFD+JXx/9d2pSEX+aW5\nsHhqepLDQBtjzEHncm1gkzGmuguBNgQW5CgelwNjc3RbjQJSjTETXDoBfZ6HKuWmT12MjPchy+80\nbcY35LLbdH4sdX5RHn6eRxzwH2NMtHP5UuBbY0xIvg/075aHH9aA+S6cA+bGmK0unYC2PJQiITGJ\nKXd9T7s19dhz424Gf277o3aUl/PU8zxeAH4QkRkiMhPHZbPP5/cgIrIMxxVVrUQkS0SGGGMycDwf\nJBLYAEx1tXBki4iIOKevu7TSHFhKWy5qVQti1LIH2DowgRpRDXm/3Wz2btkHlL5cXIjmwpGDiIiI\nQu/noi0PEfEB7geW4biM1gCrnc8yt522PCzan2spzblYu24nyx5fzyU7gzjd/zjBPauX2lzkVpo/\nF7l5aswjEQg2xpwq6IGKihYPpf4tMzOLlx78mI4LGrG7xW4GLrqLspXL2x2W8iKe6rZ6DRgvIqEi\nUjX7p6AHdTfttlLqXL6+PoydMYCMD8ril1KduW1WsHjqRa96V6WAx7qt4OyAee4NjTGmUaEjKCRt\neVi0SW7RXFgW/7CEP96Lo9Mvjfnn6t08Ne8BfP187Q7LFvq5sHhkVl1gFNDCGBOa48f2wqGUurhy\nZQMYs+AR9r94nOpb6jKtzQLWLN1md1iqmCsRYx56n4dS+bNn7yFm/ncBLTaGEHtrAk9H3mN3SMrD\nPH2fx7NAfWAicDx7vTHmaEEP7C7abaWU6yY8OZXQL0LZE5LIHbO60qBJbbtDUh7mqQHzQcDNwE/A\n2hw/yovoRQMWzYUlr1w89/bDXDK/Ln7pht+7bmJqxFeeD8wG+rlwn3wVD2NMSK7xjlBjTGhRB5df\nerWVUq7r0Lklg9fcRcwVMdScWJVXbviElOPJdoeliphHrrYSkXeMMUOcvw8yxkzO8dpsY8x/Cx1B\nIWm3lVKFt2j2Kg68lECmnx/VXhD69O1pd0iqiBXpTYIikmWM8cn9e17LdtHioZR7nD6Rwdu9Z9Ii\nuiEbbtjFiNkP4utXoIeHqmLAU2MeqhjQrjuL5sKS31yUq+jH81EPcnToGRr/3oD3Oi1g1e9bijY4\nD9PPhfuUiOKhYx5Kuc+DETfS9ddOZAQcZ88tB3ht0Gy7Q1Ju5Kkxjywczx4HqJDjd4AK2m2lVMk2\nacDH1J/XgNjGifSJvJpGzeraHZJyk2LzDPOiosVDqaK14fd/WPHEOmodqE7iA0cY9Nrddoek3EDH\nPNRZ2nVn0VxYCpuLdlc2Zcj6u4m5No46U2sy4eoZHN6X6J7gPEw/F+6jxUMplS8vfD6AKp9Xp2JS\nJX68fA2zxn5nd0jKRtptpZRySUZ6BmPv/pSwZaHsaLeDxxc9QLmK5ewOS7lIu63Qq62U8iQ/fz9G\nfd2fzA8CqHi4AV+2XcaCyCi7w1L55NHneXgzbXlY9FkFFs2FpShzcepUKm/0+YKwPxqw6bqdPDvn\nIfz8vffGQv1cWLTloZSyTfnyZXlpcT8OPZtMw9UhTOnwPX8si7Y7LOUB2vJQSrnFgfijzLxnIY23\n1uWfm3fx/KeP2B2SugC9z0OLh1Je5d2n5lDrs6rsbZhI9w870KZ9Y7tDUnnQbit1ll40YNFcWDyd\ni8ET76T9Ty0Qk8XWnrt5a8hMjx7/QvRz4T5aPJRSbte4dT2eWnMP8b32EDqrLq9fMYs9uw7YHZZy\nI+22UkoVqTW/bGPl4M1UOxLI8YeSeGzMHXaHpNBuK0Dv81DKm3W8pjlPRPch7sp46r5TjbHXRpKU\nmGJ3WKWW3ufhpC0Pi17DbtFcWLwpFz9//Sc7RhwiIC2AMk8L9zzRw6PH96Zc2E1bHkqpYuPaPpfz\n0N83sr95PBWG+/NKz2mcPp1md1iqALTloZSyxddTlnP89TNk+mZRa0Qlbunb1e6QShW9z0OLh1LF\n1umTqUz8z2xarQphc9ddPPv1g149vUlJot1W6iy9aMCiubB4cy7KVSjL8KUPcfSFk9Tf0IBpYQtZ\nsWB9kR3Pm3NR3GjxUErZ7sHnb6bX6ss5Vv0oSfceY8KdH4P2KHg17bZSSnmV9178kqofVyWx+nGu\neq81l17d3O6QSiQd89DioVSJE7vzIF/dv4xL/qlDzH/28cz0B+wOqcQp8WMeIlJBRFaLSC+7Y/F2\n2p9r0VxYimMuQhvX5n8r7yPu7j0EL6zL2+0/Z+ffuwu93+KYC2/l9cUDeBb40u4glFKe9/T7/Wi5\nuDEZfj6su34Hbz85y+6QlJNHuq1E5FOgJ3DIGNMmx/qewEdAIPCqMWZCrvd1B6oCZYEjxpiFeexb\nu62UKgUm9Iuk8fxgYpse4M7PutOgUS27QyrWisWYh4h0AU4Cn2UXDxHxBXYBDwIxwM9AL6Aj0B54\nHRgEVABaAqeB23JXCi0eSpUeq3/Zyl9PbCHoWCAp9x3jsVfvtDukYquwxcMjd+MYY1aKSEiu1Z2B\nXcaYnwFEJBK41RgzHsh+AMBI52t9gcPnqxL9+vUjJMSx+6CgIMLCws7OX5Pdx1kalnP253pDPHYu\nZ6/zlnjsXI6Ojmbo0KFeE09hlk9mJdDq7aqs/DCGllNCeWTh8/SKuIpbb++dr/dPmjSpVH8/REZG\nApz9viwMj11t5Swe83O0PP4P6GmMGeBcvh/obIwZ7OJ+teXhFKWTvp2lubCU1Fz8tGA125/bR8WU\ncmQOTKXf8Fsv+p6SmouCKM5XW7ntG1+nZHfQ/xQWzYWlpObiut6deDi6N7vbx1NlQiXGdZtOSvKp\nC76npObCFVHFbUp2EWkILMjR8rgcGGuM6eZcHgWk5h40z8d+teWhVCm34LPfOfDyMfzTfSk71Ie7\nh3h2qvfiqDi1PHIHuQZoLCLXOQtLX+B7D8ZT4mjry6K5sJSGXPS+90r6/d2TvS32UX6kP+N6TuVM\n6r+nei8NufAUjxQPEVmG44qqViKSJSJDjDEZwEAgEtgATDXGbC3I/rXbSinlX8aPkQsHcHpCFrV3\n1mNWmyXM+zjK7rC8TrHrtioq2m2llMrt1IlU3rr9M9r8Fcrmq3fx7Lc61Xtuxanbqshoy0MplVP5\nimUZuaQ/yaNOU3dzMB+3WcSPc/+0OyyvoC0PJ215WPQyRIvmwlLac5GUmMLk//uKVusa8n27xXy4\n/FVthaAtD6WUuqCgapUY/vODHBqWRO2tNZnSYT6/LI62O6xir0S0PEaNGkV4eHip/utKKXVxCXsS\n+fSeBTTdHMzWnrG88Hl/u0PyuKioKKKiohg9erT3z21VlLTbSinlqvef+5JqM6pyqNZxukxqSafw\nlnaH5HHabaXO0osGLJoLi+bCkp2LxyfcxZW/teNM+VT29DnAhH6RtsZVHGnxUEqVSsGX1OR/f9zH\nnnv3E7q4PhM7fsmGVTvtDqvYKBHdVjrmoZQqjB2b9vL9Q7/QMKYWcbfv5ZmP+todUpHRMQ8nHfNQ\nSrnLGwM/JeTr+sSHHqL3x1fRtHWw3SEVGR3zUGdp37ZFc2HRXFgulotnpvSl+eJQJMuHDddt5+3B\nn3kmsGJIi4dSSuXQumMjnlpzF7t6xVN/dm3euHwWcTsO2B2W1ykR3VY65qGUKgprf93G70M2UvNg\nEEfuTeSJN+62O6RC0zEPJx3zUEoVtfH3TqfZolB2tNzLfZ/fRN0G1e0OqdB0zEOdpX3bFs2FRXNh\nKWgunv+sP7W+rE75ExWJ6rKOj0Z85d7AiiEtHkoplQ9X3NCWgWtuIfbK3dR8vyrjr4nk8IGjdodl\nG+22UkopF0V9v5b1L+/kmknt6HBVc7vDKZDCdltp8VBKqVJIxzzQh0Fl0xxYNBcWzYVFc+G+h0GV\niCeiuCMRSilVGmTf1jB69OhC7Ue7rZRSqhTSbiullFIep8WjBNH+XIvmwqK5sGgu3EeLh1JKKZfp\nmIdSSpVCOuahlFLK40pE8dD7PBw0BxbNhUVzYdFc6H0e59D7PJRSKn/0Pg8nHfNQSinX6ZiHUkop\nj9PiUYJof65Fc2HRXFg0F+6jxUMppZTLdMxDKaVKIR3zUEop5XFeXTxEJFxEfhWRD0Skq93xeDvt\nz7VoLiyaC4vmwn28ungAWUAKUAbYa3MsXi86OtruELyG5sKiubBoLtzHI8VDRD4VkQQR2ZhrfU8R\niReRJBF5Lo+3/mqMuQl4HijcHS2lQFJSkt0heA3NhUVzYdFcuI+nWh5TgO45V4iIr3N9X6Ad8KiI\ntBCR+0VkoojUzTESnoSj9VHkXG3WXmz7C72e12sXW5f79aJshmsuCr7vos6Fq8vupLko+L5LUi48\nUjyMMSuB5FyrOwO7jDE/G2N2A5HArcaYmcaYp4wx+0XkNhGZAswA3vVErN72Yci97kIfhri4uAvG\n4irNxYVjKcz2nvyS0FxYy5qLCx/fFR67VFdEQoD5xpg2zuX/A3oaYwY4l+8HOhtjBru4X71OVyml\nCqAwl+raOTGiW770C3PySimlCsaTV1vlLhb7gNAcy43QK6qUUqpY8GTLI3cLYQ3QWESuA3bhGDjv\n5cF4lFJKFZCnLtVdBsQArUQkS0SGGGMygIE4Bso3AFONMVs9EY9SSqnCKfZzWymllPI8b7/DXCml\nlBcqccVDRCo472j/SET+a3c8dhKRUBGZJiJz7Y7FbiJyq/Mz8YWIdL/4O0ouEWnunC9urogMtDse\nOzm/L1aLSKkfb3V1LsESVzyA24E5xphHgFvsDsZOxpjY7PtoSjtjzDznZ2IgcJfd8djJGLPNGPMY\njjxcaXc8NnsW+NLuILyES3MJFovi4eLcWPWAPc7fMz0aqAcUYp6wEqeAuRgJvOe5KD3D1VyIyM3A\nAmCRp2MtSq7kwdkC3QIctiNWT3Dxc+HaXILGGK//AboAbYGNOdb5AnHAtUBDHFdztQDuA3o5t/nc\n7tjtzEWO1+faHbfducBxqfgEoJvdcdudi1zvW2B37DZ+Jl4BJgI/At/hvICoJP0U8PsiID/fGXbe\nYZ5vxpiVzulNcjo7NxaAiEQCtwLvAO85+zC/92CYHuFKLkQkARgHhInIc8aYCZ6Mtai5+Lm4HugG\nBIpIY2PMhx4Mtci5+LmoiaN7twyw0INhFjlX8mCMGelc7gscNs5vzpLExc9Fc6AHEEQ+5hIsFsXj\nPOoBsTmWY3HMjXUKeMiekGxzvlwcxdHHX5qcLxeD8dDkml7kfLlYAaywJyRb5JmH7AVjzKcej8he\n5/tcjAe+ze9OisWYx3mUuL8SCkFzYdFcWDQXDpqHc7klH8WpeOjcWBbNhUVzYdFcOGgezlUk+ShO\n3VY6N5ZFc2HRXFg0Fw6ah3MVTT7svhogn1cMLMNxDXL2zxDn+huBeBxPGnzB7jg1F5oLzYXmwZt+\nijIfOreVUkoplxWnMQ+llFJeQouHUkopl2nxUEop5TItHkoppVymxUMppZTLtHgopZRymRYPpZRS\nLtPiobyOiGSJyBs5lp8RkVFu2nekiPRxx74ucpw7ROSQc2bjnOtDsp+tICLtRORGNx6zsog8ltex\nlHI3LR7KG6UBt4lINeeyO+9kLfC+RMSV6Xz6A/2NMbXOObgxccaYNs7FS4Gb3BhDFWDQeY6llFtp\n8VDeKB34CHgq9wu5Ww4icsL5b7iILBGReSJyUkReFZF7nX/9bxWRRjl201lEdovInuxnV4uIr4i8\n7nzq2j4R6Z9rv98Cp/KI5x4RiRWRwyIy3rluGnADME9E5uTaPkRENopIGeBj4HFnS2uQiJQXkU9E\n5KiI7BKR3s739BOR2SKyAtgqIuVE5EcRSRGRGBHJftzyF0Ar5/7miEhDEdnk3EdZ576PicgWEQnP\nse9ZIvKziCSLyATnej/n+hTnuZW2qf3VRRSniRFV6TIZ+FtEXsu1PnfLIefyFTge+vQ48DeOp8M1\nwvHo2cE4ipEAVwEdgHbALOfDcvrieGxxIxx/wc8XkaXO/V6No4VwX84Di0hdHE+i6wkcBBaJyEpj\nzAARuQQYZoxZl9fJGWPOiMiDQAdjzBDn/sYB64DHgBDgOxFZ5nxLd+e57QAygKHATqAZjmcwfI/j\nmeQLslsbzvPKzs/jQFXnfrs4z/sS52vdnPtPBtaIyDtAbRwzr9bF8eS5c1pQSmnLQ3klY0wKMAMY\n4sLb/jLG/GmM2QtsBj42xpwAluL40gTHl+l0Y8wRY8xyHLOKNsfRUngWSMExYVxbILvLZ7Ux5mdj\nzMlcx+sELDHGRBtjDuIoeNfkeD33bKa5Sa5tbgDextHC2QI0ARo7X1tkjNlgHA878wPG4Hj29t9A\nI+fTAS90vCuBt40xx40xP+B4DGlTZz4WG2M2GWPicRSvhsB2HMV0AFDHGLP9IueiShktHsqbTcIx\ndlAhx7oMnC1mZ/9/mRyvncq1XfZyJue2snN+yRqsv857GGN8nD++xpjsR7QeP098Jte+hHNbQgUZ\nX2mWK4ZNzv0k59jmPuexQo0xPjiKXdl87Pt8xSVnUUwH/IwxJ4wx1wC/AIOyu7OUyqbFQ3ktY8wx\nYA6OApL9RRwH3CQiFXB03fi6uFsBHhSRGs7nGVwCbAN+BF4QkfoiUlFEeopIpYvsazVwvYhcKiK1\ngUdx7fGuJ4B6IpL9//BH4GURqS4iQSJye46YcyoLnAbOiEg/oL5z/UmgmoiUz+NYvwJPOvfbA0fr\nYlse+wYQEQkWkQ7AJmAxjlaRUmdp8VDeKOdf7G8C1XMsT8XRzXQQR6vj5Hnel3t/Jsfvv+F4IE4k\nMMAYkwZMA1YCq3A8Ve2xHO/Lc7/GmAM4xlG+xfElu9QYM9+F8/sZx7jCGREZhKMr6jiwFcd4xn/y\niB/gMxxF7zDQ0bk9xpjDOFoKic6B+pzvmwwcwVF8JwL3GWPSz3N+BqiEI9dJOLrSRubjvFQpos/z\nUEop5TJteSillHKZFg+llFIu0+KhlFLKZVo8lFJKuUyLh1JKKZdp8VBKKeUyLR5KKaVc9v9Mp9bI\nA9tOqgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f10e53034a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure()\n",
    "plt.hold(True)\n",
    "for res in global_results:\n",
    "    plt.loglog(iterations_numbers, res)\n",
    "\n",
    "plt.grid(True)\n",
    "plt.xlabel(\"Number of iterations\")\n",
    "plt.ylabel(\"Error as |y-l1|^2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The hypothesis that using a single NN instead of one per run  would reduce dispersion at low training iterations numbers isn't verified. The second (and better) hypothesis is that this dispersion is only related to the random initialization of synapses... it seems plausible but the deviation is quite big though....\n",
    "\n",
    "To answer the questions above : the bend point seems stable with respect to number of iterations and error, no matter if we use one or several NN. $\\Leftarrow$ **Answer lies in maths**\n",
    "\n",
    "The second goal was to implement an error controled training process : let's modify the TwoLayersNN class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class TwoLayersNN:\n",
    "    \n",
    "    def __init__(self, in_data, out_data):\n",
    "        \n",
    "        self.in_data = in_data\n",
    "        self.out_data = out_data\n",
    "        \n",
    "        self._init_synapses()\n",
    "        \n",
    "    def _init_synapses(self):\n",
    "        \n",
    "        self.syn0 = 2*np.random.random((len(self.in_data[0]), len(self.out_data[0]))) - 1\n",
    "        self.trained = False\n",
    "        \n",
    "        \n",
    "    def NL(self, x, deriv=False):\n",
    "        \n",
    "        if(deriv==True):\n",
    "            return x*(1-x)\n",
    "        return 1/(1+np.exp(-x))\n",
    "                \n",
    "    def train(self, condition=lambda _: False, verbose=False):\n",
    "                    \n",
    "        if self.trained:\n",
    "            self._init_synapses()\n",
    "        \n",
    "        self.training_iter = 0\n",
    "        while condition(self):\n",
    "            if verbose: print('Counter : {}'.format(self.training_iter))\n",
    "            l0 = self.in_data\n",
    "            l1 = self.NL(np.dot(l0, self.syn0))\n",
    "            \n",
    "            l1_error = self.out_data - l1\n",
    "            l1_delta = l1_error*self.NL(l1, True)\n",
    "            \n",
    "            self.syn0 += np.dot(l0.T, l1_delta)\n",
    "            \n",
    "            self.training_iter += 1\n",
    "            \n",
    "        self.trained = True\n",
    "        return self.syn0\n",
    "    \n",
    "    def train_iter(self, iter_nb):\n",
    "        self.train(lambda x: x.training_iter < iter_nb)\n",
    "        \n",
    "    def train_error(self, goal_error, *a, **kw):\n",
    "        self.train(lambda s: (s.training_iter < 10000) or (np.mean(np.power(np.abs(self.out_data- s.test(self.in_data)), 2)) > goal_error), *a, **kw)\n",
    "    \n",
    "    def test(self, in_data, batch=True):\n",
    "        \n",
    "        if not batch:\n",
    "            in_data = [in_data]\n",
    "            \n",
    "        return self.NL(np.dot(in_data, self.syn0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean error : 9.999868933361674e-06\n"
     ]
    }
   ],
   "source": [
    "NN = TwoLayersNN(X,y)\n",
    "NN.train_error(1e-5)\n",
    "mean_error = np.mean(np.power(np.abs(y- NN.test(X)), 2))\n",
    "\n",
    "print('Mean error : {}'.format(mean_error))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "3 Layers NN\n",
    "----------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error:0.496410031903\n",
      "Error:0.00858452565325\n",
      "Error:0.00578945986251\n",
      "Error:0.00462917677677\n",
      "Error:0.00395876528027\n",
      "Error:0.00351012256786\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "def nonlin(x,deriv=False):\n",
    "\tif(deriv==True):\n",
    "\t    return x*(1-x)\n",
    "\treturn 1/(1+np.exp(-x))\n",
    "    \n",
    "X = np.array([\n",
    "    [0,0,1],\n",
    "    [0,1,1],\n",
    "    [1,0,1],\n",
    "    [1,1,1]\n",
    "])\n",
    "                \n",
    "y = np.array([\n",
    "    [0],\n",
    "\t[1],\n",
    "\t[1],\n",
    "\t[0]\n",
    "])\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "# randomly initialize our weights with mean 0\n",
    "syn0 = 2*np.random.random((3,4)) - 1\n",
    "syn1 = 2*np.random.random((4,1)) - 1\n",
    "\n",
    "for j in range(60000):\n",
    "\n",
    "\t# Feed forward through layers 0, 1, and 2\n",
    "    l0 = X\n",
    "    l1 = nonlin(np.dot(l0,syn0))\n",
    "    l2 = nonlin(np.dot(l1,syn1))\n",
    "\n",
    "    # how much did we miss the target value?\n",
    "    l2_error = y - l2\n",
    "    \n",
    "    if (j% 10000) == 0:\n",
    "        print(\"Error:\" + str(np.mean(np.abs(l2_error))))\n",
    "        \n",
    "    # in what direction is the target value?\n",
    "    # were we really sure? if so, don't change too much.\n",
    "    l2_delta = l2_error*nonlin(l2,deriv=True)\n",
    "\n",
    "    # how much did each l1 value contribute to the l2 error (according to the weights)?\n",
    "    l1_error = l2_delta.dot(syn1.T)\n",
    "    \n",
    "    # in what direction is the target l1?\n",
    "    # were we really sure? if so, don't change too much.\n",
    "    l1_delta = l1_error * nonlin(l1,deriv=True)\n",
    "\n",
    "    syn1 += l1.T.dot(l2_delta)\n",
    "    syn0 += l0.T.dot(l1_delta)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.abs(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}