shotahorii/DeepLearningFromScratch_1.ipynb

## DeepLearningFromScratch_1.ipynb
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deep Learning from scratch (Section 2-3)\n",
    "Koki Saitoh (2016) : Deep Learning from the scratch, Section 2-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Perceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def base(x,w,b):\n",
    "    tmp = np.sum(w*x) + b\n",
    "    if tmp <= 0:\n",
    "        return 0\n",
    "    else:\n",
    "        return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def AND(x1, x2):\n",
    "    x = np.array([x1, x2])\n",
    "    w = np.array([0.5,0.5])\n",
    "    b = -0.7\n",
    "    return base(x,w,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def NAND(x1, x2):\n",
    "    x = np.array([x1, x2])\n",
    "    w = np.array([-0.5,-0.5])\n",
    "    b = 0.7\n",
    "    return base(x,w,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def OR(x1, x2):\n",
    "    x = np.array([x1, x2])\n",
    "    w = np.array([0.5,0.5])\n",
    "    b = -0.2\n",
    "    return base(x,w,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def XOR(x1, x2):\n",
    "    s1 = NAND(x1,x2)\n",
    "    s2 = OR(x1,x2)\n",
    "    return AND(s1,s2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def step_function(x):\n",
    "    return np.array(x>0, dtype=np.int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEI5JREFUeJzt3X+M5HV9x/Hn60Aa0wqpmND0CChSoKUqGr3S2MoIWk6a\ncqb/FGw0YGpoK2rSpoVqGzbGpNrY1B+0hWuBRlODrZqKjeJpZdLYCGIrgnonZzHXu9NisLUNJpi7\n490/5it+u7I7c9zMzn6+Ph8JufnOfHbnM9zs87772c/OpKqQJLVvy7InIEmaD4MuSQNh0CVpIAy6\nJA2EQZekgTDokjQQx2/knSVxj6QkPQFVlWljNvwMvaoG+99111239Dn4+Hx8P2yP7Yfh8c3KJRdJ\nGgiDLkkDYdDnaDQaLXsKC+Xja9eQHxsM//HNKkezPnPMd5bURt6fJA1BEmoz/lBUkrQYBl2SBsKg\nS9JAGHRJGgiDLkkDYdAlaSAMuiQNhEGXpIGYGvQkNyV5MMm964x5V5K9Se5Jct58pyhJmsUsZ+i3\nABevdWOSlwHPrKqfAq4CbpjT3CRJR2Fq0Kvq08B/rzNkB/CebuxdwElJTpnP9CRJs5rHG1xsBfb3\njg921z04h88tbagq+OQn4ZFHlj0TbTYnnggXXLDsWaxvQ9+xCGBlZeWxy6PRyFdJ06Zy8CBceim8\n5CXLnok2mzPO2Ligj8djxuPxUX/cTK+2mOR04CNV9ezHue0G4I6qen93vAe4oKp+4AzdV1vUZvfA\nA3DRRfC1ry17JtL3zfvVFtP993huA17V3en5wLcfL+ZSC44cgeM3/PtWaT6mPnWTvA8YAScn+Q/g\nOuAEoKpqZ1V9NMklSb4KfAe4cpETlhbp8GGDrnZNfepW1StmGHP1fKYjLZdBV8v8TVGpx6CrZQZd\n6jHoaplBl3oMulpm0KWeI0fguOOWPQvpiTHoUo9n6GqZQZd6DLpaZtClHoOulhl0qcegq2UGXeox\n6GqZQZd63OWilhl0qcczdLXMoEs9Bl0tM+hSj0FXywy61GPQ1TKDLvUYdLXMoEs9hw+7y0XtMuhS\nj29Bp5YZdKnHJRe1zKBLPQZdLTPoUo9BV8sMutRj0NUygy71uMtFLTPoUo+7XNQygy71uOSilhl0\nqcegq2UGXeox6GqZQZd6DLpaZtClHne5qGUzBT3J9iR7ktyf5JrHuf3EJLcluSfJfUmumPtMpQ3g\nLhe1bGrQk2wBrgcuBs4FLk9yzqphrwW+VFXnAS8G/jSJXxZqjksuatksZ+jbgL1Vta+qDgG3AjtW\njSngKd3lpwDfqqrD85umtDEMulo2S9C3Avt7xwe66/quB34mydeBLwBvmM/0pI1l0NWyeT11LwY+\nX1UXJnkm8Ikkz66qh1cPXFlZeezyaDRiNBrNaQrSsTPo2gzG4zHj8fioPy5Vtf6A5Hxgpaq2d8fX\nAlVVb+uN+Ufgj6vqX7rjfwKuqarPrfpcNe3+pGV6+cvhiismf0qbRRKqKtPGzbLkcjdwZpLTk5wA\nXAbctmrMPuAl3R2fApwFPHB0U5aW78gRty2qXVO/uayqI0muBnYx+QfgpqraneSqyc21E3gL8DdJ\n7u0+7Per6r8WNmtpQVxyUctmeupW1e3A2auuu7F3+RtM1tGlphl0tczfFJV6DLpaZtClHoOulhl0\nqcegq2UGXepxl4taZtClHs/Q1TKDLvUYdLXMoEs9Bl0tM+hSj0FXywy61GPQ1TKDLvX4FnRqmUGX\nenwLOrXMoEs9LrmoZQZd6jHoaplBl3oMulpm0KUeg66WGXSpx10uaplBl3rc5aKWGXSpU+WrLapt\nBl3qfC/mmfre6tLmZNCljj8QVesMutQx6GqdQZc67nBR6wy61HGHi1pn0KWOSy5qnUGXOgZdrTPo\nUsegq3UGXeoYdLXOoEsdd7modTMFPcn2JHuS3J/kmjXGjJJ8PskXk9wx32lKi+cuF7Vu6tM3yRbg\neuAi4OvA3Uk+XFV7emNOAv4c+KWqOpjkaYuasLQoLrmodbOcoW8D9lbVvqo6BNwK7Fg15hXAB6vq\nIEBVPTTfaUqLZ9DVulmCvhXY3zs+0F3Xdxbw1CR3JLk7ySvnNUFpoxh0tW5eT9/jgecBFwI/Cnwm\nyWeq6qtz+vzSwhl0tW6Wp+9B4LTe8anddX0HgIeq6hHgkST/DDwH+IGgr6ysPHZ5NBoxGo2ObsbS\nghh0bRbj8ZjxeHzUH5eqWn9AchzwFSY/FP0G8Fng8qra3RtzDvBuYDvwI8BdwK9V1ZdXfa6adn/S\nstxxB7z5zZM/pc0kCVU19ZX6p56PVNWRJFcDu5isud9UVbuTXDW5uXZW1Z4kHwfuBY4AO1fHXNrs\nPENX62Z6+lbV7cDZq667cdXx24G3z29q0sYy6GqdvykqdQy6WmfQpY5BV+sMutQx6GqdQZc6R474\n4lxqm0GXOp6hq3UGXeoYdLXOoEsdg67WGXSpY9DVOoMudQy6WmfQpY5vQafWGXSp41vQqXUGXeq4\n5KLWGXSpY9DVOoMudQy6WmfQpY5BV+sMutRxl4taZ9Cljrtc1DqDLnVcclHrDLrUMehqnUGXOgZd\nrTPoUsegq3UGXeq4y0WtM+hSx10uap1Blzouuah1Bl3qGHS1zqBLHYOu1hl0qWPQ1TqDLnXc5aLW\nzRT0JNuT7Elyf5Jr1hn3giSHkvzq/KYobQx3uah1U4OeZAtwPXAxcC5weZJz1hj3VuDj856ktBFc\nclHrZjlD3wbsrap9VXUIuBXY8TjjXgd8APjmHOcnbRiDrtbNEvStwP7e8YHuusck+Ung5VX1l0Dm\nNz1p4xh0tW5ePxR9B9BfWzfqao5BV+tmefoeBE7rHZ/aXdf3fODWJAGeBrwsyaGqum31J1tZWXns\n8mg0YjQaHeWUpcUw6NosxuMx4/H4qD8uVbX+gOQ44CvARcA3gM8Cl1fV7jXG3wJ8pKo+9Di31bT7\nk5bluc+Fm2+e/CltJkmoqqkrH1PPR6rqSJKrgV1MlmhuqqrdSa6a3Fw7V3/IE5qxtGSeoat1Mz19\nq+p24OxV1924xthXz2Fe0oYz6GqdvykqdQy6WmfQpY5BV+sMutQx6GqdQZc6vjiXWmfQpY4vzqXW\nGXSp45KLWmfQpY5BV+sMutQx6GqdQZc6Bl2tM+hSx10uap1Bl4BHH4Uq2OJXhBrm01fi+1sW4yv5\nq2EGXcL1cw2DQZcw6BoGgy5h0DUMBl3CHS4aBoMu4eu4aBgMuoRLLhoGgy5h0DUMBl3CoGsYDLqE\nQdcwGHQJd7loGAy6hLtcNAwGXcIlFw2DQZcw6BoGgy5h0DUMBl3CoGsYDLqEQdcwGHSJyS4Xty2q\ndTMFPcn2JHuS3J/kmse5/RVJvtD99+kkz5r/VKXF8QxdQzA16Em2ANcDFwPnApcnOWfVsAeAF1XV\nc4C3AH8174lKi2TQNQSznKFvA/ZW1b6qOgTcCuzoD6iqO6vqf7rDO4Gt852mtFgGXUMwS9C3Avt7\nxwdYP9i/AXzsWCYlbTSDriGY61M4yYuBK4FfWGvMysrKY5dHoxGj0WieU5CeEIOuzWQ8HjMej4/6\n41JV6w9IzgdWqmp7d3wtUFX1tlXjng18ENheVf++xueqafcnLcN73wu7dk3+lDabJFRVpo2bZcnl\nbuDMJKcnOQG4DLht1Z2dxiTmr1wr5tJm5hm6hmDqU7iqjiS5GtjF5B+Am6pqd5KrJjfXTuCPgKcC\nf5EkwKGq2rbIiUvzZNA1BDM9havqduDsVdfd2Lv8GuA1852atHEMuobA3xSVMOgaBoMuYdA1DAZd\nwreg0zAYdAnfgk7DYNAlXHLRMBh0CYOuYTDoEgZdw2DQJQy6hsGgS7jLRcNg0CXc5aJhMOgSLrlo\nGAy6hEHXMBh0CYOuYTDoEgZdw2DQJdzlomEw6BLuctEwGHQJl1w0DAZdwqBrGAy6hEHXMBh0CYOu\nYTDoEu5y0TAYdAl3uWgYDLqESy4aBoMuYdA1DAZdwqBrGAy6hEHXMBh0CYOuYTDoEpNdLm5bVOtm\nCnqS7Un2JLk/yTVrjHlXkr1J7kly3nynKS2WZ+gagqlBT7IFuB64GDgXuDzJOavGvAx4ZlX9FHAV\ncMMC5iotjEHXEMxyhr4N2FtV+6rqEHArsGPVmB3AewCq6i7gpCSnzHWm0gIZdA3BLE/hrcD+3vEB\nJpFfb8zB7roHj2l2x2DfPrj33mXdu1rz8MMGXe3b8KfwysrKY5dHoxGj0Wgh97N7N+zcuZBPrQG6\n8EI4+eRlz0KaGI/HjMfjo/64VNX6A5LzgZWq2t4dXwtUVb2tN+YG4I6qen93vAe4oKoeXPW5atr9\nSZL+vyRUVaaNm2UN/W7gzCSnJzkBuAy4bdWY24BXdXd8PvDt1TGXJC3W1CWXqjqS5GpgF5N/AG6q\nqt1JrprcXDur6qNJLknyVeA7wJWLnbYkabWpSy5zvTOXXCTpqM1zyUWS1ACDLkkDYdAlaSAMuiQN\nhEGXpIEw6JI0EAZdkgbCoEvSQBh0SRoIgy5JA2HQ5+iJvNxlS3x87RryY4PhP75ZGfQ5GvqTysfX\nriE/Nhj+45uVQZekgTDokjQQG/7yuRt2Z5I0ILO8fO6GBl2StDguuUjSQBh0SRqIpQQ9yeuS7E5y\nX5K3LmMOi5bkd5M8muSpy57LPCX5k+7v7p4kH0xy4rLndKySbE+yJ8n9Sa5Z9nzmKcmpST6V5Evd\n19vrlz2neUuyJcm/JVn95vWDkOSkJH/ffd19KcnPrTV2w4OeZAT8CvCsqnoW8PaNnsOiJTkVeCmw\nb9lzWYBdwLlVdR6wF/iDJc/nmCTZAlwPXAycC1ye5JzlzmquDgO/U1XnAj8PvHZgjw/gDcCXlz2J\nBXon8NGq+mngOcDutQYu4wz9t4C3VtVhgKp6aAlzWLQ/A35v2ZNYhKr6ZFU92h3eCZy6zPnMwTZg\nb1Xtq6pDwK3AjiXPaW6q6j+r6p7u8sNMYrB1ubOan+7k6RLgr5c9l0XovgP+xaq6BaCqDlfV/641\nfhlBPwt4UZI7k9yR5PlLmMPCJLkU2F9V9y17Lhvg1cDHlj2JY7QV2N87PsCAgteX5OnAecBdy53J\nXH3v5Gmo2/WeATyU5JZuWWlnkievNfj4RcwgySeAU/pXMfkf/ofdff54VZ2f5AXA3wFnLGIeizLl\n8b2RyXJL/7amrPP43lRVH+nGvAk4VFXvW8IUdZSS/BjwAeAN3Zl685L8MvBgVd3TLeU297U2g+OB\n5wGvrarPJXkHcC1w3VqD566qXrrWbUl+E/hQN+7u7geHJ1fVtxYxl0VY6/El+Vng6cAXkoTJcsS/\nJtlWVd/cwCkek/X+/gCSXMHk29wLN2RCi3UQOK13fGp33WAkOZ5JzN9bVR9e9nzm6IXApUkuAZ4M\nPCXJe6rqVUue1zwdYPId/+e64w8Aa/7gfhlLLv9AF4IkZwFPainm66mqL1bVT1TVGVX1DCZ/Gc9t\nKebTJNnO5FvcS6vqu8uezxzcDZyZ5PQkJwCXAUPbLXEz8OWqeueyJzJPVfXGqjqtqs5g8vf2qYHF\nnKp6ENjftRLgItb5AfBCztCnuAW4Ocl9wHeBQf0FrFIM79vAdwMnAJ+YfBPCnVX128ud0hNXVUeS\nXM1k984W4KaqWnMXQWuSvBD4deC+JJ9n8px8Y1XdvtyZ6Si8HvjbJE8CHgCuXGugv/ovSQPhb4pK\n0kAYdEkaCIMuSQNh0CVpIAy6JA2EQZekgTDokjQQBl2SBuL/AK2CKchN/meoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10a36ffd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.arange(-5.0, 5.0, 0.1)\n",
    "y = step_function(x)\n",
    "plt.plot(x,y)\n",
    "plt.ylim(-0.1,1.1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD7CAYAAAB68m/qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGh5JREFUeJzt3XuUVNWZ9/Hvg0rGKCCIgqBihDA6eCEauYyiFYmKJpHR\nJEs0ExeQN+FVQJMYRc0b7ZXoqJm4otFRQgQVo4JCFDCoSKDwNkCj3AxXbwSaAANyTQSBft4/dok9\nnW66mq6qXXXq91nrrO7qOt31VFp+2f3sffYxd0dEREpfs9gFiIhIbijQRUQSQoEuIpIQCnQRkYRQ\noIuIJIQCXUQkIQ4u5IuZmdZIiogcAHe3hs4p+Ajd3RN73H777dFr0PvT+yu391YO7y9barmIiCSE\nAl1EJCEU6DmUSqVil5BXen+lK8nvDZL//rJljenPNPnFzLyQrycikgRmhhfjpKiIiOSHAl1EJCEU\n6CIiCaFAFxFJCAW6iEhCKNBFRBJCgS4ikhAKdBGRhGgw0M1stJmtN7NF+znnN2a20swWmFn33JYo\nIiLZyGaE/ihwUX1PmtnFQGd3/yIwBBiZo9pERKQRGgx0d38d2LyfU/oDYzPnzgFamVm73JQnIiLZ\nykUPvSOwusbjqszXRESkgAp6xyKAioqKfZ+nUintkiYiUks6nSadTjf6+7LabdHMOgFT3P20Op4b\nCcx09/GZx8uA89x9fR3nardFEYnGHbZtg40bYdMm+Oijz47Nm2HLFti6NXzcti18vn17OE48EWbN\nilN3trstZjtCt8xRl8nAUGC8mfUCttQV5iIi+bJ3L6xbB6tXQ1UVrF0bPq5bB+vXh48bNoQg/9zn\noG1bOPJIaNMmHK1bh4/HHAMnnQRHHAEtW352tGgBrVrFfpcNazDQzewpIAUcaWZ/AW4HmgPu7qPc\nfaqZXWJm7wJ/Awbls2ARKU8ffQQrVsC778J778EHH8CHH4Zj7doQ0McdBx07huOYY6BrV2jfHtq1\ng6OPhqOOgn/6p9jvJH90gwsRKSobNsCiRfDOO7BkSTiWLYPdu+GLX4QuXaBz59AC+cIXoFOnEOTN\nm8euPH+ybbko0EUkmr/+FebOhXnz4K234O23YdcuOO00OPVU6NYNTj45HEcfDdZgpCWTAl1Eikp1\ndRh1v/oqvPYazJ4NO3ZAz57w5S/DGWeE47jjyje466NAF5Ho3nsPpk2D6dMhnQ6TkX36hONf/zW0\nTxTeDVOgi0jB7dkTRt+TJ8PUqWHp34UXwle/Cn37QocOsSssTQp0ESmI3bvDCPyZZ2DKFDjhBOjf\nH772NejeHZppT9cmU6CLSN64h8nMxx6DCRPC6pMrroDLLoPjj49dXfLk+sIiERE2bYJHH4UxY8LI\nfOBAqKwMo3KJT4EuIg2aNw9+85vQUrn0Uhg1Cs4+WxOaxUYtFxGpU3V1CPB774VVq2DYMBg8OFyR\nKYWllouIHJC9e2H8eLjzTjj0ULjxRvjmN+FgpUXR069IRIAwIn/2WbjttrBe/N574aKL1FYpJQp0\nEeGVV2DEiLDE8MEHw7pxBXnpUaCLlLEVK+DHP4bly+Guu0JrRUFeurTkX6QM/e1vcNNN4fL7VCrs\nsfKtbynMS51G6CJl5o9/hKFD4ZxzQpC3bx+7IskVBbpImdi0CYYPDxcCPfJI6JNLsqjlIlIGpkwJ\ne4y3bw8LFyrMk0ojdJEE+/vf4Uc/CqtYnn4azj03dkWSTxqhiyTU4sVw1lnhJhILFijMy4ECXSSB\nHn8czj8/rGT5/e/Dnesl+dRyEUmQXbvghz+EGTPCHYK6dYtdkRSSAl0kIdavD/uRt2sX9ipv1Sp2\nRVJoarmIJMDixeFmyxdcABMnKszLlUboIiXupZfg6qvhvvvgqqtiVyMxKdBFStjjj4dNtZ5/PlzG\nL+VNgS5Sgtzhnntg5Mgw+XnSSbErkmKgQBcpMe5hVP7ii/DGG9CxY+yKpFgo0EVKSHU1XHcdzJkD\ns2ZBmzaxK5JiktUqFzPrZ2bLzGyFmY2o4/mWZjbZzBaY2WIzG5jzSkXKXHU1/OAH4arP6dMV5vKP\nGrxJtJk1A1YAfYG1QCUwwN2X1TjnFqClu99iZm2B5UA7d99T62fpJtEiB6C6GoYMgZUr4YUX4PDD\nY1ckhZTtTaKzGaH3AFa6+yp33w2MA/rXOseBFpnPWwCbaoe5iBwYdxg2DJYuVZjL/mXTQ+8IrK7x\neA0h5Gt6EJhsZmuBw4ErclOeSHlzD7eIe/ttmDZNYS77l6tJ0YuA+e5+vpl1Bl4xs9PcfUftEysq\nKvZ9nkqlSKVSOSpBJHnuuANmzgyHNtgqH+l0mnQ63ejvy6aH3guocPd+mcc3A+7u99Q45wXgLnd/\nI/P4T8AId59X62ephy6SpZEj4T//MyxN1G3iylsue+iVQBcz62RmzYEBwORa56wCvpp54XZAV+D9\nxpUsIp+aMAF+8YvQZlGYS7YabLm4+14zGwZMI/wfwGh3X2pmQ8LTPgq4A3jMzBZlvu0md/8ob1WL\nJNgbb8C114Yw79w5djVSShpsueT0xdRyEdmvlSuhT5+wR8tFF8WuRopFLlsuIlIAGzfCJZeEVovC\nXA6ERugiReCTT+DCC8Oe5vfc0/D5Ul6yHaEr0EWKwDXXQFVV2Aa3mf5ullqyDXRtziUS2UMPwWuv\nwZtvKsylaTRCF4lo1iy44oqwskUrWqQ+mhQVKXJVVXDllTB2rMJcckOBLhLBJ5/At78NQ4eGyVCR\nXFDLRSSC4cNh1SpNgkp2NCkqUqSeeQamToW33lKYS25phC5SQO++C717w0svwZlnxq5GSoUmRUWK\nzK5dYUXLbbcpzCU/NEIXKZDrrgsrWyZMAGtwrCXyGfXQRYrICy/A5Mkwf77CXPJHgS6SZ+vWwfe/\nHyZDW7eOXY0kmXroInlUXQ0DB4ZA79MndjWSdAp0kTx64AHYujVMhIrkmyZFRfJkyRI47zyYPVuX\n9kvTaNmiSES7d8PVV8OddyrMpXAU6CJ5cMcdcPTRoXcuUiha5SKSY3PnwsiRsGCBlihKYWmELpJD\nO3eGVS333w/HHBO7Gik3mhQVyaFbboEVK3Q1qOSWrhQVKbC5c2HMGFi0SGEucajlIpIDu3bBoEGh\n1dKuXexqpFwp0EVy4I47oGvXsJuiSCzqoYs00aJF0LcvLFwIHTrErkaSSBcWiRTAnj3wve/B3Xcr\nzCU+BbpIE9x3H7RsCYMHx65EJMtAN7N+ZrbMzFaY2Yh6zkmZ2Xwze8fMZua2TJHi8/77YWQ+apRW\ntUhxaLCHbmbNgBVAX2AtUAkMcPdlNc5pBbwJXOjuVWbW1t031vGz1EOXRHCHfv1C7/ymm2JXI0mX\nyx56D2Clu69y993AOKB/rXOuAia6exVAXWEukiRPPQXr18OPfhS7EpHPZBPoHYHVNR6vyXytpq5A\nGzObaWaVZvbdXBUoUmw2bYIbbgitlkMOiV2NyGdydaXowcAZwPnAYcB/m9l/u/u7Ofr5IkXjxhvD\nevMePWJXIvK/ZRPoVcDxNR4fm/laTWuAje6+E9hpZq8CpwP/EOgVFRX7Pk+lUqRSqcZVLBLRq6/C\nK6+Em1eI5Es6nSadTjf6+7KZFD0IWE6YFP0rMBe40t2X1jjnJOABoB/wOWAOcIW7L6n1szQpKiXr\nk0+ge/dwVejll8euRspJzjbncve9ZjYMmEbouY9296VmNiQ87aPcfZmZvQwsAvYCo2qHuUipu/de\nOPFEuOyy2JWI1E2X/otk4f33Q8983jw44YTY1Ui50aX/IjniDsOHw09+ojCX4qb90EUa8Pzz8MEH\n8NxzsSsR2T+1XET2Y8cO+Jd/gbFjQQuyJJZsWy4KdJH9GDEC1q6FJ56IXYmUMwW6SBMtWQLnnQeL\nF0P79rGrkXKmSVGRJnCHa6+F225TmEvpUKCL1OHpp2HrVrjmmtiViGRPLReRWrZuhZNPhokToXfv\n2NWIqIcucsB++EPYvh1Gj45diUiQs0v/RcrJokVhr/M//zl2JSKNpx66SManE6E//zkcdVTsakQa\nT4EukvHEE7BzJ3z/+7ErETkw6qGLAFu2hInQSZN04wopPpoUFWmE66+Hjz8Ot5UTKTaaFBXJ0sKF\nYd257kIkpU49dClr7jB0aJgIbds2djUiTaNAl7L2xBOwa5cmQiUZ1EOXsrVlS9gad9IkOOus2NWI\n1E+ToiIN0ESolApNiorsx8KFMG6crgiVZFEPXcpOdXW4IvSOOzQRKsmiQJeyM3Ys7NkD3/te7EpE\ncks9dCkrmzeHK0L/+Ec488zY1YhkR5OiInW49trw8aGH4tYh0hiaFBWppbISnntOV4RKcqmHLmVh\n795wO7l77oHWrWNXI5IfCnQpC7/9LXz+8/Dd78auRCR/1EOXxFu/Hk45BWbODB9FSk22PfSsRuhm\n1s/MlpnZCjMbsZ/zzjKz3WZ2eWOKFcmnG26AQYMU5pJ8DU6Kmlkz4EGgL7AWqDSzSe6+rI7z7gZe\nzkehIgdi+nR4/XVdESrlIZsReg9gpbuvcvfdwDigfx3nDQcmABtyWJ/IAdu5MyxTfPBBOOyw2NWI\n5F82gd4RWF3j8ZrM1/Yxsw7Av7n7w0CDfR6RQrj77tBm+frXY1ciUhi5Wod+H1Czt65Ql6iWLw8j\n8/nzY1ciUjjZBHoVcHyNx8dmvlbTl4FxZmZAW+BiM9vt7pNr/7CKiop9n6dSKVKpVCNLFtk/dxgy\nBH72MzjuuNjViDReOp0mnU43+vsaXLZoZgcBywmTon8F5gJXuvvSes5/FJji7n+o4zktW5S8GzMG\nHn4YZs+Ggw6KXY1I0+Xs0n9332tmw4BphJ77aHdfamZDwtNe+/YASmyJZsMGuOUWePllhbmUH11Y\nJIny7/8OHTrAL38ZuxKR3NHmXFJ2pk6FN9+ExYtjVyIShwJdEmH79rD51pgxWnMu5UstF0mEYcPC\nDZ9Hj45diUjuqeUiZeP118M+5++8E7sSkbi0fa6UtI8/DvcGfeAB7XMuopaLlLSf/ATWrIFx42JX\nIpI/arlI4r35Jjz5pFa1iHxKLRcpSR9/DIMHh/1a2raNXY1IcVDLRUrSDTeEVsv48bErEck/tVwk\nsdLp0DNftCh2JSLFRS0XKSnbtsHAgfC738GRR8auRqS4qOUiJWXwYDjkEPjtb2NXIlI4arlI4jz3\nHMyaBQsXxq5EpDhphC4loaoKzjgDJk2CXr1iVyNSWNmO0NVDl6JXXQ1XXx32a1GYi9RPgS5F71e/\ngk8+gVtvjV2JSHFTy0WK2ty58PWvQ2UldOoUuxqRONRykZK3ZQtccQWMHKkwF8mGRuhSlNzhW98K\nt5N74IHY1YjEpWWLUtL+67/gww/hqadiVyJSOjRCl6IzZw584xthN8UuXWJXIxKfeuhSkjZsgG9/\nO1zarzAXaRyN0KVo7NkDF14IvXvDnXfGrkakeGQ7QlegS9G46SaYPx9eegkOOih2NSLFQ5OiUlJ+\n/3uYODH0zxXmIgdGI3SJ7tOLh2bMgFNOiV2NSPHRpKiUhKoquPxyeOQRhblIUynQJZrt28PIfOhQ\nuPTS2NWIlD61XCSKPXugf3845piwRNEa/GNSpHzltOViZv3MbJmZrTCzEXU8f5WZLcwcr5vZqQdS\ntJQHdxg+HHbvhocfVpiL5EqDq1zMrBnwINAXWAtUmtkkd19W47T3gXPdfauZ9QN+B2jnaqnTXXfB\nG2/Aa6+F28mJSG5ks2yxB7DS3VcBmNk4oD+wL9DdfXaN82cDHXNZpCTHqFFhAvT116FVq9jViCRL\nNi2XjsDqGo/XsP/A/j/Ai00pSpJp4kSoqIBp08IuiiKSWzm9sMjMvgIMAs6p75yKiop9n6dSKVKp\nVC5LkCL10ktwzTXw8svao0WkIel0mnQ63ejva3CVi5n1AircvV/m8c2Au/s9tc47DZgI9HP39+r5\nWVrlUoamT4errgo3eO7dO3Y1IqUnl6tcKoEuZtbJzJoDA4DJtV7seEKYf7e+MJfyNGsWXHklTJig\nMBfJtwZbLu6+18yGAdMI/wcw2t2XmtmQ8LSPAn4GtAEeMjMDdrt7j3wWLsVvxgwYMADGj4dzz41d\njUjy6cIiyYupU2HgQHj2WTjvvNjViJQ27eUi0fzhDzBoEEyerDAXKSQFuuTUyJEwbFhY1dJLl5aJ\nFJT2Q5eccA9rzJ98MlwB2rlz7IpEyo8CXZps1y4YMgTeeSdc0t+uXeyKRMqTWi7SJBs3wgUXwLZt\nYYmiwlwkHgW6HLDFi6FnTzjnnLDO/LDDYlckUt7UcpED8tRTcP31cN998J3vxK5GRECBLo20axfc\neGNYZ/6nP8Fpp8WuSEQ+pZaLZG358nD5/urVMG+ewlyk2CjQpUHuMHp06JX/4AfhwqEjjohdlYjU\nppaL7NfatSHE16yBmTPhlFNiVyQi9dEIXerkDmPHwpe+BGeeCXPnKsxFip1G6PIPVqwIN6PYsiVM\nfp55ZuyKRCQbGqHLPjt2wE9/CmefDZdeCnPmKMxFSokCXaiuhscfh3/+57CCZcGCsMb8YP39JlJS\n9E+2jLmHlsqtt8LnPx9Wr/TsGbsqETlQCvQy5A6vvAI//3nok995Z2ixWIPb54tIMVOgl5Hqapgy\nBf7jP2D79tAvHzAADjoodmUikgsK9DLw97+HHvmvfw0tW8LNN8Pll0MzzaCIJIoCPcGWLQt3EHri\niXCV5yOPQJ8+aq2IJJUCPWG2bw83Zn7ssbCefPBgeOstOOGE2JWJSL6Zuxfuxcy8kK9XLnbuDPfw\nHDcufEylwk2aL74YmjePXZ2INJWZ4e4N/m2tQC9R27bBiy/C88+HEO/ePUxwfvOb0LZt7OpEJJcU\n6AlTXR3u2TltWlg7XlkZ+uKXXRaWHLZvH7tCEckXBXqJc4elS+HVV8O9OmfMgBYtwv07L7kEvvIV\nOPzw2FWKSCEo0EvMpk1h8nLOnHDMng2tWoVVKeeeC337QqdOsasUkRgU6EVqzx744IPQPlm4EBYt\ngrffhs2bQx+8Z0/o1SscHTrErlZEioECPSJ3WL8e3nsvHCtWhGP5cli5MvS7u3WD008Pt3Hr3h26\ndNGFPiJSt5wGupn1A+4j7M442t3vqeOc3wAXA38DBrr7gjrOKflAd4etW8OdfNauhaqqcDef1avh\nL3+BDz+EVavgsMPgxBOhc2fo2vWz46STwnMiItnKWaCbWTNgBdAXWAtUAgPcfVmNcy4Ghrn718ys\nJ3C/u/eq42cVVaC7h8vit2wJLY/Nm+Gjj0I/e+PG8PF//iccGzaEUfe6dWFtd4cO0LFj+HjccXDs\nsXD88eECnk6dwgSmiEguZBvo2Vwp2gNY6e6rMj94HNAfWFbjnP7AWAB3n2Nmrcysnbuvb3zpuTF7\nNjz9dLhpw44d4QrKT49t28Ioe9u2EM5HHBGO1q2hTZtwtG0LRx4ZRtVHHRWO9u2hXbuw1ayISLHJ\nJtA7AqtrPF5DCPn9nVOV+Vq0QD/00NDyOPzw0OJo0eKzo1WrsElVy5a6klJEkqPge7lUVFTs+zyV\nSpFKpfLyOqefHg4RkVKTTqdJp9ON/r5seui9gAp375d5fDPgNSdGzWwkMNPdx2ceLwPOq91yKbYe\nuohIKci2h57NQrlKoIuZdTKz5sAAYHKtcyYDV2deuBewJWb/XESkHDXYcnH3vWY2DJjGZ8sWl5rZ\nkPC0j3L3qWZ2iZm9S1i2OCi/ZYuISG26sEhEpMjlsuUiIiIlQIEuIpIQCnQRkYRQoIuIJIQCXUQk\nIRToIiIJoUAXEUkIBbqISEIo0EVEEkKBLiKSEAr0HDqQ7S5Lid5f6Urye4Pkv79sKdBzKOn/Uen9\nla4kvzdI/vvLlgJdRCQhFOgiIglR8O1zC/ZiIiIJks32uQUNdBERyR+1XEREEkKBLiKSEFEC3cyG\nm9lSM1tsZnfHqCHfzOwGM6s2szaxa8klM/tl5ne3wMwmmlnL2DU1lZn1M7NlZrbCzEbErieXzOxY\nM5thZn/O/Hu7LnZNuWZmzczsbTOrffP6RDCzVmb2bObf3Z/NrGd95xY80M0sBXwDONXdTwV+Vega\n8s3MjgUuAFbFriUPpgHd3L07sBK4JXI9TWJmzYAHgYuAbsCVZnZS3Kpyag/wY3fvBvQGhibs/QFc\nDyyJXUQe3Q9MdfeTgdOBpfWdGGOEfg1wt7vvAXD3jRFqyLdfAzfGLiIf3H26u1dnHs4Gjo1ZTw70\nAFa6+yp33w2MA/pHriln3H2duy/IfL6DEAYd41aVO5nB0yXAI7FryYfMX8B93P1RAHff4+7b6js/\nRqB3Bc41s9lmNtPMvhyhhrwxs0uB1e6+OHYtBTAYeDF2EU3UEVhd4/EaEhR4NZnZCUB3YE7cSnLq\n08FTUpfrfQHYaGaPZtpKo8zs0PpOPjgfFZjZK0C7ml8i/A/+/zKv2drde5nZWcAzwIn5qCNfGnh/\ntxLaLTWfKyn7eX8/dfcpmXN+Cux296cilCiNZGaHAxOA6zMj9ZJnZl8D1rv7gkwrt+T+rWXhYOAM\nYKi7zzOz+4CbgdvrOznn3P2C+p4zs/8L/CFzXmVm4vBId9+Uj1ryob73Z2anACcAC83MCO2It8ys\nh7tvKGCJTbK/3x+AmQ0k/Jl7fkEKyq8q4Pgaj4/NfC0xzOxgQpg/4e6TYteTQ2cDl5rZJcChQAsz\nG+vuV0euK5fWEP7in5d5PAGod+I+RsvleTJBYGZdgUNKKcz3x93fcff27n6iu3+B8Mv4UimFeUPM\nrB/hT9xL3X1X7HpyoBLoYmadzKw5MABI2mqJMcASd78/diG55O63uvvx7n4i4fc2I2FhjruvB1Zn\nshKgL/uZAM7LCL0BjwJjzGwxsAtI1C+gFid5fwY+ADQHXgl/hDDb3a+NW9KBc/e9ZjaMsHqnGTDa\n3etdRVBqzOxs4DvAYjObT/hv8lZ3fyluZdII1wFPmtkhwPvAoPpO1KX/IiIJoStFRUQSQoEuIpIQ\nCnQRkYRQoIuIJIQCXUQkIRToIiIJoUAXEUkIBbqISEL8fxoite9SSC6aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10a38d278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.arange(-5.0, 5.0, 0.1)\n",
    "y = sigmoid(x)\n",
    "plt.plot(x,y)\n",
    "plt.ylim(-0.1,1.1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def relu(x):\n",
    "    return np.maximum(0,x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD8CAYAAABekO4JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEsZJREFUeJzt3X1sneV5x/HflRLakLBqLClBjUrKkgxCgCZlsCmMnVFV\nRXSjfxCr7Sa1y8vSjYCjpo220akxcpUgFKsUQiOglLzQdpDgrqvUBDORRyFGjIRhEl6SVpqWwYbD\nWN1EVmXqzNf+eM4JxsT2eXnOuZ+X70eyOI4fn1wnxpduX76f+2fuLgBAek0JXQAAYGI0agBIORo1\nAKQcjRoAUo5GDQApR6MGgJQ7p5qLzOw/JJ2UNCJp2N2vaWZRAIB3VdWoFTfokrsPjHeBmbEhGwBq\n5O422TXVjj6smmvdPZdvGzZsCF4Dr4/Xx+vL31u1qm3ULukpMztoZn9V9bMDABpW7ehjqbu/aWaz\nFDfs19z9QDMLAwDEqmrU7v5m+b//Y2Y/lnSNpPc16o6OjjOPS6WSSqVSIkWGlpfXMR5eX7bx+rIj\niiJFUVTz59lkcxIzO0/SFHcfNLPpknok3enuPWOu81pmLgBQdGYmr+KXidWsqC+U9OPyro5zJP1g\nbJMGADTPpCvqqp+IFTWADIoi6dprpWnTWv93V7ui5s5EAIV17Ji0bJn01luhK5kYjRpAIY2MSKtW\nSRs2SBdfHLqaidGoARTS1q2Su7RmTehKJseMGkDhHD8uXX219Mwz0qWXhquDGTUAnIW79JWvSF/9\natgmXQsaNYBC2blTOnFCWr8+dCXVY/QBoDD6+6WrrpL27pUWLw5dTfWjDxo1gMJoa5Pmz5c2bgxd\nSSzJOxMBIPO6u6XDh+PRR9awogaQewMD0uWXS48/Ll13Xehq3sXoAwDKli+XZsyQ7rsvdCXvxegD\nACT19Ej79klHjoSupH5szwOQW4OD8Z7pBx6Qzj8/dDX1Y/QBILfa26WTJ6Xt20NXcnaMPgAUWm+v\ntHu39PLLoStpHKMPALkzNCStXBn/8vCCC0JX0zgaNYDc6eyMt+PdckvoSpLB6ANArvT1SQ89FN/c\nkhesqAHkxvCwtGKFdPfd0uzZoatJDo0aQG50dUmzZklf/nLoSpLF9jwAuXDsWHx7+MGD0ty5oaup\nDsEBAAqjkn/4zW9mp0nXgkYNIPOylH9YD0YfADItLfmH9WD0ASD3KvmH69Zlr0nXgkYNILMq+Ydf\n/3roSpqL0QeATKrkH+7ZIy1ZErqa+hAcACDXli2TFixIT/5hPTg9D0BuPfFEHATw6KOhK2kNVtQA\nMuWXv5QWLUpf/mE9GH0AyKXly6Xp06UtW0JX0jhGHwByJw/5h/Vgex6ATBgclFavzn7+YT0YfQDI\nhPZ26dQpadu20JUkJ/HRh5lNkXRI0hvufnMjxQFALZ59Vtq1S3rlldCVhFHL6GOtpFebVQgAnM3Q\nUBwGkJf8w3pU1ajNbI6kmyR9r7nlAMB7VfIPly0LXUk41Y4+vi1pvaQPN7EWAHiPSv7hSy+FriSs\nSVfUZvZZSSfcvU+Sld8AoKlG5x9edFHoasKqZkW9VNLNZnaTpGmSzjezHe7+pbEXdnR0nHlcKpVU\nKpUSKhNA0XR1STNn5iv/MIoiRVFU8+fVtD3PzP5Y0tfOtuuD7XkAknLsmLR0qXToUD6jtSoIDgCQ\nSZX8ww0b8t2ka8ENLwBS5f77pR/9SNq/X5qS86UkhzIByJws5x/Wg9EHgEwpSv5hPWjUAFKhKPmH\n9WD0ASC4Sv7h3r3S4sWhq2kdZtQAMmPZMmn+fGnTptCVtBbBAQAyoWj5h/VgRQ0gmIGB+MClPOQf\n1oPRB4DUW75cmjEjPsK0iBh9AEi1nh7p6aell18OXUn6sT0PQMtV8g8ffLB4+Yf1YPQBoOXymH9Y\nD0YfAFKpt1favZuRRy0YfQBomaGh+GS8e+8tbv5hPWjUAFqms1O67LJi5x/Wg9EHgJYg/7B+rKgB\nNB35h42hUQNouq4uadasfOUfthLb8wA0VVHyD+tBcACA4Mg/TAaNGkDTfPe7cXLLmjWhK8k2Rh8A\nmuL4cemTn5QOHCBaazyMPgAEQ/5hsmjUABK3Y0ecf7h+fehK8oHRB4BE9fdLV14pPflksfIP60Fw\nAIAg2tri/MONG0NXkn6cngeg5bq7pcOHpZ07Q1eSL6yoASSi6PmH9WD0AaClVqyQpk8vbv5hPRh9\nAGiZSv7hkSOhK8kntucBaMjgYLxn+oEHyD9sFkYfABqydq30q19J27eHriR7GH0AaLreXmnXLvIP\nm43RB4C6VPIP77uP/MNmo1EDqEtnp7RwoXTLLaEryb9JRx9m9kFJ+yWdW75+t7vf2ezCAKRXJf/w\n8OHQlRTDpI3a3d8xsz9x91+b2Qck9ZrZHnd/vgX1AUiZ0fmHs2eHrqYYqhp9uPuvyw8/qLi5s70D\nKKjNm8k/bLWqdn2Y2RRJL0j6XUn3u/vBplYFIJWOHo2Dag8dkmzSTWVISrUr6hF3XyxpjqRrzWxh\nc8sCkDbkH4ZT0z5qdz9lZvsk3Sjp1bEf7+joOPO4VCqpVCo1WB6AtCD/sHFRFCmKopo/b9I7E81s\npqRhdz9pZtMkPSnpLnf/2ZjruDMRyCnyD5sjyTsTL5K0vTynniLpsbFNGkB+kX8YHmd9AJjQ9u3S\nPfdIzz8vTZ0aupp84TxqAA2r5B/u3SstWRK6mvyhUQNoWFubNG+etGlT6EryidPzADSE/MP0YEUN\n4H3IP2wNRh8A6rZihXTeedKWLaEryTdGHwDqQv5h+nAeNYAzyD9MJ0YfAM5Yu1Y6eVLati10JcXA\n6ANATZ59lvzDtGL0AUBDQ9LKleQfphWNGoC+9a14Ox75h+nE6AMouL4+6cEHyT9MM1bUQIGdPh2P\nPMg/TDcaNVBgmzdLM2eSf5h2bM8DCurYMWnp0jj/kGitMKrdnseKGigg8g+zhUYNFBD5h9nC6AMo\nGPIP04PRB4D3If8wm2jUQIHs2CGdOCGtXx+6EtSC0QdQEOQfpg/BAQDeg/zD9OH0PABnkH+Ybayo\ngZwj/zC9GH0AkBTnH06fHh9hinRh9AFATz1F/mEesD0PyKnBQWn1avIP84DRB5BT5B+mH6MPoMB6\ne8k/zBNGH0DODA3FJ+ORf5gfNGogZzo7pYULyT/ME0YfQI709UkPPUT+Yd6wogZyYng43jNN/mH+\n0KiBnOjqkmbNIv8wjybdnmdmcyTtkHShpBFJD7n7vWe5ju15QCDkH2ZTYreQm9lsSbPdvc/MZkh6\nQdLn3P3omOto1EAAIyPS9ddLn/+8dPvtoatBLRJLeHH3fnfvKz8elPSapI82XiKAJGzdGv+X/MP8\nqunORDObKymStKjctEd/jBU10GLkH2Zb4pmJ5bHHbklrxzZpAK1H/mFxVLWP2szOUdykd7r7T8a7\nrqOj48zjUqmkUqnUYHkAxrNzJ/mHWRNFkaIoqvnzqhp9mNkOSW+7+7oJrmH0AbRIf7901VVx/uHi\nxaGrQb2S3PWxVNJ+SUckefntDnffO+Y6GjXQIm1t0vz50saNoStBIxI7Pc/deyV9IJGqADSsuzsO\nAiD/sDg4jxrIEPIP84XMRCCHyD/MF4IDgJzp6SH/sKg4lAnIgMHBeM80+YfFxOgDyID29jj/cPv2\n0JUgSYw+gJzo7ZV27yb/sMgYfQApNjQkrVxJ/mHR0aiBFOvsjLfjkX9YbIw+gJR68cU4//Cll0JX\ngtBYUQMpNDr/8KKLQleD0GjUQAp1dUkf+Qj5h4ixPQ9IGfIPiyPx4AAAzTcyIq1aJW3YQJPGu2jU\nQIps3Ro3a/IPMRqjDyAlyD8sHkYfQIaQf4iJ0KiBFCD/EBNh9AEEVsk/3LNHWrIkdDVoJYIDgIxo\na5PmzZM2bQpdCVqN0/OADOjulg4fJv8QE2NFDQQyMCAtWiQ99hj5h0XF6ANIuRUrpPPOk7ZsCV0J\nQmH0AaQY+YeoBdvzgBYbHJRWryb/ENVj9AG0WHu7dOqUtG1b6EoQGqMPIIXIP0Q9GH0ALUL+IepF\nowZahPxD1IvRB9AC5B+iEayogSYbHo5HHuQfol40aqDJurqkmTPJP0T92J4HNBH5h5gIwQFAYCMj\n8ciD/EM0ikYNNMnWrXFyy623hq4EWTfp6MPMHpb0p5JOuPuVE1zH6AMoq+QfPvOMdNlloatBWiU5\n+nhE0mcaLwkohtH5hzRpJGHSRu3uByQNtKAWIBfIP0TSuOEFSFB/f9yg9+6Vpk4NXQ3ygl8mAgm6\n/fZ4p8fixaErQZ4kuqLu6Og487hUKqlUKiX59ECqdXfHQQDkH2I8URQpiqKaP6+qG17MbK6kn7r7\nFRNcw64PFNbAQHzg0q5d8Q0uQDUS2/VhZj+U9KykBWb2n2a2PIkCgTxZty4+FY8mjWaYdPTh7n/e\nikKArOrpkfbtI/8QzcMvE4EGDA7Ge6bJP0QzcSgT0ADyD9EIMhOBJiP/EK3C6AOow9CQtGoV+Ydo\nDRo1UIfOzvgcD/IP0QqMPoAa9fWRf4jWYkUN1GB4WFqxgvxDtBaNGqgB+YcIge15QJXIP0TSyEwE\nEkT+IUKiUQNVqOQfrlkTuhIUEaMPYBKV/MMDB6RLLw1dDfKE0QeQgNH5hzRphEKjBiawYwf5hwiP\n0Qcwjv5+6corpSefJFoLzVHt6INGDYxj2TJpwQJp48bQlSCvOD0PaEAl//DRR0NXArCiBt6nkn/4\n+OPSddeFrgZ5xugDqNPy5dKMGfERpkAzMfoA6kD+IdKI7XlAGfmHSCtGH0BZe7t08qS0fXvoSlAU\njD6AGpB/iDRj9IHCI/8QaUejRuF1dkoLF5J/iPRi9IFCI/8QWcCKGoV1+jT5h8gGGjUKa/NmadYs\n8g+RfmzPQyGRf4g0IDgAGMfISLzLg/xDZAWNGoWzdWvcrMk/RFYw+kChkH+INGH0AYxB/iGyikaN\nwti5k/xDZFNVjdrMbjSzo2b2czP722YXBSStvz9u0A8/LE2dGroaoDaTzqjNbIqkn0v6lKT/lnRQ\n0hfc/eiY65hRI7Xa2qR586RNm0JXArwrydPzrpH0C3c/Xn7if5T0OUlHJ/wsICW6u6XDh+PRB5BF\n1Yw+Pirp9VHvv1H+MyD1Bgak226LRx4f+lDoaoD6ZO5Qpt/8hm84VM893uVBSC2yrJpG/V+SPjbq\n/TnlP3ufjo6OM49LpZJKpVIDpZ3d1KnxYTqtFEVRU15LWuT99e3fH0kqBa6iefL+9cvT64uiSFEU\n1fx51TTqg5LmmdnFkt6U9AVJXzzbhaMbdbOYxW+ttH9/pBtuKLX2L22hvL++PH2jnw2vLzvGLmDv\nvPPOqj5v0kbt7v9nZrdJ6lE8037Y3V+rr0wAQK2qmlG7+15Jv9fkWgAAZ5HoWR+JPBEAFEg1+6gT\na9QAgObgrA8ASDkaNQCkXKKN2sxuN7PXzOyImd2V5HOnhZl9zcxGzOyC0LUkyczuLn/t+szsCTP7\nrdA1NSrPh4mZ2Rwze9rMXil/v7WHrqkZzGyKmf2bmf1z6FqSZmYfNrNd5e+7V8zs2vGuTaxRm1lJ\n0p9JusLdr5C0OannTgszmyPp05KOh66lCXokXe7un5D0C0l/H7iehpQPE9si6TOSLpf0RTPL0ynU\npyWtc/fLJf2hpDU5e30VayW9GrqIJvmOpJ+5+2WSrpI07rbnJFfUfyPpLnc/LUnu/naCz50W35aU\ny9OM3f1f3H2k/O5ziu9AzbIzh4m5+7CkymFiueDu/e7eV348qPibPFdn8JQXRjdJ+l7oWpJW/on1\nj9z9EUly99Pufmq865Ns1AskXW9mz5nZPjO7OsHnDs7Mbpb0ursfCV1LC6yQtCd0EQ0qzGFiZjZX\n0ick/WvYShJXWRjlcWvaxyW9bWaPlEc7D5rZtPEurulQJjN7StKFo/9I8T/iP5Sf67fd/Q/M7Pcl\nPS7pkprLD2iS13eH4rHH6I9lygSv7xvu/tPyNd+QNOzuPwxQImpkZjMk7Za0tryyzgUz+6ykE+7e\nVx6rZu77bRLnSFoiaY27HzKzeyT9naQN411cNXf/9HgfM7O/ltRdvu5g+Rduv+Pu/1vL3xHSeK/P\nzBZJmivpJTMzxWOBF8zsGnd/q4UlNmSir58kmdlfKv5R84aWFNRcVR8mllVmdo7iJr3T3X8Sup6E\nLZV0s5ndJGmapPPNbIe7fylwXUl5Q/FP6IfK7++WNO4vvJMcffyTyt/gZrZA0tQsNemJuPvL7j7b\n3S9x948r/kdenKUmPRkzu1Hxj5k3u/s7oetJwJnDxMzsXMWHieVt58D3Jb3q7t8JXUjS3P0Od/+Y\nu1+i+Gv3dI6atNz9hKTXy71SihO0xv2laZLnUT8i6ftmdkTSO5Jy8496Fq78/Sh2n6RzJT0V/9Cg\n59z91rAl1S/vh4mZ2VJJfyHpiJm9qPj/yTvK5/IgG9ol/cDMpkr6d0nLx7uQW8gBIOW4MxEAUo5G\nDQApR6MGgJSjUQNAytGoASDlaNQAkHI0agBIORo1AKTc/wN7JDy/LKRnbQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10f4cf160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = np.arange(-5.0, 5.0, 0.1)\n",
    "y = relu(x)\n",
    "plt.plot(x,y)\n",
    "plt.ylim(-0.1,5.1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Functions for the output layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def identity_function(a):\n",
    "    return a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def softmax(a):\n",
    "    c = np.max(a)\n",
    "    exp_a = np.exp(a-c) # to avoid overflow\n",
    "    sum_exp_a = np.sum(exp_a)\n",
    "    y = exp_a / sum_exp_a\n",
    "    return y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Basic Neural Network (Forward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.31682708  0.69627909]\n"
     ]
    }
   ],
   "source": [
    "# input\n",
    "X = np.array([1.0, 0.5])\n",
    "\n",
    "# 1st layer\n",
    "W1 = np.array([[0.1, 0.3, 0.5], [0.2, 0.4, 0.6]]) # weight of the 1st layer\n",
    "B1 = np.array([0.1, 0.2, 0.3]) # bias of the 1st layer\n",
    "\n",
    "A1 = np.dot(X, W1) + B1 # apply weight & bias\n",
    "Z1 = sigmoid(A1) # activation function\n",
    "\n",
    "# 2nd layer\n",
    "W2 = np.array([[0.1,0.4],[0.2,0.5],[0.3,0.6]])\n",
    "B2 = np.array([0.1, 0.2])\n",
    "\n",
    "A2 = np.dot(Z1, W2) + B2\n",
    "Z2 = sigmoid(A2)\n",
    "\n",
    "# output layer\n",
    "W3 = np.array([[0.1, 0.3], [0.2, 0.4]])\n",
    "B3 = np.array([0.1, 0.2])\n",
    "\n",
    "A3 = np.dot(Z2, W3) + B3\n",
    "Y = identity_function(A3)\n",
    "\n",
    "print(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def init_network():\n",
    "    network = {}\n",
    "    network['W1'] = np.array([[0.1, 0.3, 0.5], [0.2, 0.4, 0.6]])\n",
    "    network['b1'] = np.array([0.1, 0.2, 0.3])\n",
    "    network['W2'] = np.array([[0.1,0.4],[0.2,0.5],[0.3,0.6]])\n",
    "    network['b2'] = np.array([0.1, 0.2])\n",
    "    network['W3'] = np.array([[0.1, 0.3], [0.2, 0.4]])\n",
    "    network['b3'] = np.array([0.1, 0.2])\n",
    "    \n",
    "    return network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def forward(network, x):\n",
    "    W1, W2, W3 = network['W1'], network['W2'], network['W3']\n",
    "    b1, b2, b3 = network['b1'], network['b2'], network['b3']\n",
    "    \n",
    "    a1 = np.dot(x, W1) + b1\n",
    "    z1 = sigmoid(a1)\n",
    "    a2 = np.dot(z1, W2) + b2\n",
    "    z2 = sigmoid(a2)\n",
    "    a3 = np.dot(z2, W3) + b3\n",
    "    y = identity_function(a3)\n",
    "    \n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.31682708  0.69627909]\n"
     ]
    }
   ],
   "source": [
    "network = init_network()\n",
    "x = np.array([1.0, 0.5])\n",
    "y = forward(network, x)\n",
    "print(y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}