ANN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>\n",
       "table {float:left}\n",
       "</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%html\n",
    "<style>\n",
    "table {float:left}\n",
    "</style>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 Define Training Data\n",
    "## A xor B\n",
    "\n",
    "| A   | B   | A xor B  |\n",
    "|:---:|:---:|:--------:|\n",
    "|  F  |  F  |    F     |\n",
    "|  F  |  T  |    T     |\n",
    "|  T  |  F  |    T     |\n",
    "|  T  |  T  |    F     |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Define A XOR B\n",
    "#              A, B\n",
    "x = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])\n",
    "\n",
    "#              T/F\n",
    "t = np.array([[0], [1], [1], [0]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 2 Define a Neural Network Layer\n",
    "\n",
    "For each layer $l$, define:\n",
    "\n",
    "1) The input as $a_{l-1}$\n",
    "\n",
    "2) The output as $a_{l}$\n",
    "\n",
    "3)  $ a_{l} = f_{l} ( W_{l-1}^T a_{l-1} ) $\n",
    "\n",
    "Where $W_{l-1}$ is the inter-connecting weights between 2 layers.\n",
    "\n",
    "Therefore the output of the $(l-1)$-th layer is the input of the $l$-th layer and that forms the Multi-layer Nerual Netwrok.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# sigmoid is one of the most common activation function = \n",
    "def sigmoid(v):\n",
    "    return 1/(1+np.exp(-v))\n",
    "\n",
    "def d_sigmoid(sigmoided_v):\n",
    "    return sigmoided_v * (1 - sigmoided_v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class Layer:\n",
    "    def __init__(self, output_dimension, activation_function):\n",
    "        # the number of neruals of this layer\n",
    "        self.output_dimension = output_dimension\n",
    "        \n",
    "        # the activation function of this layer (f_l)\n",
    "        self.activation_function = activation_function\n",
    "        \n",
    "        # define the output of this layer (a_l)\n",
    "        self.output_matrix = np.random.rand(self.output_dimension, 1)\n",
    "    \n",
    "    # caluate this layer output a_l by the previous layer output a_l-1\n",
    "    def feed_forward(self, previous_output, previous_weight):\n",
    "        # previous_output: number_sample * 2\n",
    "        # previous_weight: 2 * self.output_dimension\n",
    "        self.output_matrix = self.activation_function(previous_output.dot(previous_weight))\n",
    "        return self.output_matrix\n",
    "    \n",
    "    # back_propagate the error\n",
    "    def get_new_weight(self, output_error, previous_output, previous_weight):\n",
    "        delta = output_error * d_sigmoid(self.output_matrix)\n",
    "#         print(delta.shape, previous_weight.shape)\n",
    "        self.input_error = delta.dot(previous_weight.transpose())\n",
    "        return previous_weight - previous_output.transpose().dot(delta)\n",
    "         \n",
    "        \n",
    "        \n",
    "class InputLayer(Layer):\n",
    "    def __init__(self, input_dimension):\n",
    "        Layer.__init__(self, input_dimension, lambda x: x)\n",
    "    \n",
    "    \n",
    "class NeuralNetwork():\n",
    "\n",
    "    def __init__(self, layers):\n",
    "        self.weights = []\n",
    "        self.layers = layers\n",
    "        # init weights based on layers' dimension (inter connecting weights)\n",
    "        for l in range(1, len(self.layers)):\n",
    "            print('W'+str(l-1), self.layers[l-1].output_dimension, 'x',self.layers[l].output_dimension)\n",
    "            self.weights.append(np.random.rand(self.layers[l-1].output_dimension, self.layers[l].output_dimension))\n",
    "            \n",
    "    def feed_forward(self, input_matrix):\n",
    "        self.layers[0].output_matrix = input_matrix\n",
    "        # skip the input layer, start from the first hidden layer\n",
    "        for l in range(1, len(self.layers)):\n",
    "            self.layers[l].feed_forward(self.layers[l-1].output_matrix, self.weights[l-1])\n",
    "        return self.layers[-1].output_matrix\n",
    "    \n",
    "    def back_propagate(self, y, t):\n",
    "        # 4 * 1\n",
    "        output_error = y - t\n",
    "        self.weights[-1] = self.layers[-1].get_new_weight(output_error, self.layers[-2].output_matrix, self.weights[-1])\n",
    "    \n",
    "        # hidden layers' delta\n",
    "        for l in range(len(self.layers)-2, 0, -1):\n",
    "            self.weights[l-1] = self.layers[l].get_new_weight(self.layers[l+1].input_error, \n",
    "                                                               self.layers[l-1].output_matrix, self.weights[l-1])\n",
    "    \n",
    "\n",
    "    def train(self, training_samples, training_labels, num_epoch):\n",
    "        for i in range(num_epoch):\n",
    "            y = self.feed_forward(training_samples)\n",
    "            t = training_labels\n",
    "            self.back_propagate(y, t)\n",
    "            if i % 1000 == 0:\n",
    "                print(((y - t) ** 2).mean())\n",
    "                \n",
    "        \n",
    "    def predict(self, input_matrix):\n",
    "        y = self.feed_forward(input_matrix)\n",
    "        return y\n",
    "              "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 407,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "il = InputLayer(2)\n",
    "h1 = Layer(4, sigmoid)\n",
    "h2 = Layer(4, sigmoid)\n",
    "ol = Layer(1, sigmoid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W0 2 x 4\n",
      "W1 4 x 4\n",
      "W2 4 x 1\n"
     ]
    }
   ],
   "source": [
    "ann = NeuralNetwork([\n",
    "        il, h1, h2, ol  \n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 409,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.83194717],\n",
       "       [ 0.84688709],\n",
       "       [ 0.84732564],\n",
       "       [ 0.85644279]])"
      ]
     },
     "execution_count": 409,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# feeding one sample\n",
    "ann.feed_forward(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.368095842577\n",
      "0.00801063624191\n",
      "0.0004579154445\n",
      "0.000218962186552\n",
      "0.000141510148473\n",
      "0.000103753391784\n",
      "8.15548102088e-05\n",
      "6.69993371867e-05\n",
      "5.67461786334e-05\n",
      "4.91473538819e-05\n"
     ]
    }
   ],
   "source": [
    "ann.train(x, t, 10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00744752])"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ann.predict(x[0]) # should be F: 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.99330799])"
      ]
     },
     "execution_count": 412,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ann.predict(x[1]) # should be T: 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 413,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.99317682])"
      ]
     },
     "execution_count": 413,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ann.predict(x[2]) # should be T: 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00513696])"
      ]
     },
     "execution_count": 414,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ann.predict(x[3]) # should be F: 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}