gistfile1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multilayer Perceptron with Backpropagation\n",
    "This is an implemention of and mlp with backpropagation as the chosen learning algorithm. It currently only performs a single cycle of forward pass, backward pass and weights update."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "## ======== FUNCTION DEFINITIONS ======== ##\n",
    "\n",
    "# initialises weights of the network to small random values\n",
    "def init_weights(n_inputs, n_hiddens, n_outputs, rand):\n",
    "    weights = []\n",
    "    hidden_layer = random.uniform(-rand, rand, [n_hiddens, n_inputs])\n",
    "    weights.append(hidden_layer)\n",
    "    output_layer = random.uniform(-rand, rand, [n_outputs,n_hiddens])\n",
    "    weights.append(output_layer)\n",
    "    return weights\n",
    "\n",
    "# caluclates activation for a node by calculating the sum of the products of corresponding inputs and weights\n",
    "def activate(weights, inputs):\n",
    "    activation = sum(weights[i]*inputs[i] for i in range(len(weights)))\n",
    "    return activation\n",
    "\n",
    "# sigmoid activation function which will be the default\n",
    "def sigmoid(x):\n",
    "    return 1.0/(1.0+x+np.exp(-x))\n",
    "\n",
    "# alternative being the tanh function\n",
    "def tanh(x):\n",
    "    return (np.exp(2*x)-1)/(np.exp(2*x)+1)\n",
    "\n",
    "# calculates outputs of the network according to specific weights and inputs\n",
    "def forward_pass(weights, x):\n",
    "    outputs = []\n",
    "    inputs = x\n",
    "    for layer in weights:\n",
    "        layer_output = []\n",
    "        for neuron in layer:\n",
    "            activation = activate(neuron, inputs)\n",
    "            layer_output.append(sigmoid(activation))\n",
    "            #layer_output.append(tanh(activation))\n",
    "        inputs = layer_output\n",
    "        outputs.append(layer_output)\n",
    "    return outputs\n",
    "\n",
    "def sigmoid_derivative(x):\n",
    "    return (x * (1.0 - x))\n",
    "\n",
    "def tanh_derivative(x):\n",
    "    return (1.0 - (x**2))\n",
    "\n",
    "# calculates beta values for nodes in the output and hidden layers\n",
    "def backward_pass(weights, outputs, desired):\n",
    "    betas = [[],[]]\n",
    "    # betas for output neurons\n",
    "    for i in range(len(outputs[1])):\n",
    "        error = desired[i]-outputs[1][i]\n",
    "        beta = sigmoid_derivative(outputs[1][i])*error \n",
    "        betas[1].append(beta)\n",
    "    # betas for hidden neurons\n",
    "    for i in range(len(outputs[0])):\n",
    "        hidden_error = sum(weights[1][j][i]*betas[1][j] for j in range(len(outputs[1])))\n",
    "        hidden_beta = sigmoid_derivative(outputs[0][i])*hidden_error\n",
    "        betas[0].append(hidden_beta)\n",
    "    return betas\n",
    "\n",
    "# updates all weights in network according to learning rate and beta values\n",
    "def update_weights(_weights, betas, learn_rate):\n",
    "    weights = _weights\n",
    "    for i in range(len(weights)):\n",
    "        for j in range(len(layer)):\n",
    "            delta = weights[i][j] * betas[i][j]\n",
    "            weights[i][j] += delta \n",
    "    return weights\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "weights:\n",
      "[[ 0.4955273   0.35401722]\n",
      " [-0.03074457 -0.32982116]\n",
      " [ 0.29108339 -0.46446613]]\n",
      "[[ 0.10413163  0.44872311  0.23057438]\n",
      " [-0.17983462 -0.30787025  0.2373545 ]]\n",
      "outputs:\n",
      "[0.47510974533779682, 0.49988065465979553, 0.4905479421620727]\n",
      "[0.48401405245752571, 0.49803969165036127]\n",
      "betas:\n",
      "[-0.0087668526932405816, -0.02321890823523217, 0.00047817585800280483]\n",
      "[-0.12087982307215453, 0.12548814814990658]\n",
      "updated weights:\n",
      "[[ 0.49118308  0.3509136 ]\n",
      " [-0.03003072 -0.32216307]\n",
      " [ 0.29108339 -0.46446613]]\n",
      "[[ 0.09154422  0.39448154  0.20270259]\n",
      " [-0.20240174 -0.34650432  0.26713968]]\n"
     ]
    }
   ],
   "source": [
    "# x: input data / y: desired output\n",
    "x = [1, 0]\n",
    "y = [0, 1]\n",
    "# learn rate controls the rate at which weights are updated\n",
    "learn_rate = 1.0\n",
    "\n",
    "# weights initialised according to size of x and y and number of hidden nodes\n",
    "weights = init_weights(len(x), 3, len(set(y)), 0.5)\n",
    "print('weights:')\n",
    "for layer in weights:\n",
    "    print(layer)\n",
    "\n",
    "# outputs of hiddens and output nodes calculated\n",
    "outputs = forward_pass(weights, x)\n",
    "print('outputs:')\n",
    "for layer in outputs:\n",
    "    print(layer) \n",
    "    \n",
    "# betas caulculated for each nodes by backpropagating error signal\n",
    "betas = backward_pass(weights, outputs, y)\n",
    "print('betas:')\n",
    "for layer in betas:\n",
    "    print(layer)\n",
    "    \n",
    "# weights updated according to nodes beta values and learning rate\n",
    "weights = update_weights(weights, betas, learn_rate)\n",
    "print('updated weights:')\n",
    "for layer in weights:\n",
    "    print(layer)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}