Working through the elements of creating a confusion matrix in theano

ConfusionMatrixTheano.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion Matrix Theano\n",
    "===="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Import relevant libraries\n",
    "import theano\n",
    "import numpy as np\n",
    "from theano import tensor as t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Not Own Code\n",
    "Taken from:\n",
    "<https://groups.google.com/forum/#!msg/theano-users/MMuXRzYPePA/8h84l0z6CQAJ>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 2.  0.  0.]\n",
      " [ 0.  0.  1.]\n",
      " [ 1.  0.  2.]]\n"
     ]
    }
   ],
   "source": [
    "x = t.vector('x')\n",
    "classes = t.scalar('n_classes')\n",
    "\n",
    "onehot = t.eq(x.dimshuffle(0,'x'),t.arange(classes).dimshuffle('x',0)).astype('int64')\n",
    "oneHot = theano.function([x,classes],onehot)\n",
    "\n",
    "y = t.matrix('y')\n",
    "y_pred = t.matrix('y_pred')\n",
    "\n",
    "confMat = t.dot(y.T,y_pred)\n",
    "confusionMatrix = theano.function(inputs=[y,y_pred],outputs=confMat)\n",
    "\n",
    "def confusion_matrix(x,y,n_class):\n",
    "    return confusionMatrix(oneHot(x,n_class),oneHot(y,n_class))\n",
    "y_true =[2, 0, 2, 2, 0, 1]\n",
    "y_out = [0, 0, 2, 2, 0, 2]\n",
    "\n",
    "print confusion_matrix(y_true,y_out,3)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## Re-implementing for understanding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make boolean Matrix of Predictions and True values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimsh1 for array a and b\n",
      "[[ 0.]\n",
      " [ 1.]\n",
      " [ 2.]\n",
      " [ 1.]\n",
      " [ 0.]] \n",
      "[[ 0.]\n",
      " [ 0.]\n",
      " [ 2.]\n",
      " [ 1.]\n",
      " [ 2.]]\n",
      "Dimsh2 for scalar [0 1 2]\n",
      "Type Dimsh2 <type 'numpy.ndarray'>\n",
      "Result comparing the column and row vectors\n",
      "one_hot_a\n",
      "[[1 0 0]\n",
      " [0 1 0]\n",
      " [0 0 1]\n",
      " [0 1 0]\n",
      " [1 0 0]]\n",
      "one_hot_b\n",
      "[[1 0 0]\n",
      " [1 0 0]\n",
      " [0 0 1]\n",
      " [0 1 0]\n",
      " [0 0 1]]\n"
     ]
    }
   ],
   "source": [
    "# Define a numpy vector of 'predictions'\n",
    "a = np.array([0, 1, 2, 1, 0])\n",
    "# Initialise a test vector of 'true' values\n",
    "b = np.array([0, 0, 2, 1, 2])\n",
    "# Define a variable to test num class\n",
    "n_classes = 3\n",
    "\n",
    "# Define theano tensor variables\n",
    "one_hot_input = t.vector('input')\n",
    "classes = t.scalar('num_classes')\n",
    "\n",
    "# Transpose row vector into column vector\n",
    "dimsh1 = one_hot_input.dimshuffle(0, 'x')\n",
    "# Make a row vector out of a range from 0 to n_classes\n",
    "# The .astype('int64') is a cast so minus numbers can be checked apparently\n",
    "dimsh2 = t.arange(classes).astype('int64')\n",
    "\n",
    "# Compile the above into theano functions\n",
    "f_dimsh1 = theano.function([one_hot_input], dimsh1)\n",
    "f_dimsh2 = theano.function([classes], dimsh2)\n",
    "# See what the output is \n",
    "print 'Dimsh1 for array a and b\\n', f_dimsh1(a), '\\n',f_dimsh1(b)\n",
    "print 'Dimsh2 for scalar', f_dimsh2(n_classes)\n",
    "print 'Type Dimsh2', type(f_dimsh2(n_classes))\n",
    "\n",
    "# Compile above into equals operation\n",
    "one_hot_eq_op = t.eq(dimsh1, dimsh2)\n",
    "# Compile a theano function to test equal op\n",
    "one_hot_eq = theano.function([one_hot_input, classes], one_hot_eq_op) \n",
    "\n",
    "print 'Result comparing the column and row vectors'\n",
    "one_hot_a = one_hot_eq(a, n_classes)\n",
    "print 'one_hot_a'\n",
    "print one_hot_a\n",
    "one_hot_b = one_hot_eq(b, n_classes)\n",
    "print 'one_hot_b'\n",
    "print one_hot_b"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So what the above does is take a vector of predictions, change them into a column vector and then creates a row vector of the different classes and compares each entry in the column vector to all the entries in the row vector.\n",
    "\n",
    "For each index(row) in the column vector, the comparison between this scalar and the row vector of possible classes geneterates a row vector where 1 represents that the class at the same index in the row vector was predicted for this sample. i.e. a one hot encoded matrix. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get a dot product of the one-hot matrices for true positives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.  0.  1.]\n",
      " [ 1.  1.  0.]\n",
      " [ 0.  0.  1.]]\n"
     ]
    }
   ],
   "source": [
    "# Define variables to hold the (one-hot matrices of) predictions  \n",
    "a_pred = t.matrix('y_pred')\n",
    "b_true = t.matrix('y_true')\n",
    "\n",
    "# Count the number of true positives and false negatives for each class\n",
    "# By getting the dot product of one hot predictions and \n",
    "conf_matrix = t.dot(a_pred.T, b_true)\n",
    "\n",
    "# Compile a function to test the confusion matrix \n",
    "conf_test = theano.function([a_pred, b_true], conf_matrix)\n",
    "\n",
    "print conf_test(one_hot_a, one_hot_b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "raw_mimetype": "text/markdown"
   },
   "source": [
    "This is right .. I checked it by hand ! :) \n",
    "![Bit of the aul fashioned hand-maths](http://localhost:8888/notebooks/conf_check.jpg)\n",
    "\n",
    "It is worth noting here that the order these parameters are passed to the dot product most definitely matter. In this: dot(y_pred.T, y_true).\n",
    "\n",
    "The above configuration will result in the actual class across the top of the matrix and the predicted class along the left of the matrix. i.e. "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "raw_mimetype": "text/html"
   },
   "source": [
    "<style>\n",
    "table, th, td {\n",
    "    border: 1px solid black; \n",
    "}\n",
    "</style>\n",
    "</head>\n",
    "<body>\n",
    "\n",
    "<table align=\"center\">\n",
    "  <tr>\n",
    "    <th colspan=\"2\" rowspan=\"2\"></th>\n",
    "    <th colspan=\"3\">Actual</th>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>c1</td>\n",
    "    <td>c2</td>\n",
    "    <td>c3</td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td rowspan=\"3\">Predicted</td>\n",
    "    <td>c1</td>\n",
    "    <td>tp</td>\n",
    "    <td></td>\n",
    "    <td></td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>c2</td>\n",
    "    <td></td>\n",
    "    <td>tp</td>\n",
    "    <td></td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>c3</td>\n",
    "    <td></td>\n",
    "    <td></td>\n",
    "    <td>tp</td>\n",
    "  </tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you were to code it as t.dot(one_hot_b.T, one_hot_a) i.e. dot(y_true.T, y_pred) it would give the predicted class along the top and the actual class along the side. i.e. "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "raw_mimetype": "text/html"
   },
   "source": [
    "<style>\n",
    "table, th, td {\n",
    "    border: 1px solid black; padding: 5px 10px 5px 10px; \n",
    "}\n",
    "</style>\n",
    "</head>\n",
    "<body>\n",
    "\n",
    "<table align=\"center\">\n",
    "  <tr>\n",
    "    <th colspan=\"2\" rowspan=\"2\"></th>\n",
    "    <th colspan=\"3\">Predicted</th>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>c1</td>\n",
    "    <td>c2</td>\n",
    "    <td>c3</td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td rowspan=\"3\">Actual</td>\n",
    "    <td>c1</td>\n",
    "    <td>tp</td>\n",
    "    <td></td>\n",
    "    <td></td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>c2</td>\n",
    "    <td></td>\n",
    "    <td>tp</td>\n",
    "    <td></td>\n",
    "  </tr>\n",
    "  <tr>\n",
    "    <td>c3</td>\n",
    "    <td></td>\n",
    "    <td></td>\n",
    "    <td>tp</td>\n",
    "  </tr>\n",
    "</table>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now all that's left to do is compile everything into the one function so we can input the original numpy arrays and the class numbers so it can write the confusion matrix from that. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 0 1]\n",
      " [1 1 0]\n",
      " [0 0 1]]\n"
     ]
    }
   ],
   "source": [
    "# Putting it all together\n",
    "\n",
    "# Define theano tensor variables\n",
    "predictions = t.vector('input')\n",
    "classes = t.scalar('num_classes')\n",
    "actual = t.vector('true')\n",
    "\n",
    "# Get dot product \n",
    "final_conf = t.dot( \n",
    "                    t.eq(# One hot encode predictions and transpose\n",
    "                        predictions.dimshuffle(0, 'x'), t.arange(classes).astype('int64')\n",
    "                    ).T,\n",
    "                    t.eq(# One hot encode the actual values \n",
    "                        actual.dimshuffle(0,'x'), t.arange(classes).astype('int64')\n",
    "                    )\n",
    "            )\n",
    "# Compile it into a function \n",
    "final_conf_mat_f = theano.function([predictions, actual, classes], final_conf)\n",
    "# Test it\n",
    "print final_conf_mat_f(a, b, n_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "and there you go!"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}