Example implementation of a Convolutional Neural Net for the MNIST data

mnist_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import numpy as np\n",
    "from keras.preprocessing import image\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Lambda, Flatten\n",
    "from keras.layers import Convolution2D, MaxPooling2D, ZeroPadding2D, BatchNormalization\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def basic_model():\n",
    "    return Sequential([Lambda(normalize, input_shape=(28,28,1))])\n",
    "\n",
    "def add_output_layer(model):\n",
    "    model.add(Dense(10, activation=\"softmax\"))\n",
    "\n",
    "def convolutional_model(conv_blocks=2, hidden_dense_activation=\"relu\", pool_size=2, num_dense_neurons=512, starting_filter_size=32, batchnorm=False, zero_padding=True, dropout=False):\n",
    "    model = basic_model()\n",
    "    for block in range(conv_blocks):\n",
    "        num_filters = starting_filter_size * (block + 1)\n",
    "        if zero_padding:\n",
    "            model.add(ZeroPadding2D())\n",
    "        model.add(Convolution2D(num_filters, 3, 3, activation=\"relu\"))\n",
    "        if batchnorm:\n",
    "            model.add(BatchNormalization(axis=1))\n",
    "        model.add(Convolution2D(num_filters, 3, 3, activation=\"relu\"))\n",
    "        model.add(MaxPooling2D(pool_size=pool_size))\n",
    "        if batchnorm and block is not conv_blocks - 1:\n",
    "            model.add(BatchNormalization(axis=1))\n",
    "    model.add(Flatten())\n",
    "    if batchnorm:\n",
    "        model.add(BatchNormalization(axis=1))\n",
    "    model.add(Dense(num_dense_neurons, activation=hidden_dense_activation))\n",
    "    if batchnorm:\n",
    "        model.add(BatchNormalization(axis=1))\n",
    "    if dropout:\n",
    "        model.add(Dropout(dropout))\n",
    "    add_output_layer(model)\n",
    "    return model\n",
    "\n",
    "def normalize(x):\n",
    "    x_mean = x_train.mean().astype(np.float32)\n",
    "    x_std = x_train.std().astype(np.float32)\n",
    "    return (x - x_mean) / x_std\n",
    "\n",
    "def onehot(y):\n",
    "    return keras.utils.np_utils.to_categorical(y)\n",
    "\n",
    "def get_data(sample=False):\n",
    "    if sample:\n",
    "        x,y = x_train_sample, y_train_sample\n",
    "        val_x, val_y = x_test_sample, y_test_sample\n",
    "    else:\n",
    "        x,y = x_train, y_train\n",
    "        val_x, val_y = x_test, y_test\n",
    "    return x, y, val_x, val_y\n",
    "\n",
    "def compile_model(model):\n",
    "    # Categorical cross entropy is used when you have more than 2 classes to compare against.\n",
    "    # We have 10 classes for MNIST (digits 0 - 9), so thus we use it here.\n",
    "    return model.compile(optimizer=Adam(), loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "\n",
    "def fit(model, learning_rate=None, nb_epoch=1, sample=False):\n",
    "    model.optimizer.lr = learning_rate if learning_rate else model.optimizer.lr\n",
    "    x, y, val_x, val_y = get_data(sample=sample)\n",
    "    model.fit(x, y, batch_size=64, nb_epoch=nb_epoch, validation_data=(val_x, val_y))\n",
    "    \n",
    "def multi_fit(model, reset=True, augmentation=False, sample=False, runs=1):\n",
    "    for run in range(runs):\n",
    "        for learning_rate in [0.001, 0.01, 0.1]:\n",
    "            print(\"Fitting with learning rate of: \", learning_rate)\n",
    "            fit(model, learning_rate=learning_rate, sample=sample)\n",
    "            if reset:\n",
    "                # Resetting is nice here for comparing differences in learning rate, without the compounding factor\n",
    "                # of model state across epochs\n",
    "                model.reset_states()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "x_test = np.expand_dims(x_test, -1)\n",
    "x_train = np.expand_dims(x_train, -1)\n",
    "y_train = onehot(y_train)\n",
    "y_test = onehot(y_test)\n",
    "\n",
    "sample_size = 2000\n",
    "x_test_sample = x_test[:sample_size]\n",
    "x_train_sample = x_train[:sample_size]\n",
    "y_train_sample = y_train[:sample_size]\n",
    "y_test_sample = y_test[:sample_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2000, 28, 28, 1)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train_sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = convolutional_model(conv_blocks=1, starting_filter_size=3, pool_size=(4,4), hidden_dense_activation=\"relu\", num_dense_neurons=8)\n",
    "compile_model(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 2.2972 - acc: 0.1235 - val_loss: 2.2729 - val_acc: 0.1620\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 2.2223 - acc: 0.2165 - val_loss: 2.1677 - val_acc: 0.2730\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 2.0123 - acc: 0.3275 - val_loss: 1.9015 - val_acc: 0.3410\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 1.6436 - acc: 0.4505 - val_loss: 1.6410 - val_acc: 0.4320\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 1.3507 - acc: 0.5545 - val_loss: 1.4444 - val_acc: 0.5060\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 1.1670 - acc: 0.6170 - val_loss: 1.3199 - val_acc: 0.5555\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 1.0454 - acc: 0.6585 - val_loss: 1.2576 - val_acc: 0.5995\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.9735 - acc: 0.6910 - val_loss: 1.1848 - val_acc: 0.6220\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.9027 - acc: 0.7240 - val_loss: 1.1376 - val_acc: 0.6390\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 2s - loss: 0.8446 - acc: 0.7360 - val_loss: 1.0774 - val_acc: 0.6685\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.7900 - acc: 0.7665 - val_loss: 1.0369 - val_acc: 0.6880\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.7474 - acc: 0.7805 - val_loss: 0.9916 - val_acc: 0.7000\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.6996 - acc: 0.7975 - val_loss: 0.9509 - val_acc: 0.7100\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.6600 - acc: 0.8145 - val_loss: 0.9304 - val_acc: 0.7125\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.6306 - acc: 0.8185 - val_loss: 0.8979 - val_acc: 0.7325\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.5928 - acc: 0.8360 - val_loss: 0.8700 - val_acc: 0.7365\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.5722 - acc: 0.8370 - val_loss: 0.8575 - val_acc: 0.7430\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.5524 - acc: 0.8475 - val_loss: 0.8173 - val_acc: 0.7510\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.5298 - acc: 0.8480 - val_loss: 0.8020 - val_acc: 0.7555\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.5151 - acc: 0.8550 - val_loss: 0.7851 - val_acc: 0.7550\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.4988 - acc: 0.8575 - val_loss: 0.7768 - val_acc: 0.7675\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.4804 - acc: 0.8620 - val_loss: 0.7502 - val_acc: 0.7615\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.4604 - acc: 0.8625 - val_loss: 0.7323 - val_acc: 0.7800\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.4565 - acc: 0.8660 - val_loss: 0.7260 - val_acc: 0.7800\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.4365 - acc: 0.8750 - val_loss: 0.7172 - val_acc: 0.7840\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.4284 - acc: 0.8750 - val_loss: 0.7003 - val_acc: 0.7860\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.4164 - acc: 0.8755 - val_loss: 0.6893 - val_acc: 0.7935\n",
      "Fitting with learning rate of:  0.001\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 2s - loss: 0.4164 - acc: 0.8780 - val_loss: 0.6841 - val_acc: 0.7890\n",
      "Fitting with learning rate of:  0.01\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 2s - loss: 0.4017 - acc: 0.8790 - val_loss: 0.6755 - val_acc: 0.7910\n",
      "Fitting with learning rate of:  0.1\n",
      "Train on 2000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "2000/2000 [==============================] - 1s - loss: 0.3913 - acc: 0.8855 - val_loss: 0.6702 - val_acc: 0.7950\n"
     ]
    }
   ],
   "source": [
    "multi_fit(model, sample=True, runs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}