95-865 CNN demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# This demo draws heavily from the handwritten digit example in\n",
    "# Chapter 2 of Francois Chollet's \"Deep Learning with Python\" book.\n",
    "# I've added a simpler single-layer example first before moving to\n",
    "# the 2-layer example. -George Chen (CMU Fall 2017)\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from keras.datasets import mnist\n",
    "from keras import models\n",
    "from keras import layers\n",
    "\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "\n",
    "flattened_train_images = train_images.reshape(len(train_images), -1)  # flattens out each training image\n",
    "flattened_train_images = flattened_train_images.astype(np.float32) / 255  # rescale to be between 0 and 1\n",
    "flattened_test_images = test_images.reshape(len(test_images), -1)  # flattens out each test image\n",
    "flattened_test_images = flattened_test_images.astype(np.float32) / 255  # rescale to be between 0 and 1\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "train_labels_categorical = to_categorical(train_labels)\n",
    "test_labels_categorical = to_categorical(test_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 1., 0., 0., 0., 0.], dtype=float32)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels_categorical[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 10)                7850      \n",
      "=================================================================\n",
      "Total params: 7,850\n",
      "Trainable params: 7,850\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# extremely shallow single-layer model\n",
    "single_layer_model = models.Sequential()  # this is Keras's way of specifying a model that is a single sequence of layers\n",
    "single_layer_model.add(layers.Dense(10, activation='softmax', input_shape=(784,)))\n",
    "single_layer_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "single_layer_model.compile(optimizer='rmsprop',\n",
    "                           loss='categorical_crossentropy',\n",
    "                           metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/5\n",
      "48000/48000 [==============================] - 3s 57us/step - loss: 0.6538 - acc: 0.8421 - val_loss: 0.3550 - val_acc: 0.9048\n",
      "Epoch 2/5\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.3498 - acc: 0.9035 - val_loss: 0.3073 - val_acc: 0.9142\n",
      "Epoch 3/5\n",
      "48000/48000 [==============================] - 2s 34us/step - loss: 0.3153 - acc: 0.9120 - val_loss: 0.2907 - val_acc: 0.9180\n",
      "Epoch 4/5\n",
      "48000/48000 [==============================] - 2s 34us/step - loss: 0.2994 - acc: 0.9163 - val_loss: 0.2821 - val_acc: 0.9207\n",
      "Epoch 5/5\n",
      "48000/48000 [==============================] - 2s 33us/step - loss: 0.2897 - acc: 0.9189 - val_loss: 0.2750 - val_acc: 0.9246\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb3f8ee898>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "single_layer_model.fit(flattened_train_images,\n",
    "                       train_labels_categorical,\n",
    "                       validation_split=0.2,\n",
    "                       epochs=5,\n",
    "                       batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_2 (Dense)              (None, 512)               401920    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 407,050\n",
      "Trainable params: 407,050\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# two-layer model\n",
    "two_layer_model = models.Sequential()  # this is Keras's way of specifying a model that is a single sequence of layers\n",
    "two_layer_model.add(layers.Dense(512, activation='relu', input_shape=(784,)))\n",
    "two_layer_model.add(layers.Dense(10, activation='softmax'))\n",
    "two_layer_model.compile(optimizer='rmsprop',\n",
    "                        loss='categorical_crossentropy',\n",
    "                        metrics=['accuracy'])\n",
    "two_layer_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/5\n",
      "48000/48000 [==============================] - 10s 200us/step - loss: 0.2864 - acc: 0.9160 - val_loss: 0.1539 - val_acc: 0.9539\n",
      "Epoch 2/5\n",
      "48000/48000 [==============================] - 8s 158us/step - loss: 0.1179 - acc: 0.9653 - val_loss: 0.1049 - val_acc: 0.9688\n",
      "Epoch 3/5\n",
      "48000/48000 [==============================] - 6s 132us/step - loss: 0.0769 - acc: 0.9775 - val_loss: 0.0952 - val_acc: 0.9713\n",
      "Epoch 4/5\n",
      "48000/48000 [==============================] - 6s 124us/step - loss: 0.0555 - acc: 0.9833 - val_loss: 0.0824 - val_acc: 0.9753\n",
      "Epoch 5/5\n",
      "48000/48000 [==============================] - 6s 123us/step - loss: 0.0407 - acc: 0.9880 - val_loss: 0.0828 - val_acc: 0.9768\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x10ca91b00>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "two_layer_model.fit(flattened_train_images,\n",
    "                    train_labels_categorical,\n",
    "                    validation_split=0.2,\n",
    "                    epochs=5,\n",
    "                    batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reshape images to have an additional dimension for color (even though there's no color)\n",
    "scaled_train_images = train_images.reshape(len(train_images), train_images.shape[1], train_images.shape[2], -1)\n",
    "scaled_test_images = test_images.reshape(len(test_images), test_images.shape[1], test_images.shape[2], -1)\n",
    "\n",
    "# rescale to be between 0 and 1\n",
    "scaled_train_images = scaled_train_images.astype(np.float32) / 255\n",
    "scaled_test_images = scaled_test_images.astype(np.float32) / 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "print(scaled_train_images.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_1 (Conv2D)            (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 13, 13, 32)        0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 5408)              0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                54090     \n",
      "=================================================================\n",
      "Total params: 54,410\n",
      "Trainable params: 54,410\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "simple_convnet_model = models.Sequential()\n",
    "simple_convnet_model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
    "simple_convnet_model.add(layers.MaxPooling2D((2, 2)))\n",
    "simple_convnet_model.add(layers.Flatten())\n",
    "simple_convnet_model.add(layers.Dense(10, activation='softmax'))\n",
    "simple_convnet_model.summary()\n",
    "\n",
    "simple_convnet_model.compile(optimizer='rmsprop',\n",
    "                             loss='categorical_crossentropy',\n",
    "                             metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/5\n",
      "48000/48000 [==============================] - 19s 397us/step - loss: 0.3819 - acc: 0.8976 - val_loss: 0.1872 - val_acc: 0.9467\n",
      "Epoch 2/5\n",
      "48000/48000 [==============================] - 18s 378us/step - loss: 0.1501 - acc: 0.9570 - val_loss: 0.1148 - val_acc: 0.9687\n",
      "Epoch 3/5\n",
      "48000/48000 [==============================] - 18s 377us/step - loss: 0.0995 - acc: 0.9715 - val_loss: 0.0924 - val_acc: 0.9745\n",
      "Epoch 4/5\n",
      "48000/48000 [==============================] - 18s 383us/step - loss: 0.0782 - acc: 0.9777 - val_loss: 0.0774 - val_acc: 0.9785\n",
      "Epoch 5/5\n",
      "48000/48000 [==============================] - 18s 382us/step - loss: 0.0665 - acc: 0.9811 - val_loss: 0.0769 - val_acc: 0.9769\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb561818d0>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_convnet_model.fit(scaled_train_images,\n",
    "                         train_labels_categorical,\n",
    "                         validation_split=0.2,\n",
    "                         epochs=5,\n",
    "                         batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_2 (Conv2D)            (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d_2 (MaxPooling2 (None, 13, 13, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 11, 11, 32)        9248      \n",
      "_________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2 (None, 5, 5, 32)          0         \n",
      "_________________________________________________________________\n",
      "flatten_2 (Flatten)          (None, 800)               0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                8010      \n",
      "=================================================================\n",
      "Total params: 17,578\n",
      "Trainable params: 17,578\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "deeper_convnet_model = models.Sequential()\n",
    "deeper_convnet_model.add(layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)))\n",
    "deeper_convnet_model.add(layers.MaxPooling2D((2, 2)))\n",
    "deeper_convnet_model.add(layers.Conv2D(32, (3, 3), activation='relu'))\n",
    "deeper_convnet_model.add(layers.MaxPooling2D((2, 2)))\n",
    "deeper_convnet_model.add(layers.Flatten())\n",
    "deeper_convnet_model.add(layers.Dense(10, activation='softmax'))\n",
    "deeper_convnet_model.summary()\n",
    "\n",
    "deeper_convnet_model.compile(optimizer='rmsprop',\n",
    "                             loss='categorical_crossentropy',\n",
    "                             metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/5\n",
      "48000/48000 [==============================] - 25s 515us/step - loss: 0.3525 - acc: 0.8991 - val_loss: 0.1162 - val_acc: 0.9667\n",
      "Epoch 2/5\n",
      "48000/48000 [==============================] - 24s 496us/step - loss: 0.1042 - acc: 0.9689 - val_loss: 0.0800 - val_acc: 0.9763\n",
      "Epoch 3/5\n",
      "48000/48000 [==============================] - 24s 494us/step - loss: 0.0750 - acc: 0.9777 - val_loss: 0.0722 - val_acc: 0.9797\n",
      "Epoch 4/5\n",
      "48000/48000 [==============================] - 24s 492us/step - loss: 0.0593 - acc: 0.9818 - val_loss: 0.0631 - val_acc: 0.9821\n",
      "Epoch 5/5\n",
      "48000/48000 [==============================] - 24s 500us/step - loss: 0.0509 - acc: 0.9844 - val_loss: 0.0519 - val_acc: 0.9855\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb6e0d6b70>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deeper_convnet_model.fit(scaled_train_images,\n",
    "                         train_labels_categorical,\n",
    "                         validation_split=0.2,\n",
    "                         epochs=5,\n",
    "                         batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finally evaluate on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 26us/step\n",
      "Test accuracy: 0.9227\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = single_layer_model.evaluate(flattened_test_images, test_labels_categorical)\n",
    "print('Test accuracy:', test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 47us/step\n",
      "Test accuracy: 0.9778\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = two_layer_model.evaluate(flattened_test_images, test_labels_categorical)\n",
    "print('Test accuracy:', test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 175us/step\n",
      "Test accuracy: 0.9756\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = simple_convnet_model.evaluate(scaled_test_images, test_labels_categorical)\n",
    "print('Test accuracy:', test_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 227us/step\n",
      "Test accuracy: 0.9857\n"
     ]
    }
   ],
   "source": [
    "test_loss, test_acc = deeper_convnet_model.evaluate(scaled_test_images, test_labels_categorical)\n",
    "print('Test accuracy:', test_acc)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}