sujee/cnn-mnist-1-train.ipynb

## cnn-mnist-1-train.ipynb
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab : Using Convolutional Neural Networks (CNN) to identify digits\n",
    "\n",
    "In this lab, we are going to setup a CNN to classify MNIST"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## About MNIST Data\n",
    "\n",
    "MNIST is a widely used dataset of hand written digits.\n",
    "\n",
    "<img src=\"https://www.researchgate.net/profile/Steven_Young11/publication/306056875/figure/fig1/AS:393921575309346@1470929630835/Example-images-from-the-MNIST-dataset.png\" />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running in Google COLAB :  False\n"
     ]
    }
   ],
   "source": [
    "## Determine if we are running on google colab\n",
    "\n",
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "import time\n",
    "import os,sys\n",
    "\n",
    "try:\n",
    "    import google.colab\n",
    "    RUNNING_IN_COLAB = True\n",
    "except:\n",
    "    RUNNING_IN_COLAB = False\n",
    "\n",
    "print (\"Running in Google COLAB : \", RUNNING_IN_COLAB)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass\n",
    "\n",
    "## disable info logs from TF\n",
    "#   Level | Level for Humans | Level Description                  \n",
    "#  -------|------------------|------------------------------------ \n",
    "#   0     | DEBUG            | [Default] Print all messages       \n",
    "#   1     | INFO             | Filter out INFO messages           \n",
    "#   2     | WARNING          | Filter out INFO & WARNING messages \n",
    "#   3     | ERROR            | Filter out all messages \n",
    "\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'  # or any {'0', '1', '2'}\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.get_logger().setLevel('WARN')\n",
    "from tensorflow import keras\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device mapping:\n",
      "/job:localhost/replica:0/task:0/device:XLA_CPU:0 -> device: XLA_CPU device\n",
      "/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: GeForce RTX 2070, pci bus id: 0000:01:00.0, compute capability: 7.5\n",
      "/job:localhost/replica:0/task:0/device:XLA_GPU:0 -> device: XLA_GPU device\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# for running in GPU\n",
    "\n",
    "from tensorflow.compat.v1.keras.backend import set_session\n",
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.allow_growth = True  # dynamically grow the memory used on the GPU\n",
    "config.log_device_placement = True  # to log device placement (on which device the operation ran)\n",
    "sess = tf.compat.v1.Session(config=config)\n",
    "set_session(sess)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Download Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_images shape :  (60000, 28, 28)\n",
      "train_labels shape :  (60000,)\n",
      "test_images shape :  (10000, 28, 28)\n",
      "test_labels shape :  (10000,)\n"
     ]
    }
   ],
   "source": [
    "mnist = tf.keras.datasets.mnist\n",
    "\n",
    "(train_images, train_labels), (test_images, test_labels) = tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "# backup just in case\n",
    "(train_images2, train_labels2), (test_images2, test_labels2) = tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "print(\"train_images shape : \", train_images.shape)\n",
    "print(\"train_labels shape : \", train_labels.shape)\n",
    "print(\"test_images shape : \", test_images.shape)\n",
    "print(\"test_labels shape : \", test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2: Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying train index =  58888 , label =  8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f07240f0710>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAPLUlEQVR4nO3df5DU9X3H8deL80BFbCEqohKxERWrDaYX1NIfOtiMmrbAdNIJk7GkdYKtcdSOrXFsp7HTydSJQWNmDBWjIxqjyUw0Mi2ROBRrbS3htKjgqaChihAQMRXrCMfdu3/cYk+872fP/X73drnP8zFzs3vf9372+3a9F9/d/ex3P44IARj9xrS6AQAjg7ADmSDsQCYIO5AJwg5k4pCR3NlYj4tDNX4kdwlk5T39r/bGHg9VKxV22xdKulVSh6TvRMSNqdsfqvE623PK7BJAwppYVVhr+Gm87Q5Jt0m6SNLpkhbYPr3R+wPQXGVes8+StCkiXomIvZIekDS3mrYAVK1M2I+X9Nqg37fUtn2A7UW2u21392pPid0BKKNM2Id6E+BDn72NiKUR0RURXZ0aV2J3AMooE/YtkqYO+v0ESVvLtQOgWcqEfa2k6bZPsj1W0uclLa+mLQBVa3jqLSL22b5C0koNTL3dFREbKusMQKVKzbNHxApJKyrqBUAT8XFZIBOEHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBOlVnHFyBgz8/Rkve+wzsLaz+Yfnh577J5k/W9n/VOy/oUJ25L1nt7ewtq8xy9Pjj3pbifr457dnKz37XwzWc9NqbDb3ixpt6Q+SfsioquKpgBUr4oj+/kRsbOC+wHQRLxmBzJRNuwh6Se2n7K9aKgb2F5ku9t2d6/Srw8BNE/Zp/GzI2Kr7WMkPWr7hYh4fPANImKppKWSdKQnRcn9AWhQqSN7RGytXe6Q9JCkWVU0BaB6DYfd9njbE/Zfl/QZSeuragxAtco8jZ8s6SHb++/nexHxSCVdjTJx7ieT9c1XpV/drDz328n6CYccVljrV39ybFn17n1GZ/FnAHrm3J4ePCddPvPeK5P1k657Mn0HmWk47BHxiqT0XzGAtsHUG5AJwg5kgrADmSDsQCYIO5AJTnGtwJjx45P1K+79frJ+wWG76+xhXLK6J4pPI31676HJsV/88WXJ+pj30qeZnvy9er0X6zt8bLLeuSN934ddlO4NH8SRHcgEYQcyQdiBTBB2IBOEHcgEYQcyQdiBTDDPPkwdv/xLhbW37j8qObbePHrP3vSJovMfS3/l8pRHik8jnfD9/0yOna41yXo9Zb56qN6Rpq9O/dgXN5XYe344sgOZIOxAJgg7kAnCDmSCsAOZIOxAJgg7kAnm2Ydp34xphbXHfu07pe77kiV/kayf8vX/KHX/gMSRHcgGYQcyQdiBTBB2IBOEHcgEYQcyQdiBTDDPPkx+8pnC2llP/kly7E/PSc/D37TozmT9mo5Lk/Vpd79SWNu37efJschH3SO77bts77C9ftC2SbYftb2xdjmxuW0CKGs4T+PvlnThAduuk7QqIqZLWlX7HUAbqxv2iHhc0q4DNs+VtKx2fZmkeRX3BaBijb5BNzkitklS7fKYohvaXmS723Z3r/Y0uDsAZTX93fiIWBoRXRHR1VlngUIAzdNo2LfbniJJtcsd1bUEoBkaDftySQtr1xdKeriadgA0iyPS3/xt+35J50k6StJ2SV+V9CNJP5D0cUmvSvpcRBz4Jt6HHOlJcbbnlGz54DN9bfrly+Ljnih1/1/5+bmFtX+7/dPJscd8t/jzA5LU/+67DfVUhTj3k8n6m2cenqz/Ykbx3/bsc55Pju2Pcq9wn3nw9GT9uG805zsK1sQqvR27hly4vu6HaiJiQUEpv9QCBzE+LgtkgrADmSDsQCYIO5AJwg5kou7UW5VynXo75PjjkvUTf5SetfyHKauT9cM9trDWr/Ry0CvfLV6KWpJuvvIL6X3/9OVkXZOLl7N+7WvpyaDln1qarB93SHpK842+4o9nP/zOjOTYW3782WS9nhP/uTdZP+Rfnip1/0VSU28c2YFMEHYgE4QdyARhBzJB2IFMEHYgE4QdyATz7AeBzX9ffAqrJH16Tk9h7drjHkmOPbWzo6Ge9vvu21OT9Q4Xz/MvmPB6cuzu/r3J+vndX0rWJ996aHFfjz2dHHuwYp4dAGEHckHYgUwQdiAThB3IBGEHMkHYgUwwzz7Kdfzqqcn6C9eOT9Z7Lri91P7HJI4n9c61v/BP/zxZH7uyu6GeRjPm2QEQdiAXhB3IBGEHMkHYgUwQdiAThB3IRN1VXHFw69vwYrIe/b+erKfmyYej08Xny/fW+YjH1BteStZ3PvWxZL1v55vpHWSm7v9J23fZ3mF7/aBtN9h+3fa62s/FzW0TQFnD+Wf7bkkXDrH9loiYWftZUW1bAKpWN+wR8bik9PpEANpemRdkV9h+tvY0f2LRjWwvst1tu7tXxWtvAWiuRsO+RNInJM2UtE3S4qIbRsTSiOiKiK5OpRfiA9A8DYU9IrZHRF9E9Eu6Q9KsatsCULWGwm57yqBf50taX3RbAO2h7jy77fslnSfpKNtbJH1V0nm2Z0oKSZslXdbEHlHC1r/6jWR97QU3Jev9Kl77Xar/vfH/eNP8wtriv1mSHHvHx1cl62f+5ZXJ+knXPZms56Zu2CNiwRCb72xCLwCaiI/LApkg7EAmCDuQCcIOZIKwA5ngFNdRwF1nFNZWX5meWpswJj21dtqKy5P1GdduStYnvVU8/bVw1qLk2Bd+/7Zkveu3XkjWOcH1gziyA5kg7EAmCDuQCcIOZIKwA5kg7EAmCDuQCebZR4HNfzChsFZvHn3JL6Yn6zNuSn/9YN9bbyXrKSesHHJl4fft/uzeZP3PpqxO1m885Q8La30vvZwcOxpxZAcyQdiBTBB2IBOEHcgEYQcyQdiBTBB2IBPMsx8EYvbMZP2BS76ZqKb/PX/kj2en9/3ShmS9jMMfWpOsv7j4sGT97HG96fGXH11YO/lq5tkBjFKEHcgEYQcyQdiBTBB2IBOEHcgEYQcywTz7QeC9o9LnpJ85trOw1q9Iju3Y8T/J+r5ktZyOo4vnwSVpwpj0+exjVPzfLUkxMT0+N3WP7Lan2l5tu8f2BttX1bZPsv2o7Y21y4nNbxdAo4bzNH6fpGsiYoakcyR92fbpkq6TtCoipktaVfsdQJuqG/aI2BYRT9eu75bUI+l4SXMlLavdbJmkec1qEkB5H+kNOtvTJJ0laY2kyRGxTRr4B0HSMQVjFtnutt3dqz3lugXQsGGH3fYRkn4o6eqIeHu44yJiaUR0RURXp8Y10iOACgwr7LY7NRD0+yLiwdrm7ban1OpTJO1oTosAqlB36s22Jd0pqScibh5UWi5poaQba5cPN6VD6Ijndybr9+4+trC2YMLrVbdTmY3XnJysn9rZkaz3qz9ZH/ezQz9yT6PZcObZZ0u6RNJzttfVtl2vgZD/wPalkl6V9LnmtAigCnXDHhFPSCr6Nv851bYDoFn4uCyQCcIOZIKwA5kg7EAmCDuQCU5xPQj0bXwlWf+7f51bWFvwe99Ojo170nPVm/7rnGS9nvvm3VZYO7nz3+uMTp/ae9qKy5P1U7/WXVhLn/g7OnFkBzJB2IFMEHYgE4QdyARhBzJB2IFMEHYgE8yzjwIzvlX8xUHfmn1acuxDp9T5GoJTGuno/41JHE/668yjL3j54mR9xk27kvW+Xr5KejCO7EAmCDuQCcIOZIKwA5kg7EAmCDuQCcIOZIJ59lGgb8OLhbXV501Ljl2y+HeS9Z4Lbm+kpfdd+ur5hbW1K89Ijj0xcT66JEXvGw31lCuO7EAmCDuQCcIOZIKwA5kg7EAmCDuQCcIOZMIR6W/Qtj1V0j2SjpXUL2lpRNxq+wZJX5K0f7Lz+ohYkbqvIz0pzjYLvwLNsiZW6e3YNeSqy8P5UM0+SddExNO2J0h6yvajtdotEfGNqhoF0DzDWZ99m6Rtteu7bfdIOr7ZjQGo1kd6zW57mqSzJK2pbbrC9rO277I9sWDMItvdtrt7tadUswAaN+yw2z5C0g8lXR0Rb0taIukTkmZq4Mi/eKhxEbE0IroioqtT4ypoGUAjhhV2250aCPp9EfGgJEXE9ojoi4h+SXdImtW8NgGUVTfsti3pTkk9EXHzoO1TBt1svqT11bcHoCrDeTd+tqRLJD1ne11t2/WSFtieqYHVbzdLuqwpHQKoxHDejX9C0lDzdsk5dQDthU/QAZkg7EAmCDuQCcIOZIKwA5kg7EAmCDuQCcIOZIKwA5kg7EAmCDuQCcIOZIKwA5kg7EAm6n6VdKU7s9+Q9N+DNh0laeeINfDRtGtv7dqXRG+NqrK3EyPi6KEKIxr2D+3c7o6IrpY1kNCuvbVrXxK9NWqkeuNpPJAJwg5kotVhX9ri/ae0a2/t2pdEb40akd5a+podwMhp9ZEdwAgh7EAmWhJ22xfaftH2JtvXtaKHIrY3237O9jrb3S3u5S7bO2yvH7Rtku1HbW+sXQ65xl6LervB9uu1x26d7Ytb1NtU26tt99jeYPuq2vaWPnaJvkbkcRvx1+y2OyS9JOl3JW2RtFbSgoh4fkQbKWB7s6SuiGj5BzBs/7akdyTdExFn1LZ9XdKuiLix9g/lxIj4Spv0doOkd1q9jHdttaIpg5cZlzRP0hfVwscu0dcfaQQet1Yc2WdJ2hQRr0TEXkkPSJrbgj7aXkQ8LmnXAZvnSlpWu75MA38sI66gt7YQEdsi4una9d2S9i8z3tLHLtHXiGhF2I+X9Nqg37eovdZ7D0k/sf2U7UWtbmYIkyNimzTwxyPpmBb3c6C6y3iPpAOWGW+bx66R5c/LakXYh1pKqp3m/2ZHxKckXSTpy7WnqxieYS3jPVKGWGa8LTS6/HlZrQj7FklTB/1+gqStLehjSBGxtXa5Q9JDar+lqLfvX0G3drmjxf28r52W8R5qmXG1wWPXyuXPWxH2tZKm2z7J9lhJn5e0vAV9fIjt8bU3TmR7vKTPqP2Wol4uaWHt+kJJD7ewlw9ol2W8i5YZV4sfu5Yvfx4RI/4j6WINvCP/sqS/bkUPBX39iqRnaj8bWt2bpPs18LSuVwPPiC6V9DFJqyRtrF1OaqPe7pX0nKRnNRCsKS3q7Tc18NLwWUnraj8Xt/qxS/Q1Io8bH5cFMsEn6IBMEHYgE4QdyARhBzJB2IFMEHYgE4QdyMT/AfKih+ZZtV4AAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Run this cell a few times to randomly display some digit data\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "index = random.randint(0, len(train_images))\n",
    "# index = 10\n",
    "print (\"Displaying train index = \", index, \", label = \", train_labels[index])\n",
    "\n",
    "# print(\"train label [{}] = {} \".format(index, train_labels[index]))\n",
    "# print (\"------------ raw data for train_image[{}] -------\".format(index))\n",
    "# print(train_images[index])\n",
    "# print (\"--------------------\")\n",
    "\n",
    "plt.imshow(train_images[index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 3 : Shape Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 - Shape the array to 4 dimensional\n",
    "ConvNets expect data in 4D.  Let's add a channel dimension to our data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_images shape :  (60000, 28, 28, 1)\n",
      "train_labels shape :  (60000,)\n",
      "test_images shape :  (10000, 28, 28, 1)\n",
      "test_labels shape :  (10000,)\n"
     ]
    }
   ],
   "source": [
    "## Reshape to add 'channel'.\n",
    "train_images = train_images.reshape(( train_images.shape[0],  28, 28, 1))\n",
    "test_images = test_images.reshape((test_images.shape[0], 28, 28, 1))\n",
    "\n",
    "print(\"train_images shape : \", train_images.shape)\n",
    "print(\"train_labels shape : \", train_labels.shape)\n",
    "print(\"test_images shape : \", test_images.shape)\n",
    "print(\"test_labels shape : \", test_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 - Normalize Data\n",
    "The images are stored as a 2D array of pixels.  \n",
    "Each pixel is a value from 0 to 255  \n",
    "We are going to normalize them in the range of 0 to 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Normalize pixel values to be between 0 and 1\n",
    "train_images, test_images = train_images / 255.0, test_images / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import matplotlib.pyplot as plt\n",
    "# import random\n",
    "\n",
    "# index = random.randint(0, len(train_images))\n",
    "# index = 10\n",
    "# print (\"Displaying train index = \", index)\n",
    "\n",
    "# print(\"train label [{}] = {} \".format(index, train_labels[index]))\n",
    "# print (\"------------ raw data for train_image[{}] (just printing first 3 rows) -------\".format(index))\n",
    "# print(train_images[index][0:2])\n",
    "# print (\"--------------------\")\n",
    "\n",
    "# plt.imshow(train_images[index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 4 : Create Model\n",
    "\n",
    "### Neural Net Architecture\n",
    "\n",
    "<img src=\"https://www.pyimagesearch.com/wp-content/uploads/2016/06/lenet_architecture-768x226.png\" style=\"width:80%\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.1 - Create a CNN\n",
    "\n",
    "The code below define the convolutional base using a common pattern: a stack of [Conv2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/Conv2D) and [MaxPooling2D](https://www.tensorflow.org/api_docs/python/tf/keras/layers/MaxPool2D) layers.\n",
    "\n",
    "As input, a CNN takes tensors of shape (image_height, image_width, color_channels), ignoring the batch size. If you are new to color channels, MNIST has one (because the images are grayscale), whereas a color image has three (R,G,B). In this example, we will configure our CNN to process inputs of shape (28, 28, 1), which is the format of MNIST images. We do this by passing the argument `input_shape` to our first layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 13, 13, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 11, 11, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 3, 3, 64)          36928     \n",
      "=================================================================\n",
      "Total params: 55,744\n",
      "Trainable params: 55,744\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = tf.keras.Sequential( [ \n",
    "    tf.keras.layers.Conv2D(32, (3, 3), activation='relu', input_shape=(28, 28, 1)),\n",
    "    tf.keras.layers.MaxPooling2D((2, 2)),\n",
    "    tf.keras.layers.Conv2D(64, (3, 3), activation='relu'),\n",
    "    tf.keras.layers.MaxPooling2D((2, 2)),\n",
    "    tf.keras.layers.Conv2D(64, (3, 3), activation='relu')\n",
    "])\n",
    "\n",
    "print (model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Above, you can see that the output of every Conv2D and MaxPooling2D layer is a 3D tensor of shape (height, width, channels). The width and height dimensions tend to shrink as we go deeper in the network. The number of output channels for each Conv2D layer is controlled by the first argument (e.g., 32 or 64). Typically,  as the width and height shrink, we can afford (computationally) to add more output channels in each Conv2D layer."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.2 - Add Dense layers on top\n",
    "To complete our model, we will feed the last output tensor from the convolutional base (of shape (3, 3, 64)) into one or more Dense layers to perform classification. Dense layers take vectors as input (which are 1D), while the current output is a 3D tensor. First, we will flatten (or unroll) the 3D output to 1D,  then add one or more Dense layers on top. MNIST has 10 output classes, so we use a final Dense layer with 10 outputs and a softmax activation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 26, 26, 32)        320       \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 13, 13, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 11, 11, 64)        18496     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 5, 5, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 3, 3, 64)          36928     \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 576)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 64)                36928     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                650       \n",
      "=================================================================\n",
      "Total params: 93,322\n",
      "Trainable params: 93,322\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model.add(tf.keras.layers.Flatten())\n",
    "model.add(tf.keras.layers.Dense(64, activation=tf.nn.relu))\n",
    "model.add(tf.keras.layers.Dense(10, activation=tf.nn.softmax))\n",
    "\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4.3 - Compile and Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=tf.keras.optimizers.Adam(),  # 'adam'\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 5 - Setup Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saving TB logs to :  /tmp/tensorboard-logs/cnn-mnist/2020-05-21--00-09-25\n"
     ]
    }
   ],
   "source": [
    "## This is fairly boiler plate code\n",
    "\n",
    "import datetime\n",
    "import os\n",
    "\n",
    "app_name = 'cnn-mnist' # you can change this, if you like\n",
    "\n",
    "tb_top_level_dir= '/tmp/tensorboard-logs'\n",
    "tensorboard_logs_dir= os.path.join (tb_top_level_dir, app_name, \n",
    "                                    datetime.datetime.now().strftime(\"%Y-%m-%d--%H-%M-%S\"))\n",
    "print (\"Saving TB logs to : \" , tensorboard_logs_dir)\n",
    "\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=tensorboard_logs_dir, histogram_freq=1)\n",
    "\n",
    "# Loading of tensorboard in Colab\n",
    "if RUNNING_IN_COLAB:\n",
    "    %load_ext tensorboard\n",
    "    %tensorboard --logdir $tb_top_level_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 6 : Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training starting ...\n",
      "Train on 48000 samples, validate on 12000 samples\n",
      "Epoch 1/10\n",
      "48000/48000 [==============================] - 13s 280us/sample - loss: 0.1608 - accuracy: 0.9487 - val_loss: 0.0683 - val_accuracy: 0.9810\n",
      "Epoch 2/10\n",
      "48000/48000 [==============================] - 4s 85us/sample - loss: 0.0492 - accuracy: 0.9843 - val_loss: 0.0448 - val_accuracy: 0.9876\n",
      "Epoch 3/10\n",
      "48000/48000 [==============================] - 4s 83us/sample - loss: 0.0355 - accuracy: 0.9888 - val_loss: 0.0547 - val_accuracy: 0.9842\n",
      "Epoch 4/10\n",
      "48000/48000 [==============================] - 4s 83us/sample - loss: 0.0268 - accuracy: 0.9911 - val_loss: 0.0346 - val_accuracy: 0.9914\n",
      "Epoch 5/10\n",
      "48000/48000 [==============================] - 4s 85us/sample - loss: 0.0221 - accuracy: 0.9928 - val_loss: 0.0329 - val_accuracy: 0.9903\n",
      "Epoch 6/10\n",
      "48000/48000 [==============================] - 4s 76us/sample - loss: 0.0164 - accuracy: 0.9948 - val_loss: 0.0410 - val_accuracy: 0.9893\n",
      "Epoch 7/10\n",
      "48000/48000 [==============================] - 4s 78us/sample - loss: 0.0131 - accuracy: 0.9957 - val_loss: 0.0497 - val_accuracy: 0.9887\n",
      "Epoch 8/10\n",
      "48000/48000 [==============================] - 4s 74us/sample - loss: 0.0133 - accuracy: 0.9957 - val_loss: 0.0376 - val_accuracy: 0.9906\n",
      "Epoch 9/10\n",
      "48000/48000 [==============================] - 3s 71us/sample - loss: 0.0107 - accuracy: 0.9965 - val_loss: 0.0384 - val_accuracy: 0.9908\n",
      "Epoch 10/10\n",
      "48000/48000 [==============================] - 3s 72us/sample - loss: 0.0101 - accuracy: 0.9966 - val_loss: 0.0497 - val_accuracy: 0.9877\n",
      "training done in 47,643.77 ms\n"
     ]
    }
   ],
   "source": [
    "epochs = 10\n",
    "\n",
    "print (\"training starting ...\")\n",
    "t1 = time.perf_counter()\n",
    "history = model.fit(train_images, train_labels, \n",
    "                    epochs=epochs, validation_split = 0.2, verbose=1,\n",
    "                    callbacks=[tensorboard_callback])\n",
    "t2 = time.perf_counter()\n",
    "print (\"training done in {:,.2f} ms\".format ((t2-t1)*1000))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 7 : See Training History"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9b3/8dcne0JWQsKSAEEWZQ07WqqoWKvWalVUcGm113pv92t/Xaz2WmuvtbfaRW+trdaltlhqsbXqpbVVAe2iMmGRTSJLhgSQbJON7JnP748zCZOQyCSZMJOZz/PxyCMzc86c852BvOc73/M9nyOqijHGmMgVE+oGGGOMGVoW9MYYE+Es6I0xJsJZ0BtjTISzoDfGmAgXF+oG9DRq1CgtKCgIdTOMMWZYKSoqqlTVnN6WhV3QFxQU4HK5Qt0MY4wZVkTE3dcyG7oxxpgIZ0FvjDERzoLeGGMinAW9McZEOAt6Y4yJcBb0xhgT4SzojTEmwoXdPHpjjDnVVJUOr9LudX53qNLRcfx+u9fbbXl7h996Xm/X/ePr+x7v3F7PZR3eHus6v8ekJ3HdkglBf30W9MaYiKSq1DW3U1HfQnl9s/O7roWKhhbK65opr2+hvL6FivoWapvaQt1cAOZNyLSgN8aYDq9S1XA8pLtC3Bfk5fXNvjBvoaXde8LzE+JiyE1LJDctkSk5qXxocjaZyfHExcYQGyPExcjx37Ex3e93u33isrjY7o/H9lwe47ePWL9tivNbRIbkPbOgN8aEhea2jq7gPt7z9t3vCvUWqhpa8PZyYbyM5HhyfAG+YEKW73YSuemJXY/npCWRnhQ3ZIEarizojYlgja3tHK5p5khtE0dqmjlU08SR2iaO1rXgDYPLiLa2e6n09c7rm9tPWB4jMCo1kdz0REanJzE7L6NbaB+/nUhSfGwIXsHwYEFvzDDV1uHl/dpmDtc0caS2mcO+MD9c08ThWifcaxpPHHvOSUtkTHoS8bGh79XGxcRw+pg0PjxlFLnp3YM7Ny2JkSMSiI0JfTuHOwt6Y8KQ16tUNrQ4gV3T5OuJO+F92BfmFQ0t9OyUZyTHMzYjibzMZBZMzGRsRjLjMpMYl5HMuMxkRqcnkRBns6qjjQW9MaeYqlLX1N41jNIZ5v498fdrm2nr6J7iyfGxjPWF9rJpOYzLdEJ8rC/Ex2YkMSLR/qTNiex/hTFD7EhtE28fqGZTSTVF7hrcVcdobO3otk5cjDAmwwnx+ROyGJuRTJ4vxDvDPTMlPuoOIprgsKA3JohUlf2Vx9h0oJq3S5xwL61uAmBEQizzJ2Zx1mnZznCKrxc+LjOZUamJNhZthowFvTGD0N7hZfeReifUD1TjcldT2dAKQPaIBBYVjOSmD01iccFIpo9NIy7WxsfNqWdBb0w/NLd1sK20hk0l1bxd4mGz20NDizMtMD8rmXOm5rBo0kgWFYxkcs4IG2oxYcGC3pgPUNfcRlGJp6vH/k5ZLa0dztmW00an8ol541hUMJLFk0YyNiM5xK01pncW9Mb4Ka9vZtMBj9NjP1DN7vfrUHUOls7Ky+CmpQUsKhjJwolZZI1ICHVzjQmIBb2JWqqKu6qxq7e+qaSakqpGwJnKOG9CJl86fypLJo1k7oRMUhLsz8UMT/Y/10SNDq+y5/163/i6E+7l9S2Ac6LRooKRXLdkAosKRjIrL4N4O3BqIoQFvYlIqsqR2mbeKatlx6Fath+qZfNBT1c9lbEZSZx5WjaLJo1kccFIpuamEmPTG02EsqA3w56q8n7d8VDv/F11zJnmGBsjTM1N5WOzx7LYNyMmPyvZZsSYqGFBb4YVVeVoXQvvlNU4oX7ICfXOuesxAtNGp3HeGbnMzstgdn4G08ekk5xglQ1N9LKgN2HtqK+nvv1QLdvLath+qI7KBmdcPUZgam4ay6blMic/g1l5GcwYa6FuTE8W9CZslPuFemdvvaL+eKhPyU3lnGmjmNPZUx+bbjNhjAmA/ZWYoaUKzTWQlAl+Y+Ll9c1s9w/1stquGTAxApNzUjl76ihn+CUvgxnjLNSNGSj7yzFD51gl/HYllG2iPT6V6oRxlGkuu5tHsrslm1LNpZRckrIL+PCUUczKy2BOvoW6McFmf00m6JpaO9i6bTPTXrmJ1JZyft5+BentjUxoLmdKvJur9W0S4luPP6Fe4NA4aCyAygJwF0CW7ydzIqTmdvs2YIzpHwt6M2iqyr6KBjbsqWBjcQVNB97m57E/IAblgbEPMHrm2czMz2TGuHRSE+PA64Vj5eApOfFn33qoP9x9B/EpTuB3hn9WAWRNPP5BkJBySl+vCQFV8LZDezO0t/Tyu8djHa091mmG9p6PtUCH77lJGTDnGph0LsRE3olyogFcIFhELgIeBGKBX6rq93ssnwg8AeQA1cANqlrmW/Y/wMd8q35XVX/3QftauHChulyu/r4Oc4rVN7fxj71VbCyu4PXiCg7VODXXb8jazbdbfkBHSi5yw3Mkjjm9/xtva4Ka0u4fADXu47dbG7qvnzq6+zcA/w+EtLER+Yc7rLUegzIXHHwTjm53/r27BXJL7+HNYC9mLhCXBHGJfj+++zWlzrGkjPEw93qYdz1kTgjGqz1lRKRIVRf2uuxkQS8isUAx8BGgDNgErFLVXX7r/B54SVV/JSLnAzer6o0i8jHgP4GLgURgI3C+qtb1tT8L+vCkquw6UsfG4go27Klgs9tDu1dJS4xj6ZRRLDs9h4ua/0zW+tthbCFc96wz5BL8hkBjVe/fBjxuqCsD9R5fPzbBF/4Tnd+Z450/5swJzu/U0fZBMNTqj0Lpm06wH/wXHHkHtAMQyJ4CSel+AZzk/Jv53z8hmH2PxfYI67gkiEvovo7/9mLj+x4CbGuGPf8Hm38N+zc4j512Lsy/Ec641NlWmBts0J8F3K2qH/Xd/yaAqt7nt85O4KOqWibO6Ya1qpouIl8DElX1v33rPQ68rKrP9rU/C/rw4TnWyht7K9m4p4LX36vomuo4c1w6y6blsGxaDvMnZhEfI/Daf8MbD8DUC+HqpyBhRGga3d4KtaXdvwF4SqD6gPN4k6f7+rEJkJHvC//xzodB5+2M8ZCeB7E2whkwVah8zwn0g286AV+931kWlwz5C2HCmTD+TBi/yBkyCTc1B2HrM7BlNdQehOQsmHMtzLsRxswKdev69EFBH8j/4Dyg1O9+GbCkxzrbgKtwhneuANJEJNv3+LdF5EdACnAesKvHcxGRW4FbASZMGF5flyJJh1d5p6yGjcXOWPu20hq8Cpkp8Zw9NYdzp+Vw9rRR5KYlHX9Seys8/yXY9luY/yn42I9CG4xxCZA92fnpTUu98zW9ttT5g+78XVMK7/0NGo52X19inLD3D//MCb7bE5wPifik3vcVDdpb4ci27sHeWOUsS8mGCWfBwk87v8fMcf59wl3mBDj3djjn63Bgg9PLdz0Bb/0cxs51evmzVkByZqhbGrBAevRX4/TWb/HdvxFYrKpf9FtnHPBTYBLwOk7oz1TVWhG5E7gaqADKgbdV9cG+9mc9+lOrvL6ZN4or2VBcwRvvVVDT2IYIzB2f2dVrn5Of2fv1TJvr4NlPwv71cN6dcM7Xhv/smLZmqDvkfCPo+kDw+2CoO+wbdvAzItcv/P2GhTofS0wLzWsZCk01ULbpeLAfKvKNnwMjJzuBPuFM53f25OH//6FTYzW88yxs+TUc3eEMBc243OnlF3w4LF7nkA/d9Fg/FXhXVfN7WfYM8BtVXdfX/izoh1Zbh5ctB2vYsKecjcUV7DzsHC4ZlZroBPvpOZw9ZdTJL6pRdwRWXw0Vu+HjDzkHr6JBR7szK6imxzcC/w+Ejtbuz0nKPP4NINN3XGDEKEgZ5fR6R/h+J2WERWB0U1N6fGy99C04uhNQiIlzjsV0Bvv4JUNzTCbcqMKRrU4vf/taaKmFrEnO//+510P6uJA1bbBBH4dzMHY5cAjnYOx1qrrTb51RQLWqekXkXqBDVe/yHcjNVNUqEZkDPAPMVdX2vvZnQR98h2uanOGYPRX8Y28l9S3txMUI8ydmsWxaDueensP0MemBl+ktfxdWr3DGu6/5FUy5YGhfwHDSOXW0ptQZ363p5YOg56yhTjHxTuCnZMOIbOeDoOsDYeTx2/6PxQSxro+3A8p3+YLd91NX5ixLSIPxi3299TMhb0HojsOEi9ZG2P2i08svecMZ5ptygdPLn3bRKR+mGlTQ+zZwCfATnOmVT6jqvSJyD+BS1RdEZAVwH878p9eBz6tqi4gkAZt9m6kD/kNVt37Qvizog+NA5TGeecvNhj0VvFfuBMu4jCSWnZ7Lsmk5fGhKNulJ8f3fcMk/YM0q56vrdc/CuLlBbnkUaG2ExkrnzOHGar/bfTzWXNvHhsQZJ+4Kf79vB3095n88obURDm/2G19/G1p8E+LSxh0fgplwJoyeGdwPlUhTvd85eLv1GecbX8ooKFwJ8z8JOQOYYjwAgw76U8mCfnCqGlp46NX3WP3WQWJEWHLayK6x9im5qYOrwb7jD/DHf3fmp1+/1pmyaIZeR1svHwhVzkHPrseqjv9urDrxOEKnhFQn9BNSobIYvG3O47kzugd7xvjwG0YaDrwdsPdV2PI07Pmzc5JX/iKnlz/ryiE9XmNBHwWa2zp48h8l/Gz9XhrbOli1eDxfXj6NnLQgzf/918Pw8h1OEKx8xhk2MOHJ63VO/ukK/x4fBMcqnZ57Z7iPX+xMITTB1VAB76xxxvMr90D8CJh5hTNrZ/ySoH+QWtBHMK9XeWHbYe5/eQ+Hapq4YHout198BlNyg9Rz8Hrhr3fCmz+D6ZfBlY9F93RCY/pL1TkTeMvTzrfi1gbIngrzboDCVZA2Oii7saCPUP/cV8n31u1mx6E6ZudlcMcl0zlrcnbwdtDWDH+8FXb9CZZ8Fj76PTuL1JjBaGmAXc87vfzSN0FinQO382+EKR8Z1Dkogz1hyvSHtwPefwdypg9Zz3dveT3f//O7vLK7nLzMZH5y7VwuKxwX3ItbN1bDmuucA3Uf/R6c9fngbduYaJWY6vTk590AFcXOjJ1tv3XKL6SOcQ7enn9n0HdrQR9MjdXw3L/BvtecA17zboSFNzsHL4Ogor6Fn7xSzJpNpaTEx/KNi87g5qUFJMUHeTaEx+1Mn/SUwIonnYNIxpjgypkGF34Xlt8F7/3VN5ZfPCS7sqGbYDm60+kB1x6CZV93evXvrnMKbE37KCy6BSYvH9DQR1NrB4//fT+PbNhHS7uXG86cyBfPn0J26hAUWjqyzTkRqr0ZVv4WCpYGfx/GmN55vQMeHrWhm6G283l4/nPO1Kmb1zmzGMAJ/aKnnJ/iFc4ZdIv+zTmDLoBZKx1e5Q+by/jhX4t5v66Zj84czTcuOoPTclKH5nXsfQWe/ZQzA+OTL0DuGUOzH2NM74boGJj16AfD2+FUbfz7jyB/MVzzNKSPPXG99lbY/QJsehwO/tM52WjWCif08+b3uum/v1fJvet2s/tIHYXjM/nWx6azqGAIpzRuWQ0vfsk5tnD973t/HcaYsGU9+qHQ5IHnPgN7/+ZUbbzk/r5rVsclwOwVzs/7O2DTL50CSVt/45xKvugzzvza+CT2vF/P99btZmNxBflZyfzvqnlcOmfs4E50+iCq8Pr9sP5ep/72Nb926oMbYyKG9egHony3Mx5fUwqX/MApw9pfzbWwbY0T+pXFeJNG8nrqRdx1aDE1ieP40vKp3HjWRBLjhvC08452+L+vwOZfOfN5P/7Q8Cgja4w5gfXog2nXC/D8Z53rmN70knNm4UAkZcCSf+dY4af580vPkr79Kc5vWsPGxN/SftoFxI/5d4gpCGrTu2lpgLU3O0f7z/4qnP8tO+XdmAhlQR8orxc2fM8Z5shbANf+ZlAlSTu8yu9dpfzwb8VU1GfysTn/w4ylaeTve5b4oqec6Y1ZBbDw35w5t8EsOdBQDs9c48ywufTHA/tGYowZNmzoJhDNtfCHW6H4L07oXvLDAZ8MpapsKK7g++veZc/RehZMzOKOS6azYKJfrZFeD95e5UzR7OPgbcAq98Lqq5zreF79JJx+8eC2Z4wJCzZ0MxgVe5zxeE8JXPKAE7YDHOLYebiW+9a9y9/3VjIxO4VHrp/PRbPGnHigtc+Dt6t9B29vgZlX9v/DpvRteOZap272Tf8H+QsG9DqMMcOL9eg/yLvrnJ58fBJc/asBnzx0pLaJH/61mOc2l5GRHM+Xl0/l+iUTSYjrx5zZHgdvSR7p1MdY+OnAzrzd/ZJz1m76OKfEcF/XVDXGDEtW1Ky/vF54/Qew4T7nYsArVzsXge6nhpZ2fr5hH7/8+368Xrh5aQGfO28KGckDuOBHJ1U48Dpseuz4mbdTL3R6+VMu6P2Ei7cfgz9/HcbNcy4WMmLUwPdvjAlLNnTTH8118Mf/cIoMFa5yDlbGJ/drE+0dXtZsKuUnrxRT2dDK5XPH8dULT2f8yJTBt08ETlvm/PifefvM1ScevPV64dXvwD9+AtMuhhVPQEIQ2mCMGVasR++vcq9zmbyqffDRe2HJf/R7PL6qoYWVj77Je+UNLJ40kjsvmU7h+MwharBPeyu8+yK8/cvuB2/bGmHnH53hnYvvH1QJVGNMeLMefSCKX4bnboHYePjk8zDpnAFt5uWdR3mvvIGHVs3j40N5Rqu/uAQn2Gdd5Ry8dT0O234Hbcecyngf/orNkTcmilnQe73wxg+dEgBjZjvj8ZkTBrw5l7uaUakJpy7kexozyxluuuBu58zdMbNOfRuMMWEluoO+pd45y3X3izD7Gvj4g4Mewy5ye5g/ISs0Ie8vKQPGZIS2DcaYsBC9QV+1z5kfX1kMF97rXEFpkOFcUd+Cu6qR65cM/BuBMcYEW3QG/XuvwHOfdk4cuuEPMPm8oGy2yF0NwIKJQ1hO2Bhj+im6rvSsCm/8yKkjkzEBbt0QtJAHcJV4SIiLYVaelfk1xoSP6OnRtx5zrgK163mnfMDlP4WEEUHdhcvtoTA/Y2hLCxtjTD9FR4+++gD88iNOobCP3OM7cSi4Id/c1sHOw7U2bGOMCTuR36Pf9xr8/mbn9vVrYcryIdnNttIa2jqUhf5VKI0xJgxEbo9eFf7xEPzmKkjPg1vXD1nIgzNsA3QvN2yMMWEgMnv0rY3wwhdhx1qYcTlc/jNITB3SXRa5PUzOGUHWCLsUnzEmvERe0Hvc8LvrnVIAp+j0f69XKXJ7uGjmmCHdjzHGDERkBf3+Dc54vLcDrv89TP3IKdntvooGapvaWFBgwzbGmPATOUFfUQy/vhJGTYWVz5zSC2sU+cbn7UCsMSYcRU7Q50yDy/4XZlwGiWmndNcut4fsEQlMGhXcKZvGGBMMAc26EZGLRGSPiOwVkdt7WT5RRF4VkXdEZIOI5Pst+4GI7BSR3SLykAxlta9515/ykAdfIbOJYVDIzBhjenHSoBeRWOBh4GJgBrBKRGb0WO0B4GlVnQPcA9zne+6HgKXAHGAWsAhYFrTWh4HKhhYOVB6zaZXGmLAVSI9+MbBXVferaiuwBri8xzozgFd9t9f7LVcgCUgAEoF44OhgGx1ObHzeGBPuAgn6PKDU736Z7zF/24CrfLevANJEJFtV/4UT/Ed8Py+r6u6eOxCRW0XEJSKuioqK/r6GkCpye0iIjWFWntV+N8aEp0CCvreB554Xmv0qsExEtuAMzRwC2kVkCjAdyMf5cDhfRE64Rp+qPqqqC1V1YU5OTr9eQKi5SqqZnZ9BUrwVMjPGhKdAgr4MGO93Px847L+Cqh5W1StVdR5wp++xWpze/Zuq2qCqDcCfgTOD0vIw0NzWwY5DdTZsY4wJa4EE/SZgqohMEpEEYCXwgv8KIjJKRDq39U3gCd/tgzg9/TgRicfp7Z8wdDNcbT9US2uH1w7EGmPC2kmDXlXbgS8AL+OE9LOqulNE7hGRy3yrnQvsEZFiYDRwr+/xtcA+YDvOOP42VX0xuC8hdFwlVsjMGBP+AjphSlXXAet6PHaX3+21OKHe83kdwL8Pso1hq8hdzWmjRpCdmhjqphhjTJ8it0zxEFN1CplZb94YE+4s6AdoX8UxPI1tLLRCZsaYMGdBP0BF7moAu3SgMSbsWdAPkKvEQ1ZKPJNzrJCZMSa8WdAPUNFBZ3zeCpkZY8KdBf0AVB9rZX/FMRu2McYMCxb0A1BkFwI3xgwjFvQD4HJXEx8rzMm3QmbGmPBnQT8ARSUeZuVZITNjzPBgQd9PLe0dvHOo1gqZGWOGDQv6ftpxqJbWdq8diDXGDBsW9P1khcyMMcONBX0/udweCrJTyEmzQmbGmOHBgr4fVJXNbo8N2xhjhhUL+n44UHmMqmOtVsjMGDOsWND3g8t3opTNuDHGDCcW9P1QVOIhIzmeyTmpoW6KMcYEzIK+H1zuahZMzCImxgqZGWOGDwv6AHmOtbKv4phNqzTGDDsW9AHafNDG540xw5MFfYBcbg9xMULh+MxQN8UYY/rFgj5ARSUeZlohM2PMMGRBH4DWdi/bymps2MYYMyxZ0Adgx+FaWtq9FvTGmGHJgj4ARZ2FzOyMWGPMMGRBHwCXu5oJI1PITUsKdVOMMabfLOhPQlUpcnts2MYYM2xZ0J+Eu6qRyoZWG7YxxgxbFvQncbyQmZUmNsYMTxb0J1HkriY9KY6puVbIzBgzPFnQn4SrxMN8K2RmjBnGLOg/QE1jK++VN9iBWGPMsBZQ0IvIRSKyR0T2isjtvSyfKCKvisg7IrJBRPJ9j58nIlv9fppF5BPBfhFDZcvBGgC7dKAxZlg7adCLSCzwMHAxMANYJSIzeqz2APC0qs4B7gHuA1DV9ao6V1XnAucDjcBfg9j+IeVyVxMXI8y1QmbGmGEskB79YmCvqu5X1VZgDXB5j3VmAK/6bq/vZTnACuDPqto40Maeaq4SDzPHpZOcYIXMjDHDVyBBnweU+t0v8z3mbxtwle/2FUCaiGT3WGcl8NvediAit4qIS0RcFRUVATRp6LV1OIXM5tv4vDFmmAsk6HubbqI97n8VWCYiW4BlwCGgvWsDImOB2cDLve1AVR9V1YWqujAnJyeghg+1nYfraG7z2vx5Y8ywFxfAOmXAeL/7+cBh/xVU9TBwJYCIpAJXqWqt3yrXAH9U1bbBNffUcZVUA7DQzog1xgxzgfToNwFTRWSSiCTgDMG84L+CiIwSkc5tfRN4osc2VtHHsE24KnJ7yM9KZnS6FTIzxgxvJw16VW0HvoAz7LIbeFZVd4rIPSJymW+1c4E9IlIMjAbu7Xy+iBTgfCPYGNSWDyFVxWWFzIwxESKQoRtUdR2wrsdjd/ndXgus7eO5JZx48DaslVY3UVHfwoICG583xgx/dmZsL1xu3/i89eiNMRHAgr4XLreHtMQ4po1OC3VTjDFm0Czoe1FU4mHexCxirZCZMSYCWND3UNvURnF5vQ3bGGMihgV9D5sPelC18XljTOSwoO+hqMRDbIwwd4IVMjPGRAYL+h6K3B5mjE0nJSGgmafGGBP2LOj9tHV42VpawwIbtjHGRBALej+7j9TR1NZh9W2MMRHFgt6Pq8QDYD16Y0xEsaD3U+T2kJeZzNiM5FA3xRhjgsaC3scpZFZtvXljTMSxoPcp8zRxtK7FxueNMRHHgt6nyG3j88aYyGRB7+NyV5OaGMcZY9JD3RRjjAkqC3ofV4mHeRMyrZCZMSbiWNADdc1t7Dlab8M2xpiIZEEPbDlY4ytkZleUMsZEHgt6oKikmhjBCpkZYyKSBT3OFaWmj00nNdEKmRljIk/UB327r5CZ1Z83xkSqqA/6d9+vp7G1gwUFNj5vjIlMUR/0rpJqwK4oZYyJXBb0bg9jM5IYl2mFzIwxkSnqg77I7bH588aYiBbVQX+opokjtc02bGOMiWhRHfRd4/N2INYYE8GiOuiL3B5SEmI5Y0xaqJtijDFDJqqDvrOQWVxsVL8NxpgIF7UJ19DSzrvv17HA6tsYYyJc1Ab9loMevGrz540xkS9qg95V4iFGYJ4VMjPGRLiAgl5ELhKRPSKyV0Ru72X5RBF5VUTeEZENIpLvt2yCiPxVRHaLyC4RKQhe8weuyO3h9DHppCXFh7opxhgzpE4a9CISCzwMXAzMAFaJyIweqz0APK2qc4B7gPv8lj0N3K+q04HFQHkwGj4Y7R1ethz02LCNMSYqBNKjXwzsVdX9qtoKrAEu77HODOBV3+31nct9Hwhxqvo3AFVtUNXGoLR8EPYcredYawcLCyzojTGRL5CgzwNK/e6X+R7ztw24ynf7CiBNRLKBaUCNiPxBRLaIyP2+bwjdiMitIuISEVdFRUX/X0U/Fbk9AFb6wBgTFQIJ+t6ulq097n8VWCYiW4BlwCGgHYgDzvYtXwScBtx0wsZUH1XVhaq6MCcnJ/DWD5CrxMOY9CTyrJCZMSYKBBL0ZcB4v/v5wGH/FVT1sKpeqarzgDt9j9X6nrvFN+zTDjwPzA9KywehyO1hQUEWIr19hhljTGQJJOg3AVNFZJKIJAArgRf8VxCRUSLSua1vAk/4PTdLRDq76ecDuwbf7IE7UtvEoZomFkywYRtjTHQ4adD7euJfAF4GdgPPqupOEblHRC7zrXYusEdEioHRwL2+53bgDNu8KiLbcYaBHgv6q+gHV4kzPm8HYo0x0SKgq2Gr6jpgXY/H7vK7vRZY28dz/wbMGUQbg6rI7SE5PpbpY9ND3RRjjDklou7MWJe7mrnjM4m3QmbGmCgRVWl3rKWd3UfqbdjGGBNVoirot5bW0OFVmz9vjIkqURX0rhIPIjDfgt4YE0WiK+jd1Zw+Oo10K2RmjIkiURP0HV5ly8EaG7YxxkSdqAn6Pe/X09DSbgdijTFRJ2qCvuig70Qpu3SgMSbKRE/Ql1STm5ZIfpYVMjPGRJeoCXqX28NCK2RmjIlCURH0R+uaKfM0scCGbYwxUSigWjfDXWchM5txY8yJ2traKCsro7m5OdRNMQFISkoiPz+f+PjAp4lHR9C7q0mKj2HmOCtkZkxPZWVlpKWlUVBQYEObYU5VqaqqoqysjEmTJgX8vKgYuilyeyjMt0JmxvSmubmZ7OxsC/lhQMFGHqYAAA3ySURBVETIzs7u97eviE++xtZ2dh6us/nzxnwAC/nhYyD/VhEf9J2FzGz+vDEmWkV80Bf5DsTOt0sHGmOiVMQHvcvtYdroVDJSrJCZMeGopqaGn/3sZ/1+3iWXXEJNTc0QtCjyRPSsG69X2XzQw6VzxoW6KcYMC995cSe7DtcFdZszxqXz7Y/P7HN5Z9B/7nOf6/Z4R0cHsbGxfT5v3bp1fS4LBydr/6kU0T364vJ66pvbWWjz540JW7fffjv79u1j7ty5LFq0iPPOO4/rrruO2bNnA/CJT3yCBQsWMHPmTB599NGu5xUUFFBZWUlJSQnTp0/nM5/5DDNnzuTCCy+kqampz/099thjLFq0iMLCQq666ioaGxsBOHr0KFdccQWFhYUUFhbyz3/+E4Cnn36aOXPmUFhYyI033gjATTfdxNq1xy+TnZqaCsCGDRsCbv9f/vIX5s+fT2FhIcuXL8fr9TJ16lQqKioA8Hq9TJkyhcrKykG/x6hqWP0sWLBAg+XX/yrRid94SUsqG4K2TWMiza5du0K6/wMHDujMmTNVVXX9+vWakpKi+/fv71peVVWlqqqNjY06c+ZMraysVFXViRMnakVFhR44cEBjY2N1y5Ytqqp69dVX669//es+99f5fFXVO++8Ux966CFVVb3mmmv0xz/+saqqtre3a01Nje7YsUOnTZumFRUV3dryqU99Sn//+993bWfEiBH9an95ebnm5+d3rde5zt13393VhpdfflmvvPLKXl9Db/9mgEv7yNWI7tEXuT2MSk1kwsiUUDfFGBOgxYsXdzsZ6KGHHqKwsJAzzzyT0tJS3nvvvROeM2nSJObOnQvAggULKCkp6XP7O3bs4Oyzz2b27NmsXr2anTt3AvDaa6/x2c9+FoDY2FgyMjJ47bXXWLFiBaNGjQJg5MiTz94LpP1vvvkm55xzTtd6ndv99Kc/zdNPPw3AE088wc0333zS/QUiosfoi9weFk60QmbGDCcjRozour1hwwZeeeUV/vWvf5GSksK5557b68lCiYmJXbdjY2M/cOjmpptu4vnnn6ewsJCnnnqKDRs29LmuqvaaH3FxcXi93q51Wltb+9X+vrY7fvx4Ro8ezWuvvcZbb73F6tWr+2xbf0Rsj768vpmD1Y12opQxYS4tLY36+vpel9XW1pKVlUVKSgrvvvsub7755qD3V19fz9ixY2lra+sWpMuXL+eRRx4BnAOpdXV1LF++nGeffZaqqioAqqurAef4QFFREQB/+tOfaGtr61f7zzrrLDZu3MiBAwe6bRfglltu4YYbbuCaa64J2sHciA36IitkZsywkJ2dzdKlS5k1axZf+9rXui276KKLaG9vZ86cOfzXf/0XZ5555qD3993vfpclS5bwkY98hDPOOKPr8QcffJD169cze/ZsFixYwM6dO5k5cyZ33nkny5Yto7CwkK985SsAfOYzn2Hjxo0sXryYt956q1svPpD25+Tk8Oijj3LllVdSWFjItdde2/Wcyy67jIaGhqAN2wCIM4YfPhYuXKgul2vQ2/nuS7v4zZtutt/9URLiIvbzzJhB2717N9OnTw91M4yPy+Xitttu44033uhznd7+zUSkSFUX9rZ+xI7Ru3yFzCzkjTHDxfe//30eeeSRoI3Nd4rIFGxq7WDnoVoW2Pi8MVHr85//PHPnzu328+STT4a6WR/o9ttvx+128+EPfzio243IHv22shravWonShkTxR5++OFQNyFsRGSPvshtB2KNMaZTRAa9q6SaKbmpZKYkhLopxhgTchEX9F6vdp0oZYwxJsCgF5GLRGSPiOwVkdt7WT5RRF4VkXdEZIOI5Pst6xCRrb6fF4LZ+N7srWigrrndhm2MMcbnpEEvIrHAw8DFwAxglYjM6LHaA8DTqjoHuAe4z29Zk6rO9f1cFqR298nlO1FqYYFdUcqYSNRZKdIELpBZN4uBvaq6H0BE1gCXA7v81pkB3Oa7vR54PpiN7A+Xu5rsEQkUZFshM2P67c+3w/vbg7vNMbPh4u8Hd5thoL29nbi44TFxMZChmzyg1O9+me8xf9uAq3y3rwDSRCTbdz9JRFwi8qaIfKK3HYjIrb51XJ21mAeqyO1hgRUyM2bY+MY3vtHtClN333033/nOd1i+fDnz589n9uzZ/OlPfwpoWw0NDX0+r7e68r3VoC8pKWHWrFldz3vggQe4++67ATj33HO54447WLZsGQ8++CAvvvgiS5YsYd68eVxwwQUcPXq0qx0333wzs2fPZs6cOTz33HM8/vjj3HbbbV3bfeyxx7pKKgy5vuoXd/4AVwO/9Lt/I/C/PdYZB/wB2AI8iPNhkNG5zPf7NKAEmPxB+xtMPfryumad+I2X9Bcb9w54G8ZEm1DXo9+8ebOec845XfenT5+ubrdba2trVVW1oqJCJ0+erF6vV1WP137vTVtbW6/P66uufG816P3r46uq3n///frtb39bVVWXLVumn/3sZ7uWVVdXd7Xrscce06985Suqqvr1r39dv/zlL3dbr6GhQU877TRtbW1VVdWzzjpL33nnnf6+Xara/3r0gXzvKAPG+93PBw73+LA4DFwJICKpwFWqWuu3DFXdLyIbgHnAvv58GAXq+Px5G583ZriYN28e5eXlHD58mIqKCrKyshg7diy33XYbr7/+OjExMRw6dIijR48yZsyYD9yWqnLHHXec8Ly+6sq/9tprXfXfO2vQezyeD9yHfwGysrIyrr32Wo4cOUJra2tXfflXXnmFNWvWdK2XleVMDjn//PN56aWXmD59Om1tbV1XoRpqgQzdbAKmisgkEUkAVgLdZs+IyCgR6dzWN4EnfI9niUhi5zrAUrqP7QdVkbuahLgYZuWlD9UujDFDYMWKFaxdu5bf/e53rFy5ktWrV1NRUUFRURFbt25l9OjRvdah76mv52kf9d97419rHjhhv/6VKr/4xS/yhS98ge3bt/OLX/yia92+9nfLLbfw1FNP8eSTTwa1OuXJnDToVbUd+ALwMrAbeFZVd4rIPSLSOYvmXGCPiBQDo4F7fY9PB1wisg3nIO33VXXIgt4pZJZBYlx4XJDXGBOYlStXsmbNGtauXcuKFSuora0lNzeX+Ph41q9fj9vtDmg7fT2vr7ryvdWgHz16NOXl5VRVVdHS0sJLL730gfvLy3MOWf7qV7/qevzCCy/kpz/9adf9zm8JS5YsobS0lGeeeYZVq1YF+vYMWkDz6FV1napOU9XJqnqv77G7VPUF3+21qjrVt84tqtrie/yfqjpbVQt9vx8fqhfS3NbBjkO1zLf588YMOzNnzqS+vp68vDzGjh3L9ddfj8vlYuHChaxevbpb3fgP0tfz+qor31sN+vj4eO666y6WLFnCpZde+oH7vvvuu7n66qs5++yzu4aFAL71rW/h8XiYNWsWhYWFrF+/vmvZNddcw9KlS7uGc06FiKlHX17fzH+/tJtrF41n6ZRRJ3+CMQawevSn2qWXXsptt93G8uXLB7yN/tajj5gSCLlpSTy0ap6FvDEmLNXU1DBt2jSSk5MHFfIDMTxm+xtjjJ/t27d3zYXvlJiYyFtvvRWiFp1cZmYmxcXFIdm3Bb0xpl+zUsLB7Nmz2bp1a6ibERIDGW6PmKEbY8zAJCUlUVVVNaAAMaeWqlJVVUVSUlK/nmc9emOiXH5+PmVlZQy2/Ig5NZKSksjPzz/5in4s6I2JcvHx8V1ndJrIZEM3xhgT4SzojTEmwlnQG2NMhAu7M2NFpAIIrLBF70YBlUFqznBn70V39n50Z+/HcZHwXkxU1ZzeFoRd0A+WiLj6Og042th70Z29H93Z+3FcpL8XNnRjjDERzoLeGGMiXCQG/aOhbkAYsfeiO3s/urP347iIfi8ibozeGGNMd5HYozfGGOPHgt4YYyJcxAS9iFwkIntEZK+I3B7q9oSSiIwXkfUisltEdorIl0PdplATkVgR2SIifV8ANEqISKaIrBWRd33/R84KdZtCSURu8/2d7BCR34pI/0pDDgMREfQiEgs8DFwMzABWiciM0LYqpNqB/6eq04Ezgc9H+fsB8GWci9sbeBD4i6qeARQSxe+LiOQBXwIWquosIBZYGdpWBV9EBD2wGNirqvtVtRVYA1we4jaFjKoeUdXNvtv1OH/IeaFtVeiISD7wMeCXoW5LqIlIOnAO8DiAqraqak1oWxVycUCyiMQBKcDhELcn6CIl6POAUr/7ZURxsPkTkQJgHhC+11gbej8Bvg54Q92QMHAaUAE86RvK+qWIjAh1o0JFVQ8BDwAHgSNArar+NbStCr5ICfreroEW9fNGRSQVeA74T1WtC3V7QkFELgXKVbUo1G0JE3HAfOARVZ0HHAOi9piWiGThfPufBIwDRojIDaFtVfBFStCXAeP97ucTgV+/+kNE4nFCfrWq/iHU7QmhpcBlIlKCM6R3voj8JrRNCqkyoExVO7/hrcUJ/mh1AXBAVStUtQ34A/ChELcp6CIl6DcBU0Vkkogk4BxMeSHEbQoZca7y/DiwW1V/FOr2hJKqflNV81W1AOf/xWuqGnE9tkCp6vtAqYic7ntoObArhE0KtYPAmSKS4vu7WU4EHpyOiEsJqmq7iHwBeBnnqPkTqrozxM0KpaXAjcB2Ednqe+wOVV0XwjaZ8PFFYLWvU7QfuDnE7QkZVX1LRNYCm3Fmq20hAsshWAkEY4yJcJEydGOMMaYPFvTGGBPhLOiNMSbCWdAbY0yEs6A3xpgIZ0FvjDERzoLeGGMi3P8HCykFbRkxEMMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(history.history['accuracy'], label='train_accuracy')\n",
    "plt.plot(history.history['val_accuracy'], label='val_accuracy')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 8 - Predict\n",
    "\n",
    "**==> Compare prediction time vs training time.  Prediction is very quick!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predicting on 10,000 images\n",
      "prediction done in 323.48 ms\n"
     ]
    }
   ],
   "source": [
    "\n",
    "t1 = time.perf_counter()\n",
    "print (\"predicting on {:,} images\".format(len(test_images)))\n",
    "predictions = model.predict(test_images)\n",
    "t2 = time.perf_counter()\n",
    "print (\"prediction done in {:,.2f} ms\".format ((t2-t1)*1000))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "random index =  3236\n",
      "test_label[3236] = 7.  So the number is 7\n",
      "prediction of test_image[3236] = [ 0.000  0.000  0.014  0.000  0.000  0.000  0.000  0.983  0.004  0.000]\n",
      "max softmax output =  0.98250765\n",
      "index of max softmax output = 7.  So the prediction is same (7)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f06c45c8590>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAM30lEQVR4nO3dbawc5XnG8evCNiZ24sinrqnjOCWlIAKRauDUtHFVkaBSh6Y1fEiFK4LTQp2WUAUpH0rdD7EqVbJKSUKlKI0JFiaiRJEIwapQg+VaIlElh4Pj2qZObUoc8EttqKvYRMH45e6HM65OzNlnj3dmdza+/z9ptbtzz+zcWp3rzOw+u/s4IgTgwndR2w0AGAzCDiRB2IEkCDuQBGEHkpg+yJ1d7JlxiWYPcpdAKm/qJ3orTniyWq2w214m6SFJ0yR9NSLWlta/RLN1g2+qs0sABVtjc8daz6fxtqdJ+pKkj0q6WtIK21f3+ngA+qvOa/Ylkl6KiJcj4i1JX5e0vJm2ADStTtgXSnp1wv391bKfYXuV7THbYyd1osbuANRRJ+yTvQnwts/eRsS6iBiNiNEZmlljdwDqqBP2/ZIWTbj/XkkH67UDoF/qhP15SVfYfr/tiyXdLmljM20BaFrPQ28Rccr2vZK+rfGht/UR8WJjnQFoVK1x9oh4RtIzDfUCoI/4uCyQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ1JrFFYNx5J4PFeuL79zZsfbK6iuL207/1xd66gk/f2qF3fY+ScclnZZ0KiJGm2gKQPOaOLJ/OCJeb+BxAPQRr9mBJOqGPSQ9a/sF26smW8H2KttjtsdO6kTN3QHoVd3T+KURcdD2fEmbbP8gIp6buEJErJO0TpLmeCRq7g9Aj2od2SPiYHV9RNJTkpY00RSA5vUcdtuzbb/r7G1JN0va1VRjAJpV5zT+UklP2T77OP8UEf/SSFcXmL2PXVdeocuLm+995IFi/d0XXdKxtmf9s8Vt/2j7nxTrC+88UKyfPnasWMfw6DnsEfGypF9rsBcAfcTQG5AEYQeSIOxAEoQdSIKwA0nwFdcBiFPl/6l7fvcrXR6h89BaN1fOuLhY3/brjxfrVz/8yWL98nv2F+un/+dosY7B4cgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0k4YnA/HjPHI3GDbxrY/oaFr7+mWH/9ujnF+rV37yjWN+++qmNt04cfKm572fRZxfqZLt+/vWrL3cX6r97x/WIdzdoam3UsjnqyGkd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcfYL3E9vLc/bseVL/1isdxtn3/LT8nftH/jjOzrWLvoOY/BNY5wdAGEHsiDsQBKEHUiCsANJEHYgCcIOJMHvxl/g3vGt7xXrN/rPi/XHvvhgsX7zrGnF+l99oPM4/LzvFDdFw7oe2W2vt33E9q4Jy0Zsb7K9t7qe2982AdQ1ldP4RyUtO2fZ/ZI2R8QVkjZX9wEMsa5hj4jnJJ07h89ySRuq2xsk3dpwXwAa1usbdJdGxCFJqq7nd1rR9irbY7bHTupEj7sDUFff342PiHURMRoRozM0s9+7A9BBr2E/bHuBJFXXR5prCUA/9Br2jZJWVrdXSnq6mXYA9EvXcXbbT0i6UdI82/slfU7SWknfsH2XpFckfbyfTaJ/Zj21tVj/s3tvL9b/+ary//n//eCZjrV5xS3RtK5hj4gVHUr8CgXwc4SPywJJEHYgCcIOJEHYgSQIO5AEX3FF0fQ7Ow+dSdIXv31lsf7ox77Ssbb2H24rbnv6pR8W6zg/HNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2VF06sDBYv3VN0eK9fvm7ulY++EdC4rbvm8N4+xN4sgOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSfB9dhT5+muK9Y+8uzxl80Vyx1p0LqEPuh7Zba+3fcT2rgnL1tg+YHt7dbmlv20CqGsqp/GPSlo2yfIvRMTi6vJMs20BaFrXsEfEc5KODqAXAH1U5w26e23vqE7z53ZayfYq22O2x07qRI3dAaij17B/WdLlkhZLOiTpwU4rRsS6iBiNiNEZmtnj7gDU1VPYI+JwRJyOiDOSHpa0pNm2ADStp7DbnvgbwLdJ2tVpXQDDoes4u+0nJN0oaZ7t/ZI+J+lG24slhaR9kj7Vxx7Roteun1Os/96sHxfrpdndHT00hJ51DXtErJhk8SN96AVAH/FxWSAJwg4kQdiBJAg7kARhB5LgK64oGr17e9stoCEc2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZUfQHI9+vtf2JONmxNv0ntR4a54kjO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kwTg7annyjXnF+t9+dbIfJx73ngf+rel2UMCRHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSYJw9uR/9zW8W68vesa1Yn+Y3i/UHfsy8zMOi65Hd9iLbW2zvtv2i7c9Uy0dsb7K9t7qe2/92AfRqKqfxpyR9NiI+IOk3JH3a9tWS7pe0OSKukLS5ug9gSHUNe0Qcioht1e3jknZLWihpuaQN1WobJN3aryYB1Hdeb9DZvkzStZK2Sro0Ig5J4/8QJM3vsM0q22O2x07qRL1uAfRsymG3/U5JT0q6LyKOTXW7iFgXEaMRMTpDM3vpEUADphR22zM0HvTHI+Kb1eLDthdU9QWSjvSnRQBN6Dr0ZtuSHpG0OyI+P6G0UdJKSWur66f70iFadUblobPVhxcX6/Of2FV4bAzSVMbZl0r6hKSdts9O1r1a4yH/hu27JL0i6eP9aRFAE7qGPSK+K8kdyjc12w6AfuHjskAShB1IgrADSRB2IAnCDiTBV1yTW7via7W2f/aVq4r1+cd/UOvx0RyO7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOPsF7j/vu9Dxfrvzyr/VPRfHFxarL/nnvKPFp0qVjFIHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2S9wx645WWv75w+/r1gfObCn1uNjcDiyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjijPv217kaTHJP2SxqfUXhcRD9leI+lPJb1Wrbo6Ip4pPdYcj8QNZuJXoF+2xmYdi6OTzro8lQ/VnJL02YjYZvtdkl6wvamqfSEi/r6pRgH0z1TmZz8k6VB1+7jt3ZIW9rsxAM06r9fsti+TdK2krdWie23vsL3e9twO26yyPWZ77KRO1GoWQO+mHHbb75T0pKT7IuKYpC9LulzSYo0f+R+cbLuIWBcRoxExOkMzG2gZQC+mFHbbMzQe9Mcj4puSFBGHI+J0RJyR9LCkJf1rE0BdXcNu25IekbQ7Ij4/YfmCCavdJmlX8+0BaMpU3o1fKukTknba3l4tWy1phe3FkkLSPkmf6kuHABoxlXfjvytpsnG74pg6gOHCJ+iAJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJdP0p6UZ3Zr8m6UcTFs2T9PrAGjg/w9rbsPYl0VuvmuztlyPiFycrDDTsb9u5PRYRo601UDCsvQ1rXxK99WpQvXEaDyRB2IEk2g77upb3XzKsvQ1rXxK99WogvbX6mh3A4LR9ZAcwIIQdSKKVsNteZvs/bb9k+/42eujE9j7bO21vtz3Wci/rbR+xvWvCshHbm2zvra4nnWOvpd7W2D5QPXfbbd/SUm+LbG+xvdv2i7Y/Uy1v9bkr9DWQ523gr9ltT5O0R9LvSNov6XlJKyLiPwbaSAe290kajYjWP4Bh+7clvSHpsYj4YLXs7yQdjYi11T/KuRHxl0PS2xpJb7Q9jXc1W9GCidOMS7pV0ifV4nNX6OsPNYDnrY0j+xJJL0XEyxHxlqSvS1reQh9DLyKek3T0nMXLJW2obm/Q+B/LwHXobShExKGI2FbdPi7p7DTjrT53hb4Goo2wL5T06oT7+zVc872HpGdtv2B7VdvNTOLSiDgkjf/xSJrfcj/n6jqN9yCdM8340Dx3vUx/XlcbYZ9sKqlhGv9bGhHXSfqopE9Xp6uYmilN4z0ok0wzPhR6nf68rjbCvl/Sogn33yvpYAt9TCoiDlbXRyQ9peGbivrw2Rl0q+sjLffz/4ZpGu/JphnXEDx3bU5/3kbYn5d0he33275Y0u2SNrbQx9vYnl29cSLbsyXdrOGbinqjpJXV7ZWSnm6xl58xLNN4d5pmXC0/d61Pfx4RA79IukXj78j/l6S/bqOHDn39iqR/ry4vtt2bpCc0flp3UuNnRHdJ+gVJmyXtra5Hhqi3r0naKWmHxoO1oKXefkvjLw13SNpeXW5p+7kr9DWQ542PywJJ8Ak6IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUji/wB2W+AteOC0yQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "## Print a sample prediction\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import numpy as np\n",
    "from pprint import pprint\n",
    "\n",
    "np.set_printoptions(formatter={'float': '{: 0.3f}'.format})\n",
    "\n",
    "index = random.randint(0, len(test_images))\n",
    "\n",
    "print (\"random index = \", index)\n",
    "print (\"test_label[{}] = {}.  So the number is {}\".format(index, test_labels[index], test_labels[index]))\n",
    "print (\"prediction of test_image[{}] = {}\".format(index, predictions[index]))\n",
    "print ('max softmax output = ', np.amax(predictions[index]))\n",
    "print ('index of max softmax output = {}.  So the prediction is same ({})'.format(np.argmax(predictions[index]), np.argmax(predictions[index])))\n",
    "\n",
    "plt.imshow(test_images2[index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 9 : Evaluate the Model "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.1 - Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "model metrics :  ['loss', 'accuracy']\n",
      "Metric : loss = 0.043\n",
      "Metric : accuracy = 0.987\n"
     ]
    }
   ],
   "source": [
    "metric_names = model.metrics_names\n",
    "print (\"model metrics : \" , metric_names)\n",
    "\n",
    "metrics = model.evaluate(test_images, test_labels, verbose=0)\n",
    "\n",
    "for idx, metric in enumerate(metric_names):\n",
    "    print (\"Metric : {} = {:,.3f}\".format (metric_names[idx], metrics[idx]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.2 - Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predictions shape :  (10000, 10)\n",
      "prediction 0 :  [ 0.000  0.000  0.000  0.000  0.000  0.000  0.000  1.000  0.000  0.000]\n",
      "prediction 1 :  [ 0.000  0.000  1.000  0.000  0.000  0.000  0.000  0.000  0.000  0.000]\n"
     ]
    }
   ],
   "source": [
    "## our predictions is an array of arrays\n",
    "print('predictions shape : ', predictions.shape)\n",
    "print ('prediction 0 : ' , predictions[0])\n",
    "print ('prediction 1 : ' , predictions[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction2 0 :  7\n",
      "prediction2 1 :  2\n"
     ]
    }
   ],
   "source": [
    "## We need to find the final output (max of softmax probabilities for each prediction)\n",
    "predictions2 = [ np.argmax(p) for p in predictions]\n",
    "print ('prediction2 0 : ' , predictions2[0])\n",
    "print ('prediction2 1 : ' , predictions2[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 971,    0,    3,    0,    1,    0,    3,    1,    1,    0],\n",
       "       [   0, 1112,    0,    3,    0,    0,   15,    1,    4,    0],\n",
       "       [   0,    1, 1014,   14,    0,    0,    1,    1,    1,    0],\n",
       "       [   0,    0,    1, 1006,    0,    2,    0,    0,    1,    0],\n",
       "       [   0,    0,    0,    0,  972,    0,    2,    0,    4,    4],\n",
       "       [   0,    0,    0,    6,    0,  882,    2,    1,    0,    1],\n",
       "       [   2,    1,    0,    0,    1,    2,  951,    0,    1,    0],\n",
       "       [   0,    2,   11,    4,    0,    0,    0, 1008,    0,    3],\n",
       "       [   0,    0,    0,    6,    0,    1,    0,    0,  967,    0],\n",
       "       [   1,    0,    0,    3,    2,    6,    0,    0,    7,  990]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import seaborn as sns\n",
    "\n",
    "cm = confusion_matrix(test_labels, predictions2, labels = [0,1,2,3,4,5,6,7,8,9])\n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcsAAAFlCAYAAACEFMPZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeXxU1f3/8ddJIMgaQEhYQtmRRaqyqIAgsgUIEAQUEXCHulRoKX5BqbjV1mq/Wlxqv/kpCgqCohUqIirIIqAsiggigsgaGELZCUuW8/sjQxokySCz3MPwfj4e95GZO8t5z5k7+cw5c+eOsdYiIiIiRYvxOoCIiIjrVCxFREQCULEUEREJQMVSREQkABVLERGRAFQsRUREAigR7gamVkxw5rspN6dv9DqCiIi3ysSbcN313aZCUP/v/2kPhS1bsMJeLEVE5MIQzVOV0fzYREREQkIjSxERCYkY4+wsatBULEVEJCSieapSxVJEREIiJnoHllH9RkBERCQkNLIUEZGQiObRl4qliIiEhHbwERERCUAjSxERkQC0g4+IiMgFzJliecndw+i5dCE9ly3iknuGA9BuYho9Fs+nx+L59Fmzkh6L5wMQV6kSnf/9Hjfs+IlWT/8lojkXLVlGct8BdO3Tj7SJkyLa9s+dOHGCAUNuo8+NN5PSfyDPv5zmaR5wq38efPQJ2nRKpteAmzzNcYpLfePatuPac+VaHpe2neLEBLm4zIl88U0aU/+WIczt3J0511xHzeRulK9XlyV3DGdO+07Mad+J7bNms/3fswHIOXGCNU/+la8ffjSiOXNycnj8qad55cUJzH53Oh98NJdNP26OaIaC4uLimJT2D2a9PZX3p01h8dJlrF7zrWd5XOuffr1TeOWlCZ61X5BrfePatuPScwVu5XFt2ymOMSaoxWUBi6UxprExZowx5nljzAT/6SahDFGhUUP2rlxFzrFj2Jwc9ixZSlKvlNOu86u+fdg64z0AcjIzyfjiS3JOHA9ljIDWrF1H7VpJ1EqqSVzJkqQkd2PegkURzVCQMYayZcoAkJ2dTXZ2tqcbnGv907plC+LjK3jWfkGu9Y1r245LzxW4lce1bac4F+zI0hgzBpgGGGA5sMJ/+i1jzNhQhTi4/nsS2rYhrlIlYkuXpkbXLpRJqpF/edW2V3M8I4PDm38KVZPnxLcng2qJifnnExMT8GVkeJgo711n6sDBtO2cTNurr+Sy5pd6lsXF/nGFi33j0rYjRXNx2ylKjAlucVmgvWHvBJpZa7MKrjTGPAusA54q7EbGmOHAcIA7S5ejU1zpYhs59MNGvpvwAp3ef4fso0fZv3YdNjs7//I6/fux9d1/BXww4WY586favH5+Y2NjmTl9CocOH+a+Uf/DD5t+pFGD+p5kcbF/XOFi37i07UjRXNx2LkSBRr65QI1C1lf3X1Yoa22atbaVtbZVoEJ5yuY3pvLRtV34tGcqJ/fv5/CPeaNIExtLUu8Utr73/lndTzhVS0hgt8+Xf97n20NC1aoeJvqvCuXLc1WrFixeusyzDC73j9dc7hsXth0pmsvbzs9dsNOwwO+AecaYOcaYNP/yETAPGBnKIKWqVAGgTFJNknqnsMX/+WS1jh04tHEjx9J3hbK5c9K8WVO2bNvO9p07OZmVxey5H9OpY3vP8uzbt59Dhw8DcPz4cZZ+uZx6dWp7lse1/nGJa33j2rYjRXNt2ylOjDFBLS4rdhrWWvuRMaYRcCVQk7zR/w5ghbU2J5RB2k+eSKnKlcjNzmbl6LFkHTwIQO3+17N1xplTsH3WrKRk+fLElIwjKaUH8/vdyKENP4Qy0hlKlCjB+DEPcNe9I8jJzaV/am8a1vdu2mrP3r2MHf8YObm52NxcunftwnUdvHsRudY/o8b+keWrVrH/wAE6JPfi/ruHccP1qZ5kca1vXNt2XHquXMvj2rZTHNdHh8Ew1p45Hx5KUysmhLeBX+Dm9I1eRxAR8VaZ+LAN4Z68qHJQ/+/HHd/n7PBSh7sTEZGQcH2P1mCoWIqISEhE8zSsiqWIiIRETBR/qUXFUkREQiKap2GjedQsIiISEhpZiohISETz6EvFUkREQiKap2FVLEVEJCSieQefaB41i4hIBIX7V0eMMRONMXuMMWsLrKtsjPnEGLPR/7eSf73x/7TkJmPMGmNMiwK3udV//Y3GmFvP6rH98u4QERHxxOtA95+tGwvMs9Y2JO+45ad+PrIH0NC/DAdehrziCjwCXEXeoVwfOVVgi6NiKSIiIRHuXx2x1i4C9v1sdSowyX96EtC3wPrJNs8XQEVjTHUgGfjEWrvPWrsf+IQzC/AZwv6ZpUvHY727bC2vI5zmn0e3ex1Bzle5If0dg+DFxHqdQBwQ7A4+BX8L2S/NWpsW4GaJ1tpdANbaXcaYBP/6mkDBf7I7/OuKWl8s7eAjIiIhEewOPv7CGKg4nq3Cwthi1hdL07AiInI+8/mnV/H/3eNfvwMoOJ2YBKQXs75YKpYiIhIS4d4btgizgFN7tN4KzCyw/hb/XrFXAwf907VzgW7GmEr+HXu6+dcVS9OwIiISEuH+lqUx5i2gI1DFGLODvL1anwLeNsbcCWwDbvBf/UOgJ7AJyARuB7DW7jPGPAGs8F/vcWvtz3caOoOKpYiIhES4j+BjrR1UxEWdC7muBe4r4n4mAhN/SdsqliIiEhI6go+IiMgFTCNLEREJCR1IXUREJIBonqpUsRQRkZCI4oGliqWIiIRGjInecnlejJoXLVlGct8BdO3Tj7SJkwLf4BwMffUlnvb9yMPffpG/rsWAvoxf+yX/yDnAr1pekb++bOXK/H7+B/z9cDo3vfC3/PUlS5fmvg/e4dH1Kxm/9kv6/uXRsGQtKBJ9c7ZOnDjBgCG30efGm0npP5DnXw7VUavOjUt940KeBx/7E2269KTXjYPz173wf6/QvntvUgfdQuqgW1j4+dKI5wJ48NEnaNMpmV4DbvKk/cLk5OTQ96Yh/GbE772O4vm2I+dBsczJyeHxp57mlRcnMPvd6Xzw0Vw2/bg55O0se30KL3Tvd9q69LXf8X/9BrNp0ZLT1mcdP86sh//Eu6P/eMb9fPK353m0SSuevOIa6re7mmbdu4Y86ymR6puzFRcXx6S0fzDr7am8P20Ki5cuY/Wabz3J4lrfuJCnX+8UXnnhuTPW33bzTcx8azIz35rMtde0jWimU/r1TuGVlyZ40nZRJk+dRv26dbyO4cS2c7ZMkIvLnC+Wa9auo3atJGol1SSuZElSkrsxb8GikLezafFSMvftP23d7u9/wPfDpjOuezIzkx+XfEH28eOnrc86dowfFiwGICcri+1ffUOlpBohz3pKpPrmbBljKFumDADZ2dlkZ2djPJqWca1vXMjTusUVxMdXiGibZ6t1yxZOZdvt87Hg8yUMuD7V6yhObDtnS8WyEMaY20MZpCi+PRlUS0zMP5+YmIAvIyMSTQeldHw8zXt35/t5C8PWhot9k5OTQ+rAwbTtnEzbq6/ksuaXepLDtb5xLU9BU96eQe+BQ3jwsT9x8NAhr+M44c/PPMcDI+8nJsb78YTL287PqVgW7rGQpSiGLeSXU5zv1NhY7nxrIp89/3/s/WlL2NpxsW9iY2OZOX0KC+d+wJq13/HDph89yeFa37iW55RBA/rxycwZzHxrMglVqvDUc897Hclzny1aTOXKlbi0aROvowDubjsXmmKLpTFmTRHLt0BiMbcbboxZaYxZmTbx9aACVktIYLfPl3/e59tDQtWqQd1nuA1Oe549G39k/oR/hLUdl/umQvnyXNWqBYuXLvOkfdf6xrU8p1S5uDKxsbHExMRww/WpfLtuvdeRPPfV6jXMX7iYTj1TGTV2HF+sWMnoceM9y+PqtlMYY0xQi8sCjSwTgVuA3oUs/ynqRtbaNGttK2ttq+F33BZUwObNmrJl23a279zJyawsZs/9mE4d2wd1n+HU54mHKR1fgXd+NybsbbnWN/v27efQ4cMAHD9+nKVfLqdendqeZHGtb1zLc8qejL35pz/9bAEN69fzMI0b/jDiPhbN/YD5H87k2aee5OrWrfjbk497lsfVbacw0TwNG+h7lh8A5ay1q39+gTFmQVgS/UyJEiUYP+YB7rp3BDm5ufRP7U3D+vVD3s6dUyfSqOM1lKtyMX/Zvp5/P/JnMvftZ+ALz1CuahV+O/sdtq/+lhe6Xw/Akz99y0UVKhAbV5LL+qbwfLe+HDt0mJ5/fIBd6zfw0Fd5O/oseDGNJa9ODnleiFzfnK09e/cydvxj5OTmYnNz6d61C9d18OZF7VrfuJBn1EPjWb7yK/YfOECHHn24/zd3sXzV13y/4Qcwhpo1qvP4Q+F/k1dotrF/ZPmqVXnZkntx/93DuMGBnWtc4MK2c7a8/4Q3fEzer5iEUebBMDdw9u4uWyvwlSLon0e3ex1Bzle5OV4nOF1MrNcJ5GyViQ/bIG7mxdWC+n+f+p/dzg4wo/mNgIiISEjocHciIhISxvlPHs+diqWIiIRE9JZKFUsREQkRFUsREZEAovnHn7WDj4iISAAaWYqISEhoBx8REZEAordUqliKiEiIOH5416DoM0sREZEANLIUEZGQiOKB5YVVLF07FusLVd06GPL9vh+8jnA6HW+0aOobcVBMFJfLC6pYiohI+ERvqVSxFBGRENEOPiIiIhcwjSxFRCQkonhgqWIpIiKhoSP4iIiIBBDNB1JXsRQRkZCI4lqpHXxEREQC0chSRERCIppHliqWIiISEtrBR0REJAAdlEBEROQCdl4Uy0VLlpHcdwBd+/QjbeIkT7M8+OgTtOmUTK8BN4W1nc4vT+DOLeu5ecXi/HWlKlUk9d8zGPrNclL/PYNSFeMBqNSoAQPmz+HefTu5YuR9Z9yXiYnhpqXz6TVjashzPvjYn2jTpSe9bhx8xmWvTp7CJS3bsG//gZC3ezZc2m5cyxOp7fhsKU/xXNp2ihMT5OIy1/ORk5PD4089zSsvTmD2u9P54KO5bPpxs2d5+vVO4ZWXJoS9nfVvTmNW34GnrWv5h5HsWLCINy67kh0LFtHyDyMBOL7/AItGP8RXE14q9L4uu+837NuwMSw5+/VO4ZUXnjtj/a7dPpZ+uYIa1aqFpd1AXNtuXMsTqe34bClP0VzbdopjglxcFrBYGmMaG2M6G2PK/Wx99/DF+q81a9dRu1YStZJqEleyJCnJ3Zi3YFEkmi5U65YtiI+vEPZ20pcs4/i+/aetq5fSg/VTpgOwfsp06vXqCcCxjL3s+eprcrOyz7ifsjWqU6d7V757/c2w5Gzd4opC++Mvz07ggZH3efYZhmvbjWt5IrUdny3lKZpr205xjDFBLS4rtlgaY0YAM4H7gbXGmNQCF/85nMFO8e3JoFpiYv75xMQEfBkZkWjaOWUSqpK52wdA5m4fpatWCXibDk8/yZJxj2Fzc8MdL9+8hYtJqFqVxo0aRqzNn3Ntu3Etj5w/zqdt50IeWQ4DWlpr+wIdgYeNMSP9lxX52Iwxw40xK40xK9Mmvh5UQIs98/6DuscLR53u3cjM2EvG6m8i1uaxY8f556uvM/LuYRFrszCubTeu5ZHzh7YdNwT66kistfYIgLV2izGmIzDDGFObYp4va20akAZA5sEzn+lfoFpCArt9vvzzPt8eEqpWDeYuz1uZezIoUy2RzN0+ylRL5FjG3mKvX73NldRL6U6d5C7EXlSKuPLl6frqy3xy5z1hy7htxw52pO8iddBQAHbvyaDf4Nt4Z/KrVK1ycdja/TnXthvX8sj543zadqK5iAcaWe42xlx+6oy/cPYCqgDNwxnslObNmrJl23a279zJyawsZs/9mE4d20eiaef89OFHNBmct9NPk8ED2Tx7TrHXX/bIn3it0a+Z1LQFc28dzo6Fn4e1UAJc0rAByz79kPkf/Iv5H/yLaglVeW/K6xEtlODeduNaHjl/nE/bTjR/ZhloZHkLcNpeI9babOAWY8z/hS1VASVKlGD8mAe4694R5OTm0j+1Nw3r149E04UaNfaPLF+1iv0HDtAhuRf33z2MG65PDXzDXyj59TRqtm/HRRdX5vYf1vDln/7Kqv+dQPc3XqXpLUM4vGMHc4bcAUCZxAQGLv6UuPLlsbm5XH7fb3izZVuyDh8Jea6fG/XQeJav/CqvP3r04f7f3MUNffuEvd1AXNtuXMsTqe1YeYLn2rZTnGj+1RFjbVCzpIEFOQ0bzV6o6tYGf7/vB68jnC4m1usEItGnTHzYStrXNWsH9f/+ip1bnS23OtydiIiEhInioaWKpYiIhITjHzsGxfkj+IiIyPnBmOCWs2vD/N4Ys84Ys9YY85Yx5iJjTF1jzJfGmI3GmOnGmDj/dUv5z2/yX17nXB+biqWIiIREuPeGNcbUBEYAray1lwKxwE3AX4HnrLUNgf3Anf6b3Anst9Y2AJ7zX++cqFiKiMj5pARQ2hhTAigD7AI6ATP8l08C+vpPp/rP47+8sznH76ioWIqISEgEOw1b8Ohv/mV4wfu31u4E/gZsI69IHgRWAQf8X2sE2AHU9J+uCWz33zbbf/1z+tK3dvAREZGQCPbAAqcd/a3w+69E3mixLnAAeAfoUdhdnbpJMZf9IhpZiohISERgB58uwE/W2gxrbRbwHtAWqOiflgVIAtL9p3cAtfKymRJAPLDvXB6biqWIiIREjDFBLWdhG3C1MaaM/7PHzsB3wGfAAP91biXv17IAZvnP4798vj3HI/GoWIqIyHnBWvsleTvqfAV8S14NSwPGAKOMMZvI+0zyVf9NXgUu9q8fBYw917b1maWIiIREJA5KYK19BHjkZ6s3A1cWct3jwA2haFfF0kP3Z/zodYTTvJzg1rFq79njVv+ISPFc/+WQYKhYiohISJgo/mAvih+aiIhIaGhkKSIiIaFpWBERkQCiuFaqWIqISGhoZCkiIhJAFNdK7eAjIiISiEaWIiISEmd5yLrzkoqliIiERBTXShVLEREJDe3gIyIiEkAU18rzYwefRUuWkdx3AF379CNt4iRl8Xvw0Sdo0ymZXgNuCntbHf8xgdt+Ws/A5Yvz15WqVJFes2YwaPVyes2aQVzF+PzL2j3zZ27+Zjk3frGQKpf9On99uaSa9Jr5DjetWsrAlUso/6taYcvs0nO1a7ePocPuoUe/G0npP5BJU6d5mgfc6h/X8kTytXU2XOqbC5XzxTInJ4fHn3qaV16cwOx3p/PBR3PZ9OPmCz4LQL/eKbzy0oSItLVhyjQ+6DvwtHVXjBrJzgWLeOvyK9m5YBEtRo0E4FfdulCxfj2mXnYlC+8fRYe/P5N/m07/7x+s/vuLTGvZlnev7caxjL1hyevacxUbG8vYUSOZ897bTJ88kanT3/E0j2v941qeSL62AnGtb4oTgR9/9ozzxXLN2nXUrpVEraSaxJUsSUpyN+YtWHTBZwFo3bIF8fEVItLWriXLOLF//2nr6qb0YMOU6QBsmDKdur16AlCnVw82vPU2AL4VqygVH0+ZxEQqNW5ETGwsOz5bCED20aNkHzsWlryuPVcJVavQrEljAMqVLUu9unXxZWR4lse1/nEtTyRfW4G41jfFMTEmqMVlAYulMeZKY0xr/+mmxphRxpie4Y+Wx7cng2qJifnnExMTPPsn41IWF5ROqEqmzwdAps9H6apVAChbvTpHduzMv96R9HTK1qhOfIP6nDh4iOSprzNgyXza/OlRTEx43q+5/FztSE9n/YYNXHZpM88yuNY/ruVxyfnUNxfsyNIY8wjwPPCyMeYvwItAOWCsMWZcBPJhsWfmikTDhXApi8sK2yPOWktMiRJUb3s1Sx96hHc7dKVC3dpcMmRQWDK4+lwdzcxkxOixPDR6FOXKlfMsh2v941oel6hv3BDobf0AoB3QAbgP6GutfRxIBgYWdSNjzHBjzEpjzMq0ia8HFbBaQgK7/aMXAJ9vDwlVqwZ1n9GQxQXH9mRQxv+Ot0xiYv7nj0fS0ymXVDP/euVq1CBz126O7kxn75pvObxlKzYnh5/+/SFVL/91ofcdLBefq6ysbEaMHkPvHsl063ydp1lc6x/X8rjkfOqbGGOCWlwWqFhmW2tzrLWZwI/W2kMA1tpjQG5RN7LWpllrW1lrWw2/47agAjZv1pQt27azfedOTmZlMXvux3Tq2D6o+4yGLC7Y8uFHXDI47z3TJYMH8tPsOXnrZ3/EJYNuBCCxdUtOHDpEps/HnlVfU6piPBdVuRiAmte2Z9/3G8KSzbXnylrLuMeeoF7dutw+dLBnOU5xrX9cy+OS86lvonkaNtD3LE8aY8r4i2XLUyuNMfEUUyxDqUSJEowf8wB33TuCnNxc+qf2pmH9+pFo2uksAKPG/pHlq1ax/8ABOiT34v67h3HD9alhaavLa2nUaN+Oiy6uzNANa1jx5F/56tkJdJv8Ko1vGcKRHTv4eOgdAGyb+wm1k7tw85oVZB87xmd3jwDA5uay7KFH6PPBe2AMGV9/w/rX3ghLXteeq1Wrv2Hm7Dk0atiA1IF5xXLUb+/l2vbtPMnjWv+4lieSr61AXOub4kTzQQmMtWfOh+dfaEwpa+2JQtZXAapba78N2ELmwaIbEKe8nODWC/CePT96HUEk+pSJD1tF23t1s6D+31f5Yp2z1bbYkWVhhdK/fi8Qni/IiYiIOEaHuxMRkZCI5mlYFUsREQmJKK6VKpYiIhIaGlmKiIgEYJw/gOq5i+KHJiIiEhoaWYqISEhoGlZERCQQx385JBgqliIiEhpRPLLUZ5YiIiIBaGQpIiIhoc8sRUREAtFnlnIhcO3A5VNrNPQ6Qr6b0zd6HUHEfRpZioiIFM9E8chSO/iIiIgEoJGliIiEhqZhRUREihfN07AqliIiEhoaWYqIiAQQxSNL7eAjIiISgEaWIiISEjqCj4iISCBRPA2rYikiIqERxSNLfWYpIiISwHlRLBctWUZy3wF07dOPtImTlEV5TnPJ3cPouXQhPZct4pJ7hgPQbmIaPRbPp8fi+fRZs5Iei+cDUK3jtXRf8Ak9lyyg+4JPSOxwTUQyglvP1a7dPoYOu4ce/W4kpf9AJk2d5mkecKt/AHJycuh70xB+M+L3XkcB3MtTGBMT3OIy56dhc3JyePypp3nt5RdJTExgwOBb6XRtexrUr3dBZ1GePPFNGlP/liHM7dyd3JMnue7d6aTP/YQldwzPv84Vf3qMrEOHADix7z8svGkIx3b7iG/SmOvenc77TS8LW75TXHuuYmNjGTtqJM2aNObI0aP0v/kW2l115QW17QQyeeo06tetw5GjRz3LUJBreQqladj/MsZMDkeQoqxZu47atZKolVSTuJIlSUnuxrwFiyIZwcksypOnQqOG7F25ipxjx7A5OexZspSkXimnXedXffuwdcZ7AOxfs5Zju30AHFz/PbEXlSImLi6sGcG95yqhahWaNWkMQLmyZalXty6+jAzP8rjWP7t9PhZ8voQB16d6lqEg1/IUxcSYoBaXFVssjTGzfrb8G+h36nwkAvr2ZFAtMTH/fGJigmcvapeyKE+eg+u/J6FtG+IqVSK2dGlqdO1CmaQa+ZdXbXs1xzMyOLz5pzNuW6tPL/avWUvuyZNhzQjuPVcF7UhPZ/2GDVx2aTPPMrjWP39+5jkeGHk/MTFuzA26lqdIxgS3OCxQzycBh4Bngf/1L4cLnC6UMWa4MWalMWZl2sTXgwposWfef1D3eO5cygLKA3Doh418N+EFOr3/Dte9O439a9dhs7PzL6/Tvx9b3/3XGbeLb3wJlz82nuW/Gx3mhHlce65OOZqZyYjRY3lo9CjKlSvnWQ6X+uezRYupXLkSlzZt4lGC07mWx2vGmIrGmBnGmO+NMeuNMW2MMZWNMZ8YYzb6/1byX9cYY543xmwyxqwxxrQ413YDfWbZChgJjAMesNauNsYcs9YuLO5G1to0IA2AzINnvgp+gWoJCez2+fLP+3x7SKhaNZi7jIosyvNfm9+YyuY3pgJw2cMPkZm+CwATG0tS7xQ+6tjltOuXrlGd9m++zrK7f8uRLVvCng/ce64AsrKyGTF6DL17JNOt83WeZnGpf75avYb5Cxez6POlnDh5giNHjzJ63Hj+9uTjyhNIZKZSJwAfWWsHGGPigDLAQ8A8a+1TxpixwFhgDNADaOhfrgJe9v/9xYodWVprc621zwG3A+OMMS8S4Z2CmjdrypZt29m+cycns7KYPfdjOnVsH8kITmZRnv8qVaUKAGWSapLUO4Ut/s8nq3XswKGNGznmL54AJeMr0PHtqXzz+JPs/XJ52LOd4tpzZa1l3GNPUK9uXW4fOtizHKe41D9/GHEfi+Z+wPwPZ/LsU09ydetWnhYm1/IUxxgT1HIW918B6AC8CmCtPWmtPQCkAqd2oZ4E9PWfTgUm2zxfABWNMdXP5bGdVeGz1u4AbjDGpJA3LRsxJUqUYPyYB7jr3hHk5ObSP7U3DevXj2QEJ7Moz3+1nzyRUpUrkZudzcrRY8k6eBCA2v2vZ+uM06dgGw27k/J163DpA6O49IFRAMy//kZO7N0b1oyuPVerVn/DzNlzaNSwAakD84rlqN/ey7Xt23mSx7X+kXMU/pFlPSADeM0YcxmwirzZz0Rr7S4Aa+0uY0yC//o1ge0Fbr/Dv24Xv5CxNqhZ0sCCnIaVC9fUGg29jpDv5vSNXkcQCY0y8WGraMeHdgrq/33pNz/7DTC8wKo0/8d6ABhjWgFfAO2stV8aYyaQN4C731pbscD19ltrKxljZgN/sdZ+7l8/D/gfa+2qX5rN+e9ZiojIheG0/V0KtwPYYa390n9+BnmfT/qMMdX9o8rqwJ4C169V4PZJQPq5ZHN8P2QRETlfhPszS2vtbmC7MeYS/6rOwHfALOBW/7pbgZn+07OAW/x7xV4NHDw1XftLaWQpIiKhEZm9Ye8Hpvj3hN1M3g6oMcDbxpg7gW3ADf7rfgj0BDYBmf7rnhMVSxERCYlI/J6ltXY1eV9r/LnOhVzXAveFol1Nw4qIiASgkaWIiISG48d3DYaKpYiIhIbjx3cNhoqliIiEhOu/HBIMFUsREQmNKB5ZagcfERGRADSyFBGR0NA0rCRK6PMAACAASURBVFwQwn2c4F/IpeOxzq/T1OsIp+m05TuvI4icIRLfs/SKiqWIiISGRpYiIiIBRPHIUjv4iIiIBKCRpYiIhEYUjyxVLEVEJDRULEVERAKIid5P9qL3kYmIiISIRpYiIhIamoYVEREJQMVSREQkABVLERGRALSDj4iIyIXrvCiWi5YsI7nvALr26UfaxEnK4nCeQ4cPM+KBsXTvdyM9+g3k62++9SyLV31T6zfDuHLRfK5cOI9m/3yJmFKlqNT+Glp/+hGt539Mi3//i9J16+Rd9+7hXLX4M65c8AmXz5jORUk1I5Jx124fQ4fdQ49+N5LSfyCTpk6LSLtFefDRJ2jTKZleA27yNMcpLr2uXOubYhkT3OIw54tlTk4Ojz/1NK+8OIHZ707ng4/msunHzRd8FhfzADz5zLO0b9uGj957m5nT36R+vTqe5PCqb+KqVSPprjtY2a0ny6/tDLGxJPRN5ZKn/8K6e37Lik7d8L33PnV+PxKAw9+uZUW3Hizv2JWMD2ZTf/wfw54RIDY2lrGjRjLnvbeZPnkiU6e/4+m20693Cq+8NMGz9gty7XXlUt8EpGKZxxhzjTFmlDGmW7gC/dyateuoXSuJWkk1iStZkpTkbsxbsChSzTubxcU8R44cYcVXXzOgbx8A4kqWpEL58p5k8bJvTIkSxFx0ESY2ltjSpTnp2421lhL+vihRvjwndvsAOLBkKbnHjgNwcOUqStWoHpGMCVWr0KxJYwDKlS1Lvbp18WVkRKTtwrRu2YL4+AqetV+Qa68rl/omoAu1WBpjlhc4PQx4ESgPPGKMGRvmbAD49mRQLTEx/3xiYoJnL2qXsriYZ/vOdCpXqsSDjz5B30FDGff4k2QeO+ZJFq/65uTu3Wz7xz9p+/Vy2n37NdmHD7FvwSK+//1oLpv6Bm1Xr6TaDf3Z+vyLZ9y2xuBB7Jv3Wdgz/tyO9HTWb9jAZZc2i3jbLnLtdSVuCDSyLFng9HCgq7X2MaAbMLioGxljhhtjVhpjVqZNfD2ogJYzf5DYq/cfLmUB9/Jk5+Tw3fcbGDSgH++/9QalS19E2mvefN7jVd+UiI+navdklrW6miW/bkFsmTIkDuhHrbuH8c3NQ1l6eSt2TZtOw8cfOe12iQP6Uf6yy9j60ssRSPlfRzMzGTF6LA+NHkW5cuUi2rarXHtdnVdiYoJbHBboqyMxxphK5BVVY63NALDWHjXGZBd1I2ttGpAGQObBM7e8X6BaQgK7fb788z7fHhKqVg3mLqMii6t5qiUkcFnzSwHo3rkTaa9P9iyLF31TqUN7jm3bRtZ/9gGQMXsO8Ve2pnyzphz66uu8LO/P4vJpU067TZ3fjeCrvv2xJ0+GPeMpWVnZjBg9ht49kunW+bqItes6115X5xXHp1KDEaiUxwOrgJVAZWNMNQBjTDki9GarebOmbNm2ne07d3IyK4vZcz+mU8f2kWja6Swu5qla5WKqJSawectWAJYtX0n9unU9yeJV35zYuZMKLVsQU/oiACq1v4bMDT8QW74CpevVA6DytR04unEjAOUubUbjvz3FmqG3k7X3P2HPd4q1lnGPPUG9unW5fWiRk0QXJNdeV+eVKP7MstiRpbW2ThEX5QLXhzxNIUqUKMH4MQ9w170jyMnNpX9qbxrWrx+Jpp3O4mIegIfHjGb0uPFkZWVTK6kGf3n0YU9yeNU3h776mowPZtP607nY7GyOrF3HzjemcHzXLppPTMNaS/aBA6z/3R8AaPDow8SWLculr/4fAMd37OTbW24Pe85Vq79h5uw5NGrYgNSBecVy1G/v5dr27cLedmFGjf0jy1etYv+BA3RI7sX9dw/jhutTPcni2uvKpb4JyPGCFwxjbVCzpIEFOQ0rERTubeGXcuiFN79OU68jnKbTlu+8jiDnqzLxYXthZY+/Jah/IiUen+zOi/5ndLg7EREJCeP4TjrBULEUEZHQcGg2KNRULEVEJDRULEVERAKI4mIZvRPMIiIiIaKRpYiIhIZ28BEREQkgiqdhVSxFRCQ0orhYRu+YWUREJEQ0shQRkdCI4pGliqWIiISGdvCRC0IUvysMlmvHYp1Rs5HXEU4zYOcPXkcQF0Tx/xAVSxERCY0oLpbRO2YWEREJEY0sRUQkNPSZpYiISABRPA2rYikiIqGhYikiIhJAFBfL6J1gFhERCRGNLEVEJDS0g4+IiEgAmoYVEREJwJjglrNqwsQaY742xnzgP1/XGPOlMWajMWa6MSbOv76U//wm/+V1gnloKpYiInI+GQmsL3D+r8Bz1tqGwH7gTv/6O4H91toGwHP+650z54vlrt0+hg67hx79biSl/0AmTZ3maZ4HH32CNp2S6TXgJk9znLJoyTKS+w6ga59+pE2c5HUcp/K49Fx5tR03+M0wui5dSNelC2lw93AAmo4ZTcq61XRZNI8ui+ZRrWtnAOIqVaLDrPfou30zlz/954jkK0jbTtFc6ptimZjglkB3b0wSkAK84j9vgE7ADP9VJgF9/adT/efxX97Zf/1z4nyxjI2NZeyokcx5722mT57I1OnvsOnHzZ7l6dc7hVdemuBZ+wXl5OTw+FNP88qLE5j97nQ++Giup33jWh6XnisvtuMKTRpT99YhzO/cnU/bd6J6clfK1asLwMaX/49PO3Tm0w6d2f3JPAByTpxg3Z+fYs34R8OaqzDadormWt8UK8YEtRhjhhtjVhZYhv+shb8D/wPk+s9fDByw1mb7z+8AavpP1wS2A/gvP+i//rk9tHO9YaQkVK1CsyaNAShXtiz16tbFl5HhWZ7WLVsQH1/Bs/YLWrN2HbVrJVErqSZxJUuSktyNeQsWKY+fS8+VF9tx+UYN2bdiFTnHjmFzcti7ZCk1evUs8vo5mZn854vl5Bw/EdZchdG2UzTX+qZYQY4srbVp1tpWBZa0/Ls2phewx1q7qmCLhaSwZ3HZL1ZssTTGXGWMqeA/XdoY85gx5t/GmL8aY+LPtdFztSM9nfUbNnDZpc0i3bSTfHsyqJaYmH8+MTHB0zcSruVxVaS240Prv6dK26uJq1SJ2NKlqda1C2Vq5r3prj/sDrp8/hktX/g7JeMj/lI+g7adop1XfRPeHXzaAX2MMVuAaeRNv/4dqGiMOfXNjiQg3X96B1ArL5YpAcQD+871oQUaWU4EMv2nJ/gb+6t/3Wvn2ui5OJqZyYjRY3lo9CjKlSsXyaadZQt5k+Tljtuu5XFRJLfjwz9sZMOEF2n/r7e5ZsZbHFi3DpudzY8TJzHniqv4tH0njvt8/PpPj4U1x9nQtlM09U0ea+2D1toka20d4CZgvrV2MPAZMMB/tVuBmf7Ts/zn8V8+31obnpElEFNgLriVtfZ31trPrbWPAfWKulHBeee0ia+fa7Z8WVnZjBg9ht49kunW+bqg7y9aVEtIYLfPl3/e59tDQtWqyuMoL7bjLW9OZV7HrixM6UvW/gMc3ryZExkZkJsL1vLTpDep3PKKiGQpjradop1XfRMTE9xybsYAo4wxm8j7TPJV//pXgYv960cBY4N6aAEuX2uMud1/+htjTCsAY0wjIKuoGxWcdx5+x23B5MNay7jHnqBe3brcPnRwUPcVbZo3a8qWbdvZvnMnJ7OymD33Yzp1bK88DvJqOy5VpQoApZNqUqNXT7bP+BcXJSbkX16zV08Orf8+YnmKom2naOdV30Tge5YA1toF1tpe/tObrbVXWmsbWGtvsNae8K8/7j/fwH95UHtFBTqCz13ABGPMH4G9wDJjzHby9jC6K5iGz9aq1d8wc/YcGjVsQOrAvH8yo357L9e2bxeJ5s8wauwfWb5qFfsPHKBDci/uv3sYN1yf6kmWEiVKMH7MA9x17whycnPpn9qbhvXre5LFxTwuPVdebcdtJr9KXKVK5GZns/qBB8k6eJDL//oiFZtfirWWzG3b+er3o/Ov3+ObFZQsX56YknHU6NmDxf0HcnjDD2HNCNp2iuNa3xTrLL7+cb4yZzOFa4wpT960awlgh7XWF+Am/5V58JzniEWkcDNqNvI6wmkG7Ax/QZUQKRMfto88c157LKj/97G3P+Lsx7FndWxYa+1h4JswZxEREXGSDqQuIiKhoV8dERERCSCKf3VExVJEREIjinfwid5HJiIiEiIaWYqISGjEaBpWRESkeFE8DatiKSIioaEdfERERAKI4pFl9D4yERGRENHIUkREQkM7+EhY2FyvE5wuiqdQoo1rx2J9KcGdA3vft+dHryNcuPSZpYiISABR/IZbxVJEREIjiqdho/dtgIiISIhoZCkiIqGhaVgREZEAtIOPiIhIAFE8sozeRyYiIhIiGlmKiEhoRPHesCqWIiISGlE8DatiKSIioaEdfERERAKIid6RZfQ+MhERkRA5L4rloiXLSO47gK59+pE2cZJnOXbt9jF02D306HcjKf0HMmnqtIhnePDRP9Gmcw963XBz/ro5n8wjZcAgGrdsw7ffrY94pp/Lycmh701D+M2I33uaw5Xt5kLP0+kfE7j9p/XctHxx/rpSlSrSZ9YMBq9eTp9ZMyhVMT7/svbP/Jkh3yxn4BcLqXLZr/PXt3niEQat+JxBq5bS/pk/hy0vuPVcnThxggFDbqPPjTeT0n8gz7+c5mmeYhkT3OIw54tlTk4Ojz/1NK+8OIHZ707ng4/msunHzZ5kiY2NZeyokcx5722mT57I1OnvRDxLv94pvPLic6eta1S/Hi/87Slat7g8olmKMnnqNOrXreNpBpe2mws9z/op0/h334GnrWsxaiQ7FixiyuVXsmPBIlqMGglA7W5diK9fjzcvu5IF94+i49+fAaDaVa2pfvWVTLuqA9NaX0NCiyuo0b5dWPK69lzFxcUxKe0fzHp7Ku9Pm8LipctYveZbz/IUy8QEtzis2HTGmBHGmFqRClOYNWvXUbtWErWSahJXsiQpyd2Yt2CRJ1kSqlahWZPGAJQrW5Z6deviy8iIaIbWLa8gPr7Caevq16tLvTq1I5qjKLt9PhZ8voQB16d6msOl7eZCz7NryTJO7N9/2rq6KT34fsp0AL6fMp26vXrmre/Vgw1vvQ2Ab8Uq4uLjKZOYiLWW2IsuIiYujthSpYgpWZJje/aEJa9rz5UxhrJlygCQnZ1NdnY2xtVR2AU8snwC+NIYs9gYc68xpmokQhXk25NBtcTE/POJiQkRL1CF2ZGezvoNG7js0mZeR3HKn595jgdG3k+Mxx/0u7bdKM/pyiRUJdPnAyDT56N01SoAlK1enSM7duZf72h6OmVrVMe3fCU7F33O7ZvWcdumdWybN5/9GzaGJZvXfVOYnJwcUgcOpm3nZNpefSWXNb/U0zxFiokJbnFYoHSbgSTyimZL4DtjzEfGmFuNMeWLupExZrgxZqUxZmXaxNeDCmixZ95/UPcYvKOZmYwYPZaHRo+iXLlyHqdxx2eLFlO5ciUubdrE6yjObTfKc5YKG11YS3y9ulS6pBGTLvk1kxo1J6lDe6q3axOWCC72TWxsLDOnT2Hh3A9Ys/Y7ftikH7iOtEBfHbHW2lzgY+BjY0xJoAcwCPgbUOhI01qbBuR9Cp158Mwt7xeolpDAbv87UACfbw8JVSM+wM2XlZXNiNFj6N0jmW6dr/Msh4u+Wr2G+QsXs+jzpZw4eYIjR48yetx4/vbk4xHP4tp2ozyny9yTQZnERDJ9PsokJnIsYy+QN5Isl1Qz/3pla9Tg6K7dXHLTDfhWrCTr6FEAtn4yj2qtW7FrybKQZ/O6b4pToXx5rmrVgsVLl9GoQX2v45zJ8anUYAQaWZ72yK21WdbaWdbaQcCvwhfrv5o3a8qWbdvZvnMnJ7OymD33Yzp1bB+Jps9grWXcY09Qr25dbh862JMMLvvDiPtYNPcD5n84k2efepKrW7fypFCCW9uN8pxpy4cf0Xhw3k4/jQcP5KfZcwD4afZHXDLoRgASW7fk5KFDZPp8HN6xgxrXtMXExhJTogQ1r2nL/g0/hCWb133zc/v27efQ4cMAHD9+nKVfLndmH4UzRPEOPoFGlgOLusBaeyzEWQpVokQJxo95gLvuHUFObi79U3vTsL4376hWrf6GmbPn0KhhA1IH5hXLUb+9l2vDtFdeYUY9+DDLV33F/gMH6NC9N/ffPYyKFSrwxNP/y779B/jNiFE0adSIV/8xIWKZXOTSdnOh5+n6Who127fjoosrc+uGNSx/8q+senYC3Se/SpNbhnBkxw4+GnoHAFvnfkLt5C4MWbOC7GPHmHf3CAB+/Ncskq5tn/f1E2vZ9sl8tsyZG5a8rj1Xe/buZez4x8jJzcXm5tK9axeu6+Bd8S5WFI8sjbVBzZIGFuQ0bFSzuV4nOJ3j7+zEXS8luDMleN8efZ5XrDLxYatoOZ9NDer/fex1NztbbXW4OxERCY0ofsOtYikiIqGhn+gSEREJQCNLERGRAKJ4B5/ofRsgIiISIhpZiohIaGgaVkREpHjOHuA9BFQsRUQkNDSyFBERCSCKi2X0PjIREZEQ0chSRERCQwclkLBwbcoi3McJ/qWieGeBaOPS8VjfqtHQ6winGZQenh+pdpJr/9NCSMVSRERCI4rf4Ebv2wAREZEQ0chSRERCI4qnYaP3kYmISGQZE9wS8O5NLWPMZ8aY9caYdcaYkf71lY0xnxhjNvr/VvKvN8aY540xm4wxa4wxLc71oalYiohIaJiY4JbAsoE/WGubAFcD9xljmgJjgXnW2obAPP95gB5AQ/8yHHj5XB+aiqWIiIRGjAluCcBau8ta+5X/9GFgPVATSAUm+a82CejrP50KTLZ5vgAqGmOqn9NDO5cbiYiIhJoxZrgxZmWBZXgx160DXAF8CSRaa3dBXkEFEvxXqwlsL3CzHf51v5h28BERkdAIcgcfa20akBawGWPKAe8Cv7PWHirmAO6FXXBOXyhXsRQRkdCIwPcsjTElySuUU6y17/lX+4wx1a21u/zTrHv863cAtQrcPAlIP5d2NQ0rIiKhEeYdfEzeEPJVYL219tkCF80CbvWfvhWYWWD9Lf69Yq8GDp6arv2lNLIUEZHzRTtgKPCtMWa1f91DwFPA28aYO4FtwA3+yz4EegKbgEzg9nNt+LwYWS5asozkvgPo2qcfaRMnBb7BBZLFxTyHDh9mxANj6d7vRnr0G8jX33zrWRbX+ubBR5+gTadkeg24yesogHv940WeRncPo8fShfRctohL7vnvviQNh99Jyoql9Fy2iMsfGw9A7Rv6033x/Pzlpn27qdj80ojkdO25KlKYv2dprf3cWmustb+21l7uXz601v7HWtvZWtvQ/3ef//rWWnuftba+tba5tXbluT4050eWOTk5PP7U07z28oskJiYwYPCtdLq2PQ3q17ugs7iYB+DJZ56lfds2PP/MU5zMyuL48eOe5HCxb/r1TmHIwBsY8/CjnmU4xbX+8SJPfJPG1L9lCB937k7uyZN0fHc6O+d+QpmaNUjq2YM57TqSe/IkpapUAWDrO++y9Z13827btAkdpk7mwLdrw5bvFNeeq2LpCD7eWbN2HbVrJVErqSZxJUuSktyNeQsWXfBZXMxz5MgRVnz1NQP69gEgrmRJKpQv70kW1/oGoHXLFsTHV/A0wymu9Y8XeSo0ash/Vq4i59gxbE4Oe5YspVavFBrecRvfPfc8uSdPAnBi794zblu7//VsnfHeGevDwbXnqlgxMcEtDis2nTEmzhhzizGmi//8zcaYF40x9/n3SAo7354MqiUm5p9PTEzAl5ERiaadzuJinu0706lcqRIPPvoEfQcNZdzjT5J57JgnWVzrG9e41j9e5Dm4/nuqtm1DXKVKxJYuTY2uXSiTVIPyDepTte3VdP10Dp1nv0/lKy4/47a/6teXre/+K6z5TnHtuSqOMSaoxWWBSvlrQAow0hjzBnkfmn4JtAZeCXM2AGwhX4nxqktdygLu5cnOyeG77zcwaEA/3n/rDUqXvoi017z5fMW1vnGNa/3jRZ5DP2xk/YQXuO79d+j47jT2r11HbnY2JjaWuIrxfNKlB18//BjtXv9/p93u4pYtyMnM5OD678OcMI9rz9WFKlCxbG6tHQhcD3QDBlhr3yBvj6IrirpRwaMwpE18PaiA1RIS2O3z5Z/3+faQULVqUPcZDVlczVMtIYHL/Ds9dO/cie++3+BZFpf6xjWu9Y9XeTa/MZW513ZhXs9UTu7fz+Eff+JY+i52/Hs2APu++hqbayl18cX5t/lV/8iNKsG956pY4T82rGcCpYsxxsQB5YEyQLx/fSmgyGlYa22atbaVtbbV8DtuCypg82ZN2bJtO9t37uRkVhaz535Mp47tg7rPaMjiYp6qVS6mWmICm7dsBWDZ8pXUr1vXkyyu9Y1rXOsfr/Kc2nmnTFJNavVOYeuM99gxew6JHfLaLl+/HjElS3LiP//Ju4Ex/Cq1D1vffT/s2U5x7bkqVpj3hvVSoL1hXwW+B2KBccA7xpjN5B3tfVqYswFQokQJxo95gLvuHUFObi79U3vTsH79SDTtdBYX8wA8PGY0o8eNJysrm1pJNfjLow97ksPFvhk19o8sX7WK/QcO0CG5F/ffPYwbrk/1JItr/eNVnmsmT6RU5UrkZmezcvRYsg4eZPObU7nqxQn0WLqQ3Kwsvrz3/vzrJ7RrQ2Z6Oke3bg17tlNce66K5fjoMBjG2uIPk2eMqQFgrU03xlQEugDbrLXLz6qFzIPndBw+8UCAbSHiHH+nKW56q0ZDryOcZlD6Rq8jnK5MfNheWPbHr4L6J2Lqt3D2RR/we5bW2vQCpw8AM8KaSERExDHOH5RARETOE45/VzIYKpYiIhIaUfzRiYqliIiERhTv4BO9j0xERCRENLIUEZHQ0DSsiIhIICqWIiIixdPIUkREJIAoLpbawUdERCQAjSxFRCREondkqWIpIiKhEcXTsCqW8l9RvKEHTQeZP28M2vmD1xFOM7FaA68jnOaOQxnhu/Mo3ixVLEVEJESit1pqBx8REZEANLIUEZHQiOKPB1QsRUQkNFQsRUREAoneYqnPLEVERALQyFJEREJD07AiIiKBqFiKiIgUTyNLERGRAKK4WGoHHxERkQCcL5YPPvoEbTol02vATV5HAWDRkmUk9x1A1z79SJs4yes4TuU5ceIEA4bcRp8bbyal/0CefznNsyy7dvsYOuweevS7kZT+A5k0dZpnWU45dPgwIx4YS/d+N9Kj30C+/uZbT/O4tO24lGfzlq2k3jQkf2nR/jpen/JWRNpues9wrv9iEdd/uZim9/4GgMqXNqPXpx/Sd9lCukx/k5Lly+Vf/9ejRjJg9XL6r1pGzc7XRSRj8UyQi7ucn4bt1zuFIQNvYMzDj3odhZycHB5/6mlee/lFEhMTGDD4Vjpd254G9espDxAXF8ektH9QtkwZsrKyufmOYXRo14bLf9084lliY2MZO2okzZo05sjRo/S/+RbaXXWlZ30D8OQzz9K+bRuef+YpTmZlcfz4cc+yuLbtuJSnXp3azJz2Zn6uDt170fW6jmFvt2KTxlxy6xBmXZdM7smTJL83nR1zP6Hdi8+xYtyj7F6ylIZDbqb5yN/y1Z+eouIljajXvy/vXXkNZapXo/usGbx7xdXY3NywZy2KuZCnYY0x9Y0xo40xE4wx/2uMudsYEx+JcACtW7YgPr5CpJor1pq166hdK4laSTWJK1mSlORuzFuwSHn8jDGULVMGgOzsbLKzsz178SRUrUKzJo0BKFe2LPXq1sWXEcZfWwjgyJEjrPjqawb07QNAXMmSVChf3rM8rm07ruU5ZdnyFdRKSqJmjephb6viJY3Ys2IVOceOYXNy2LVkKbV79SS+QQN2L1kKQPpnC6jdpxcAv0rpweZ33yf35EmObN3Goc1bqNKqRdhzFsuY4BaHFVssjTEjgH8CFwGtgdJALWCZMaZj2NM5xrcng2qJifnnExMTPP0H7FoeyHsnnjpwMG07J9P26iu5rPmlnuYB2JGezvoNG7js0maeZdi+M53KlSrx4KNP0HfQUMY9/iSZx455lse1bce1PKfMnvsJvZK7RaSt/d+tp1q7NpSqXInY0qWp1a0LZZNqsn/9en7VszsAdfr2oVzNmgCUqVGdozt35t8+c2c6ZauHv6gXL3qnYQONLIcB3a21fwK6AE2tteOA7sBzRd3IGDPcGLPSGLMybeLrIQvrNcuZv2no5dPrWh7Im/6cOX0KC+d+wJq13/HDph89zXM0M5MRo8fy0OhRlCtXLvANwiQ7J4fvvt/AoAH9eP+tNyhd+iLSXvPucznXth3X8gCczMpi/qLFdO/aKSLtHfxhI2uee4Hk92eQ/N509n27jtzsbD6/dyRNht9Bn4WfUrJ8OXKyTgKFT3la1353NYqczWeWJYAcoBRQHsBau80YU7KoG1hr04C8vTsyD0bNs1ctIYHdPl/+eZ9vDwlVqypPISqUL89VrVqweOkyGjWo70mGrKxsRoweQ+8eyXTzeOeHagkJVEtIyB9pd+/cibTXJ3uax6Vtx7U8AIuWLKVZ40uocvHFEWtz4xtT2PjGFABajh/H0fR0Dm7cxNy+NwJQoUE9aiV3BeDoznTK+keZAGVq1iBz9+6IZS2U41OpwQg0snwFWGGMSQOWAS8CGGOqAvvCnM05zZs1Zcu27WzfuZOTWVnMnvsxnTq2Vx6/ffv2c+jwYQCOHz/O0i+XU69ObU+yWGsZ99gT1Ktbl9uHDvYkQ0FVq1xMtcQENm/ZCsCy5SupX7euZ3lc23ZcywMw+6OPSYnQFOwpF1WpAkDZpJrU7pPC5hnv5a/DGC5/YBTfv5o3I7Htw4+o178vMXFxlKv9K+Lr1WXvyq8imvcMUfyZZbEjS2vtBGPMp0AT4Flr7ff+9RlAhwjkY9TYP7J81Sr2HzhAh+Re3H/3MG64PjUSTZ+hRIkSjB/zAHfdO4KcDnHkqwAABmtJREFU3Fz6p/amYX1vRk0u5tmzdy9jxz9GTm4uNjeX7l27cF0Hb/7hrVr9DTNnz6FRwwakDswrlqN+ey/Xtm/nSR6Ah8eMZvS48WRlZVMrqQZ/efRhz7K4tu24lufYsbw3e4+PezCi7XZ68zVKVa6Ezcpi2R/GcPLAQZreM5wmw+4AYOus2Wx8cyoAB77fwE//mkW/FZ9js3NYNnqsp3vC5nG74AXDhH2OO4qmYeUC5tpnQY6/C/eUY8/VxOoNvY5wmjsOZYRv49m/O7jOr1TN2Q3b+e9ZiojIeSKK38SpWIqISGhEb61UsRQRkVCJ3mqpYikiIqERxdOwzh9IXURExGsaWYqISGhE8chSxVJEREJExVJERKR4UTyy1GeWIiISGhE43J0xprsxZoMxZpMxZmyYH1E+FUsRETkvGGNigZeAHkBTYJAxpmkk2laxFBGREAn771leCWyy1m621p4EpgEROVi4PrMUEZHQCP9nljWB7QXO7wCuCnejEIliWSY+JL1njBnu/51MJ7iUx6UsoDyBuJTHpSwQnXnuOJThTJawC/L/vTFmODC8wKq0nz3mwu4/IkfOP5+mYYcHvkpEuZTHpSygPIG4lMelLKA8xXEpS1hYa9Osta0KLD9/c7ADqFXgfBKQHols51OxFBGRC9sKoKExpq4xJg64CZgViYb1maWIiJwXrLXZxpjfAnOBWGCitXZdJNo+n4qla3P1LuVxKQsoTyAu5XEpCyhPcVzK4hlr7YfAh5Fu11jHflVcRETENfrMUkREJIDzolh6dXijIrJMNMbsMcas9TKHP0stY8xnxpj1xph1xpiRHue5yBiz3BjzjT/PY17m8WeKNcZ8bYz5wIEsW4wx3xpjVhtjVjqQp6IxZoYx5nv/NtTGwyyX+Pvl1HLIGPM7D/P83r8NrzXGvGWMucirLP48I/1Z1nnZLxcy56dh/Yc3+gHoSt5uwyuAQdba7zzK0wE4Aky21l7qRYYCWaoD1a21XxljygOrgL4e9o0BylprjxhjSgKfAyOttV94kcefaRTQCqhgre3lVQ5/li1AK2vtXi9znGKMmQT/v737CY2rjMI4/HvbcZGkiJLWYlukdVMUF00WVQwEaVS0SkFXFurClUgodCW0m66FUty5SbCF1khM0p2UFkTtxi6SVixEEC2tidUUrNpqoX98XdwvMJWE2c25w5wHwtxkc1+GyZy55/vuGc7ZHis7C3tt/1GDXGuBReBZ21cCzr+Z6rX7tO3bkiaBz20fa3eWkucZqkk1O4E7wGngPds/ROTpVp1wZRk23mgltr8Gfo86fzPb12zPleObwDzVhIuoPLZ9q/z6UPkJ+zQmaQvwGjAWlaGuJD0MDAPjALbv1KFQFiPAjxGFskkD6JHUAHpp0718q3gK+Mb2P7bvAV8BbwTm6UqdUCxXGm8UVhDqStJWYAA4H5xjraSLwBJw1nZkng+B94F/AzM0M3BG0myZVBLpSeA68HFpU49J6gvOtOwtYCLq5LYXgSPAVeAa8KftM1F5gEvAsKR+Sb3Abh68MT+1QScUy7DxRp1C0jpgGjhg+6/ILLbv295BNVljZ2khtZ2k14El27MR51/FkO1Bqm9MGC0t/SgNYBD4yPYA8DcQuh8AoLSD9wCfBWZ4lKp7tQ3YBPRJ2heVx/Y88AFwlqoF+y1wLypPt+qEYhk23qgTlLXBaeCk7ZnoPMtKS+9L4JWgCEPAnrJO+CmwS9KJoCwA2P6lPC4Bp6iWGKIsAAtNV/5TVMUz2qvAnO3fAjO8CFy2fd32XWAGeD4wD7bHbQ/aHqZaBsr1yjbrhGIZNt6o7sqGmnFg3vbRGuTZIOmRctxD9abzfUQW2wdtb7G9leo184XtsKsDSX1lExal3fkyVXsthO1fgZ8lbS9/GgFCNob9z14CW7DFVeA5Sb3lf2yEaj9AGEmPlccngDeJf466Tu0n+ESON1qJpAngBWC9pAXgsO3xoDhDwNvAd2WdEOBQmXAR4XHgeNnNuAaYtB1+y0ZNbAROVe+9NIBPbJ+OjcR+4GT5EPoT8E5kmLIe9xLwbmQO2+clTQFzVO3OC8RPz5mW1A/cBUZt3wjO03Vqf+tISimlFK0T2rAppZRSqCyWKaWUUgtZLFNKKaUWslimlFJKLWSxTCmllFrIYplSSim1kMUypZRSaiGLZUoppdTCf2LSMXhdo5hBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "plt.figure(figsize = (8,6))\n",
    "\n",
    "# colormaps : cmap=\"YlGnBu\" , cmap=\"Greens\", cmap=\"Blues\",  cmap=\"Reds\"\n",
    "sns.heatmap(cm, annot=True, cmap=\"Reds\", fmt='d').plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9.3 - Metrics Calculated from Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'0': {'f1-score': 0.9938587512794269,\n",
      "       'precision': 0.9969199178644764,\n",
      "       'recall': 0.9908163265306122,\n",
      "       'support': 980},\n",
      " '1': {'f1-score': 0.988005330964016,\n",
      "       'precision': 0.996415770609319,\n",
      "       'recall': 0.9797356828193833,\n",
      "       'support': 1135},\n",
      " '2': {'f1-score': 0.9839883551673945,\n",
      "       'precision': 0.9854227405247813,\n",
      "       'recall': 0.9825581395348837,\n",
      "       'support': 1032},\n",
      " '3': {'f1-score': 0.9805068226120859,\n",
      "       'precision': 0.9654510556621881,\n",
      "       'recall': 0.996039603960396,\n",
      "       'support': 1010},\n",
      " '4': {'f1-score': 0.992849846782431,\n",
      "       'precision': 0.9959016393442623,\n",
      "       'recall': 0.9898167006109979,\n",
      "       'support': 982},\n",
      " '5': {'f1-score': 0.988235294117647,\n",
      "       'precision': 0.9876819708846585,\n",
      "       'recall': 0.9887892376681614,\n",
      "       'support': 892},\n",
      " '6': {'f1-score': 0.9844720496894409,\n",
      "       'precision': 0.9763860369609856,\n",
      "       'recall': 0.9926931106471816,\n",
      "       'support': 958},\n",
      " '7': {'f1-score': 0.988235294117647,\n",
      "       'precision': 0.9960474308300395,\n",
      "       'recall': 0.980544747081712,\n",
      "       'support': 1028},\n",
      " '8': {'f1-score': 0.9867346938775511,\n",
      "       'precision': 0.9807302231237323,\n",
      "       'recall': 0.9928131416837782,\n",
      "       'support': 974},\n",
      " '9': {'f1-score': 0.9865470852017938,\n",
      "       'precision': 0.9919839679358717,\n",
      "       'recall': 0.981169474727453,\n",
      "       'support': 1009},\n",
      " 'accuracy': 0.9873,\n",
      " 'macro avg': {'f1-score': 0.9873433523809434,\n",
      "               'precision': 0.9872940753740315,\n",
      "               'recall': 0.987497616526456,\n",
      "               'support': 10000},\n",
      " 'weighted avg': {'f1-score': 0.9873175638923014,\n",
      "                  'precision': 0.9874420624726045,\n",
      "                  'recall': 0.9873,\n",
      "                  'support': 10000}}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "from pprint import pprint\n",
    "\n",
    "pprint(classification_report(test_labels, predictions2, output_dict=True))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 10 : Save the Model\n",
    "\n",
    "We are saving it to be loaded later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model.save(\"mnist_model_28x28\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TensorFlow-GPU",
   "language": "python",
   "name": "tf2-gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}