Introduction to TensorFlow (v1.13)

tensorflow_intro.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to Tensorflow\n",
    "\n",
    "Notebook uses some functions from  \n",
    "    Stanford CS231n Convolutional Neural Networks for Visual Recognition  \n",
    "    http://cs231n.stanford.edu, http://cs231n.github.io\n",
    "\n",
    "See also https://www.tensorflow.org/guide/low_level_intro"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.13.1'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (optional) import Matplotlib for visualization\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic usage\n",
    "First specify a \"computational graph\", then create a session to run the computations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: <class 'tensorflow.python.framework.ops.Tensor'> Tensor(\"Placeholder:0\", shape=(4,), dtype=float32)\n",
      "y: <class 'tensorflow.python.framework.ops.Tensor'> Tensor(\"pow:0\", shape=(4,), dtype=float32)\n",
      "z: <class 'tensorflow.python.framework.ops.Tensor'> Tensor(\"Exp:0\", shape=(4,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "# define a placeholder which will hold a concrete value when the graph is run\n",
    "x = tf.placeholder(tf.float32, shape=[4])\n",
    "\n",
    "# define variables of the \"computational graph\" (without actually running it yet)\n",
    "y = x**2\n",
    "z = tf.exp(1 - y)\n",
    "\n",
    "print('x:', type(x), x)\n",
    "print('y:', type(y), y)\n",
    "print('z:', type(z), z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_eval: [ 1.        4.       12.25      9.869605]\n",
      "z_eval: [1.0000000e+00 4.9787067e-02 1.3007298e-05 1.4059810e-04]\n"
     ]
    }
   ],
   "source": [
    "# evaluate computational graph\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "\n",
    "    # \"feed\" values into 'x' and evaluate 'y' and 'z'\n",
    "    y_eval, z_eval = sess.run((y, z), feed_dict={x: [1, 2, -3.5, np.pi]})\n",
    "\n",
    "    print('y_eval:', y_eval)\n",
    "    print('z_eval:', z_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.   9. ]\n",
      " [ 1.5 11. ]]\n"
     ]
    }
   ],
   "source": [
    "# demonstrate matrix vector multiplication in TensorFlow\n",
    "\n",
    "# another placeholder and constant matrix\n",
    "u = tf.placeholder(tf.float32, shape=[3, 2])\n",
    "W = tf.constant([[1, 2, 3], [1.5, 2.5, 3.5]])\n",
    "# matrix-matrix multiplication\n",
    "v = tf.matmul(W, u)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    # actually computes W*u\n",
    "    v_val = sess.run(v, feed_dict={u: [[0.5, 1], [1, 1], [-0.5, 2]]})\n",
    "    print(v_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "grad_x_eval: [array([-2.0000000e+00, -1.9914827e-01,  9.1051086e-05, -8.8340393e-04],\n",
      "      dtype=float32)]\n"
     ]
    }
   ],
   "source": [
    "# TensorFlow can compute gradients\n",
    "grad_x = tf.gradients(z, x)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "\n",
    "    grad_x_eval = sess.run(grad_x, feed_dict={x: [1, 2, -3.5, np.pi]})\n",
    "\n",
    "    # d/dx exp(1-x^2) == -2 x exp(1-x^2)\n",
    "    print('grad_x_eval:', grad_x_eval)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load MNIST dataset\n",
    "\n",
    "mnist = tf.keras.datasets.mnist\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "\n",
    "# normalize to [0,1] range\n",
    "X_train = X_train / 255.0\n",
    "X_test  = X_test  / 255.0\n",
    "\n",
    "# split into actual training and validation dataset\n",
    "X_val = X_train[50000:]\n",
    "y_val = y_train[50000:]\n",
    "X_train = X_train[:50000]\n",
    "y_train = y_train[:50000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 28, 28)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dtype('float64')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000,)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x1f5efb2ea90>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADgpJREFUeJzt3X+MVfWZx/HPs1j+kKI4aQRCYSnEYJW4082IjSWrxkzVDQZHrekkJjQapn8wiU02ZA3/VNNgyCrslmiamaZYSFpKE3VB0iw0otLGZuKIWC0srTFsO3IDNTjywx9kmGf/mEMzxbnfe+fec++5zPN+JeT+eM6558kNnznn3O+592vuLgDx/EPRDQAoBuEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxDUZc3cmJlxOSHQYO5u1SxX157fzO40syNm9q6ZPVrPawFoLqv12n4zmybpj5I6JQ1Jel1St7sfSqzDnh9osGbs+ZdJetfd33P3c5J+IWllHa8HoInqCf88SX8Z93goe+7vmFmPmQ2a2WAd2wKQs3o+8Jvo0OJzh/Xu3i+pX+KwH2gl9ez5hyTNH/f4y5KO1dcOgGapJ/yvS7rGzL5iZtMlfVvSrnzaAtBoNR/2u/uImfVK2iNpmqQt7v6H3DoD0FA1D/XVtDHO+YGGa8pFPgAuXYQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EVfMU3ZJkZkclnZZ0XtKIu3fk0RTyM23atGT9yiuvbOj2e3t7y9Yuv/zy5LpLlixJ1tesWZOsP/XUU2Vr3d3dyXU//fTTZH3Dhg3J+uOPP56st4K6wp+5zd0/yOF1ADQRh/1AUPWG3yXtNbM3zKwnj4YANEe9h/3fcPdjZna1pF+b2f+6+/7xC2R/FPjDALSYuvb87n4suz0h6QVJyyZYpt/dO/gwEGgtNYffzGaY2cwL9yV9U9I7eTUGoLHqOeyfLekFM7vwOj939//JpSsADVdz+N39PUn/lGMvU9aCBQuS9enTpyfrN998c7K+fPnysrVZs2Yl173vvvuS9SINDQ0l65s3b07Wu7q6ytZOnz6dXPett95K1l999dVk/VLAUB8QFOEHgiL8QFCEHwiK8ANBEX4gKHP35m3MrHkba6L29vZkfd++fcl6o79W26pGR0eT9YceeihZP3PmTM3bLpVKyfqHH36YrB85cqTmbTeau1s1y7HnB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgGOfPQVtbW7I+MDCQrC9atCjPdnJVqffh4eFk/bbbbitbO3fuXHLdqNc/1ItxfgBJhB8IivADQRF+ICjCDwRF+IGgCD8QVB6z9IZ38uTJZH3t2rXJ+ooVK5L1N998M1mv9BPWKQcPHkzWOzs7k/WzZ88m69dff33Z2iOPPJJcF43Fnh8IivADQRF+ICjCDwRF+IGgCD8QFOEHgqr4fX4z2yJphaQT7r40e65N0g5JCyUdlfSAu6d/6FxT9/v89briiiuS9UrTSff19ZWtPfzww8l1H3zwwWR9+/btyTpaT57f5/+ppDsveu5RSS+5+zWSXsoeA7iEVAy/u++XdPElbCslbc3ub5V0T859AWiwWs/5Z7t7SZKy26vzawlAMzT82n4z65HU0+jtAJicWvf8x81sriRltyfKLeju/e7e4e4dNW4LQAPUGv5dklZl91dJ2plPOwCapWL4zWy7pN9JWmJmQ2b2sKQNkjrN7E+SOrPHAC4hFc/53b27TOn2nHsJ69SpU3Wt/9FHH9W87urVq5P1HTt2JOujo6M1bxvF4go/ICjCDwRF+IGgCD8QFOEHgiL8QFBM0T0FzJgxo2ztxRdfTK57yy23JOt33XVXsr53795kHc3HFN0Akgg/EBThB4Ii/EBQhB8IivADQRF+ICjG+ae4xYsXJ+sHDhxI1oeHh5P1l19+OVkfHBwsW3vmmWeS6zbz/+ZUwjg/gCTCDwRF+IGgCD8QFOEHgiL8QFCEHwiKcf7gurq6kvVnn302WZ85c2bN2163bl2yvm3btmS9VCrVvO2pjHF+AEmEHwiK8ANBEX4gKMIPBEX4gaAIPxBUxXF+M9siaYWkE+6+NHvuMUmrJf01W2ydu/+q4sYY57/kLF26NFnftGlTsn777bXP5N7X15esr1+/Pll///33a972pSzPcf6fSrpzguf/093bs38Vgw+gtVQMv7vvl3SyCb0AaKJ6zvl7zez3ZrbFzK7KrSMATVFr+H8kabGkdkklSRvLLWhmPWY2aGblf8wNQNPVFH53P+7u5919VNKPJS1LLNvv7h3u3lFrkwDyV1P4zWzuuIddkt7Jpx0AzXJZpQXMbLukWyV9ycyGJH1f0q1m1i7JJR2V9N0G9gigAfg+P+oya9asZP3uu+8uW6v0WwFm6eHqffv2JeudnZ3J+lTF9/kBJBF+ICjCDwRF+IGgCD8QFOEHgmKoD4X57LPPkvXLLktfhjIyMpKs33HHHWVrr7zySnLdSxlDfQCSCD8QFOEHgiL8QFCEHwiK8ANBEX4gqIrf50dsN9xwQ7J+//33J+s33nhj2VqlcfxKDh06lKzv37+/rtef6tjzA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQjPNPcUuWLEnWe3t7k/V77703WZ8zZ86ke6rW+fPnk/VSqZSsj46O5tnOlMOeHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCqjjOb2bzJW2TNEfSqKR+d/+hmbVJ2iFpoaSjkh5w9w8b12pclcbSu7u7y9YqjeMvXLiwlpZyMTg4mKyvX78+Wd+1a1ee7YRTzZ5/RNK/uftXJX1d0hozu07So5JecvdrJL2UPQZwiagYfncvufuB7P5pSYclzZO0UtLWbLGtku5pVJMA8jepc34zWyjpa5IGJM1295I09gdC0tV5Nwegcaq+tt/MvijpOUnfc/dTZlVNByYz65HUU1t7ABqlqj2/mX1BY8H/mbs/nz193MzmZvW5kk5MtK6797t7h7t35NEwgHxUDL+N7eJ/Iumwu28aV9olaVV2f5Wknfm3B6BRKk7RbWbLJf1G0tsaG+qTpHUaO+//paQFkv4s6VvufrLCa4Wconv27NnJ+nXXXZesP/3008n6tddeO+me8jIwMJCsP/nkk2VrO3em9xd8Jbc21U7RXfGc391/K6nci90+maYAtA6u8AOCIvxAUIQfCIrwA0ERfiAowg8ExU93V6mtra1sra+vL7lue3t7sr5o0aKaesrDa6+9lqxv3LgxWd+zZ0+y/sknn0y6JzQHe34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCCrMOP9NN92UrK9duzZZX7ZsWdnavHnzauopLx9//HHZ2ubNm5PrPvHEE8n62bNna+oJrY89PxAU4QeCIvxAUIQfCIrwA0ERfiAowg8EFWacv6urq656PQ4dOpSs7969O1kfGRlJ1lPfuR8eHk6ui7jY8wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUObu6QXM5kvaJmmOpFFJ/e7+QzN7TNJqSX/NFl3n7r+q8FrpjQGom7tbNctVE/65kua6+wEzmynpDUn3SHpA0hl3f6rapgg/0HjVhr/iFX7uXpJUyu6fNrPDkor96RoAdZvUOb+ZLZT0NUkD2VO9ZvZ7M9tiZleVWafHzAbNbLCuTgHkquJh/98WNPuipFclrXf3581stqQPJLmkH2js1OChCq/BYT/QYLmd80uSmX1B0m5Je9x90wT1hZJ2u/vSCq9D+IEGqzb8FQ/7zcwk/UTS4fHBzz4IvKBL0juTbRJAcar5tH+5pN9IeltjQ32StE5St6R2jR32H5X03ezDwdRrsecHGizXw/68EH6g8XI77AcwNRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCavYU3R9I+r9xj7+UPdeKWrW3Vu1Lorda5dnbP1a7YFO/z/+5jZsNuntHYQ0ktGpvrdqXRG+1Kqo3DvuBoAg/EFTR4e8vePsprdpbq/Yl0VutCumt0HN+AMUpes8PoCCFhN/M7jSzI2b2rpk9WkQP5ZjZUTN728wOFj3FWDYN2gkze2fcc21m9msz+1N2O+E0aQX19piZvZ+9dwfN7F8L6m2+mb1sZofN7A9m9kj2fKHvXaKvQt63ph/2m9k0SX+U1ClpSNLrkrrd/VBTGynDzI5K6nD3wseEzexfJJ2RtO3CbEhm9h+STrr7huwP51Xu/u8t0ttjmuTMzQ3qrdzM0t9Rge9dnjNe56GIPf8ySe+6+3vufk7SLyStLKCPlufu+yWdvOjplZK2Zve3auw/T9OV6a0luHvJ3Q9k909LujCzdKHvXaKvQhQR/nmS/jLu8ZBaa8pvl7TXzN4ws56im5nA7AszI2W3Vxfcz8UqztzcTBfNLN0y710tM17nrYjwTzSbSCsNOXzD3f9Z0l2S1mSHt6jOjyQt1tg0biVJG4tsJptZ+jlJ33P3U0X2Mt4EfRXyvhUR/iFJ88c9/rKkYwX0MSF3P5bdnpD0gsZOU1rJ8QuTpGa3Jwru52/c/bi7n3f3UUk/VoHvXTaz9HOSfubuz2dPF/7eTdRXUe9bEeF/XdI1ZvYVM5su6duSdhXQx+eY2YzsgxiZ2QxJ31TrzT68S9Kq7P4qSTsL7OXvtMrMzeVmllbB712rzXhdyEU+2VDGf0maJmmLu69vehMTMLNFGtvbS2PfePx5kb2Z2XZJt2rsW1/HJX1f0n9L+qWkBZL+LOlb7t70D97K9HarJjlzc4N6Kzez9IAKfO/ynPE6l364wg+IiSv8gKAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E9f/Ex0YKZYOZcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# example: show a handwritten digit from the MNIST dataset\n",
    "plt.imshow(X_train[0], cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# corresponding label\n",
    "y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Dataset(object):\n",
    "    def __init__(self, X, y, batch_size, shuffle=False):\n",
    "        \"\"\"\n",
    "        Construct a Dataset object to iterate over data X and labels y\n",
    "        \n",
    "        Args:\n",
    "            X: numpy array of data, of any shape\n",
    "            y: numpy array of labels, of any shape but with y.shape[0] == X.shape[0]\n",
    "            batch_size: integer giving number of elements per minibatch\n",
    "            shuffle: (optional) boolean, whether to shuffle the data on each epoch\n",
    "        \"\"\"\n",
    "        assert X.shape[0] == y.shape[0], 'number of data items and labels must agree'\n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = shuffle\n",
    "\n",
    "    def __iter__(self):\n",
    "        N = self.X.shape[0]\n",
    "        B = self.batch_size\n",
    "        idxs = np.arange(N)\n",
    "        if self.shuffle:\n",
    "            np.random.shuffle(idxs)\n",
    "        return iter((self.X[idxs[i:i+B]], self.y[idxs[i:i+B]]) for i in range(0, N, B))\n",
    "\n",
    "\n",
    "train_dset = Dataset(X_train, y_train, batch_size=64, shuffle=True)\n",
    "val_dset   = Dataset(X_val, y_val, batch_size=64, shuffle=False)\n",
    "test_dset  = Dataset(X_test, y_test, batch_size=64)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Construct and train networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The [tf.keras.layers](https://www.tensorflow.org/api_docs/python/tf/keras/layers) API provides many common layer types."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class NetModel(tf.keras.Model):\n",
    "\n",
    "    def __init__(self, hidden_size, num_classes):\n",
    "        super(NetModel, self).__init__()\n",
    "        \n",
    "        # define individual layers\n",
    "        # flatten input (28x28 -> 784)\n",
    "        self.flatten = tf.keras.layers.Flatten()\n",
    "        # first dense layer and ReLU\n",
    "        self.layer1 = tf.keras.layers.Dense(units=hidden_size, activation=tf.nn.relu)\n",
    "        # second dense layer (without activation function, i.e., only affine transformation)\n",
    "        self.layer2 = tf.keras.layers.Dense(units=num_classes)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        xf = self.flatten(inputs)\n",
    "        a1 = self.layer1(xf)\n",
    "        a2 = self.layer2(a1)\n",
    "        return a2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def check_accuracy(sess, dset, x, scores):\n",
    "    \"\"\"\n",
    "    Check accuracy on a classification model.\n",
    "    \n",
    "    Args:\n",
    "        sess: TensorFlow session that will be used to run the graph\n",
    "        dset: dataset object on which to check accuracy\n",
    "        x: TensorFlow placeholder where input images should be fed\n",
    "        scores: TensorFlow tensor representing the scores output from the\n",
    "                model; this is the tensor we will ask TensorFlow to evaluate.\n",
    "      \n",
    "    Returns: nothing, but prints the accuracy of the model\n",
    "    \"\"\"\n",
    "    num_correct = 0\n",
    "    num_samples = 0\n",
    "    for x_batch, y_batch in dset:\n",
    "        scores_np = sess.run(scores, feed_dict={x: x_batch})\n",
    "        y_pred = scores_np.argmax(axis=1)\n",
    "        num_samples += x_batch.shape[0]\n",
    "        num_correct += (y_pred == y_batch).sum()\n",
    "    acc = float(num_correct) / num_samples\n",
    "    print('got {} / {} correct ({:2g}%)'.format(num_correct, num_samples, 100 * acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Anaconda3\\envs\\tensorflow1.13\\lib\\site-packages\\tensorflow\\python\\ops\\resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "starting epoch 0\n",
      "iteration 0, loss = 2.37606\n",
      "got 1623 / 10000 correct (16.23%)\n",
      "\n",
      "iteration 500, loss = 0.068479\n",
      "got 9539 / 10000 correct (95.39%)\n",
      "\n",
      "starting epoch 1\n",
      "iteration 1000, loss = 0.186645\n",
      "got 9634 / 10000 correct (96.34%)\n",
      "\n",
      "iteration 1500, loss = 0.255576\n",
      "got 9641 / 10000 correct (96.41%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# train network\n",
    "\n",
    "# clear the current TensorFlow graph\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# declare placeholders for the data and labels\n",
    "x = tf.placeholder(tf.float32, [None, 28, 28])\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "# generate network model\n",
    "network = NetModel(512, 10)\n",
    "\n",
    "net_output = network(x)\n",
    "\n",
    "# loss (cost) function; 'logits' is the network output without final activation function\n",
    "losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=net_output)\n",
    "loss = tf.reduce_mean(losses)\n",
    "\n",
    "# there are various optimizers available, see tf.train documentation\n",
    "learning_rate = 1e-2\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "train_op = optimizer.minimize(loss)\n",
    "\n",
    "# constant to control how often we print when training models\n",
    "print_every = 500\n",
    "\n",
    "num_epochs = 2\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    t = 0\n",
    "    for epoch in range(num_epochs):\n",
    "        print('starting epoch {}'.format(epoch))\n",
    "        for x_np, y_np in train_dset:\n",
    "            loss_np, _ = sess.run((loss, train_op), feed_dict={x: x_np, y: y_np})\n",
    "            if t % print_every == 0:\n",
    "                print('iteration {}, loss = {:g}'.format(t, loss_np))\n",
    "                check_accuracy(sess, val_dset, x, net_output)\n",
    "                print()\n",
    "            t += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Alternative high-level Keras Sequential API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Anaconda3\\envs\\tensorflow1.13\\lib\\site-packages\\tensorflow\\python\\keras\\layers\\core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "Epoch 1/5\n",
      "50000/50000 [==============================] - 14s 281us/sample - loss: 0.2407 - acc: 0.9285\n",
      "Epoch 2/5\n",
      "50000/50000 [==============================] - 13s 268us/sample - loss: 0.1053 - acc: 0.9675\n",
      "Epoch 3/5\n",
      "50000/50000 [==============================] - 12s 241us/sample - loss: 0.0746 - acc: 0.9772\n",
      "Epoch 4/5\n",
      "50000/50000 [==============================] - 12s 242us/sample - loss: 0.0560 - acc: 0.9825\n",
      "Epoch 5/5\n",
      "50000/50000 [==============================] - 13s 250us/sample - loss: 0.0440 - acc: 0.9856\n",
      "10000/10000 [==============================] - 1s 51us/sample - loss: 0.0762 - acc: 0.9764\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.0761783290802443, 0.9764]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# high-level Keras Sequential API\n",
    "\n",
    "model = tf.keras.models.Sequential([\n",
    "    tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "    tf.keras.layers.Dense(512, activation=tf.nn.relu),\n",
    "    tf.keras.layers.Dropout(0.2),\n",
    "    tf.keras.layers.Dense(10, activation=tf.nn.softmax)\n",
    "])\n",
    "model.compile(optimizer='adam',\n",
    "              loss='sparse_categorical_crossentropy',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(X_train, y_train, epochs=5)\n",
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorBoard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reset and re-create graph to remove clutter\n",
    "\n",
    "# reset the current TensorFlow graph\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# declare placeholders for the data and labels\n",
    "x = tf.placeholder(tf.float32, [None, 28, 28])\n",
    "y = tf.placeholder(tf.int32, [None])\n",
    "\n",
    "# generate network model\n",
    "network = NetModel(512, 10)\n",
    "\n",
    "net_output = network(x)\n",
    "\n",
    "# loss (cost) function; 'logits' is the network output without final activation function\n",
    "losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=y, logits=net_output)\n",
    "loss = tf.reduce_mean(losses)\n",
    "\n",
    "# there are various optimizers available, see tf.train documentation\n",
    "learning_rate = 1e-2\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "\n",
    "train_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    # save computational graph to disk\n",
    "    tf.summary.FileWriter('./tf_logs', sess.graph)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Start TensorBoard by calling\n",
    "```\n",
    "tensorboard --logdir=tf_logs\n",
    "```\n",
    "from the command line and then opening the specified address with a web browser."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:tensorflow1.13]",
   "language": "python",
   "name": "conda-env-tensorflow1.13-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}