Quick tensorflow tutorial

A modern introduction to tensorflow.ipynb

mnist_data.py

#  Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
"""tf.data.Dataset interface to the MNIST dataset."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import gzip
import os
import shutil
import tempfile

import numpy as np
from six.moves import urllib
import tensorflow as tf


def read32(bytestream):
  """Read 4 bytes from bytestream as an unsigned 32-bit integer."""
  dt = np.dtype(np.uint32).newbyteorder('>')
  return np.frombuffer(bytestream.read(4), dtype=dt)[0]


def check_image_file_header(filename):
  """Validate that filename corresponds to images for the MNIST dataset."""
  with tf.gfile.Open(filename, 'rb') as f:
    magic = read32(f)
    read32(f)  # num_images, unused
    rows = read32(f)
    cols = read32(f)
    if magic != 2051:
      raise ValueError('Invalid magic number %d in MNIST file %s' % (magic,
                                                                     f.name))
    if rows != 28 or cols != 28:
      raise ValueError(
          'Invalid MNIST file %s: Expected 28x28 images, found %dx%d' %
          (f.name, rows, cols))


def check_labels_file_header(filename):
  """Validate that filename corresponds to labels for the MNIST dataset."""
  with tf.gfile.Open(filename, 'rb') as f:
    magic = read32(f)
    read32(f)  # num_items, unused
    if magic != 2049:
      raise ValueError('Invalid magic number %d in MNIST file %s' % (magic,
                                                                     f.name))


def download(directory, filename):
  """Download (and unzip) a file from the MNIST dataset if not already done."""
  filepath = os.path.join(directory, filename)
  if tf.gfile.Exists(filepath):
    return filepath
  if not tf.gfile.Exists(directory):
    tf.gfile.MakeDirs(directory)
  # CVDF mirror of http://yann.lecun.com/exdb/mnist/
  url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'
  _, zipped_filepath = tempfile.mkstemp(suffix='.gz')
  print('Downloading %s to %s' % (url, zipped_filepath))
  urllib.request.urlretrieve(url, zipped_filepath)
  with gzip.open(zipped_filepath, 'rb') as f_in, \
      tf.gfile.Open(filepath, 'wb') as f_out:
    shutil.copyfileobj(f_in, f_out)
  os.remove(zipped_filepath)
  return filepath


def dataset(directory, images_file, labels_file):
  """Download and parse MNIST dataset."""

  images_file = download(directory, images_file)
  labels_file = download(directory, labels_file)

  check_image_file_header(images_file)
  check_labels_file_header(labels_file)

  def decode_image(image):
    # Normalize from [0, 255] to [0.0, 1.0]
    image = tf.decode_raw(image, tf.uint8)
    image = tf.cast(image, tf.float32)
    image = tf.reshape(image, [784])
    return image / 255.0

  def decode_label(label):
    label = tf.decode_raw(label, tf.uint8)  # tf.string -> [tf.uint8]
    label = tf.reshape(label, [])  # label is a scalar
    return tf.to_int32(label)

  images = tf.data.FixedLengthRecordDataset(
      images_file, 28 * 28, header_bytes=16).map(decode_image)
  labels = tf.data.FixedLengthRecordDataset(
      labels_file, 1, header_bytes=8).map(decode_label)
  return tf.data.Dataset.zip((images, labels))


def train(directory):
  """tf.data.Dataset object for MNIST training data."""
  return dataset(directory, 'train-images-idx3-ubyte',
                 'train-labels-idx1-ubyte')


def test(directory):
  """tf.data.Dataset object for MNIST test data."""
  return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')

Tensorflow tips and tricks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorflow tips and tricks\n",
    "\n",
    "We mention a variety of details and tips and tricks to complement the introductory tutorial. These cover a wide variety of topics, including tensorflow internals, performance, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\wenda\\AppData\\Local\\conda\\conda\\envs\\idp\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scopes and collections\n",
    "\n",
    "Scopes and collections are two important notions that help tensorflow manage the graph. Scopes can be used to nest variables and operations in a principled way, organizing both the graph display on tensorboard and reusing operations in the appropriate place. Collections are used by tensorflow to keep track of tensors in a principled way. For example, this is how tensorflow can remember all the summaries that were created, or all the trainable variables that we have created."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "my_function_default/w:0\n",
      "my_function_default_1/w:0\n",
      "test1/w:0\n",
      "outer_scope/my_function_default/w:0\n",
      "outer_scope/my_function_default_1/w:0\n",
      "outer_scope/test1/w:0\n"
     ]
    }
   ],
   "source": [
    "# Let's have a look at the effect of scopes\n",
    "def my_function(x, name=None):\n",
    "    with tf.variable_scope(name, 'my_function_default'):\n",
    "        w = tf.get_variable(name='w', shape=[], dtype=tf.float32, trainable=True)\n",
    "        print(w.name)\n",
    "        \n",
    "        return w * x\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    x = tf.random_normal(shape=[])\n",
    "    \n",
    "    my_function(x)\n",
    "    my_function(x)\n",
    "    my_function(x, 'test1')\n",
    "    \n",
    "    with tf.variable_scope('outer_scope'):\n",
    "        my_function(x)\n",
    "        my_function(x)\n",
    "        my_function(x, 'test1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable variables: [<tf.Variable 'conv2d/kernel:0' shape=(3, 3, 3, 20) dtype=float32_ref>, <tf.Variable 'conv2d/bias:0' shape=(20,) dtype=float32_ref>]\n",
      "summary ops: [<tf.Tensor 'test:0' shape=() dtype=string>]\n"
     ]
    }
   ],
   "source": [
    "# Let's inspect some commonly used collections\n",
    "with tf.Graph().as_default():\n",
    "    x = tf.random_normal(shape=[32, 224, 224, 3])\n",
    "    \n",
    "    x = tf.layers.conv2d(x, filters=20, kernel_size=3, strides=1)\n",
    "    tf.summary.scalar('test', x[0])\n",
    "    \n",
    "    print('trainable variables: {0}'.format(list(tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES))))\n",
    "    print('summary ops: {0}'.format(list(tf.get_collection(tf.GraphKeys.SUMMARIES))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calling Python functions from Tensorflow\n",
    "\n",
    "Sometimes, you may have some functionality that relies on snome other python package, and thus can only be used from python. As we have mentioned already, the programming style of tensorflow requires us to describe computation in a symbolic fashion, which is incompatible with working on the values directly. Tensorflow has an \"escape hatch\" for this scenario `py_func`, although performance often suffers substantially (in particular, this cannot be run on GPU at all)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.5356552, 0.0012485808, 0.091488756, 0.5944572]\n"
     ]
    }
   ],
   "source": [
    "from scipy.special import airy\n",
    "\n",
    "# airy returns the Ai, Bi Airy functions and their derivatives (4 values total).\n",
    "\n",
    "with tf.Graph().as_default():\n",
    "    x = tf.random_normal(shape=[])\n",
    "    y = tf.py_func(airy, [x], [tf.float32] * 4, stateful=False)\n",
    "    \n",
    "    with tf.Session() as session:\n",
    "        print(session.run(y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Estimator Hooks\n",
    "\n",
    "We may sometimes be interested in running operations before or after each / some training steps. This can be accomplished in the estimator framework by creating custom hooks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class AddNoiseHook(tf.train.SessionRunHook):\n",
    "    def __init__(self):\n",
    "        # We will create this operation in `begin`\n",
    "        self._op = None\n",
    "    \n",
    "    def begin(self):\n",
    "        # create the operations we need here\n",
    "        with tf.variable_scope('add_noise_to_weights'):\n",
    "            ops = [tf.assign_add(v, 0.1 * tf.random_normal(shape=v.shape)) for v in tf.trainable_variables()]\n",
    "\n",
    "        self._op = tf.group(*ops, name='add_noise')\n",
    "    \n",
    "    def before_run(self, context):\n",
    "        # Ask estimator to also run our add noise op\n",
    "        return tf.train.SessionRunArgs(self._op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We reproduce some of the training code we had earlier to help our examples.\n",
    "def get_mnist_dataset():\n",
    "    \"\"\" This function creates a dataset which can be used to load data from the MNIST dataset. \"\"\"\n",
    "    def _format_image(raw_data):\n",
    "        image = tf.decode_raw(raw_data, tf.uint8)\n",
    "        image = tf.to_float(image)\n",
    "        image = tf.reshape(image, [28, 28, 1])\n",
    "        image = image / 255\n",
    "        return image\n",
    "    \n",
    "    def _format_label(raw_data):\n",
    "        label = tf.decode_raw(raw_data, tf.uint8)\n",
    "        label = tf.reshape(label, [])\n",
    "        return tf.to_int32(label)\n",
    "    \n",
    "    dataset_img = tf.data.FixedLengthRecordDataset('data/train-images-idx3-ubyte', 28 * 28, header_bytes=16)\n",
    "    dataset_img = dataset_img.map(_format_image)\n",
    "    \n",
    "    dataset_label = tf.data.FixedLengthRecordDataset('data/train-labels-idx1-ubyte', 1, header_bytes=8)\n",
    "    dataset_label = dataset_label.map(_format_label)\n",
    "    \n",
    "    return tf.data.Dataset.zip((dataset_img, dataset_label))\n",
    "\n",
    "def make_input_fn(repeat_count=None, shuffle_size=1000):\n",
    "    def input_fn():\n",
    "        dataset = get_mnist_dataset()\n",
    "        # Shuffle the dataset, and repeat as necessary\n",
    "        if shuffle_size is not None and shuffle_size > 0:\n",
    "            from tensorflow.contrib.data import shuffle_and_repeat\n",
    "            dataset = dataset.apply(shuffle_and_repeat(shuffle_size, repeat_count))\n",
    "        else:\n",
    "            dataset = dataset.repeat(repeat_count)\n",
    "        dataset = dataset.prefetch(128) # Prefetch enough for a single batch for performance\n",
    "        dataset = dataset.batch(batch_size=128) # Batch it up\n",
    "        dataset = dataset.prefetch(2) # Prefetch two batches to device.\n",
    "    \n",
    "        return dataset\n",
    "    return input_fn\n",
    "\n",
    "def _evaluate(fn_or_value):\n",
    "    if callable(fn_or_value):\n",
    "        return fn_or_value()\n",
    "    else:\n",
    "        return fn_or_value\n",
    "\n",
    "def model_fn(features, labels, mode, params):\n",
    "    # Here, the params parameter is passed in from tensorflow\n",
    "    images = features\n",
    "    images = tf.layers.flatten(images)\n",
    "    logits = tf.layers.dense(images, units=10)\n",
    "    \n",
    "    predictions = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "    accuracy = tf.metrics.accuracy(labels=labels, predictions=predictions)\n",
    "    \n",
    "    metrics = {'accuracy': accuracy}\n",
    "    \n",
    "    loss = tf.losses.sparse_softmax_cross_entropy(labels, logits)\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(\n",
    "        learning_rate=_evaluate(params['learning_rate']))\n",
    "    train_op = optimizer.minimize(loss, global_step=tf.train.get_or_create_global_step())\n",
    "    \n",
    "    # This gives tensorflow the description of what must be done\n",
    "    # to construct our model.\n",
    "    return tf.estimator.EstimatorSpec(\n",
    "        loss=loss,\n",
    "        mode=mode,\n",
    "        train_op=train_op,\n",
    "        predictions=predictions,\n",
    "        eval_metric_ops=metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: C:\\Users\\wenda\\AppData\\Local\\Temp\\tmp5qmv_t67\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'C:\\\\Users\\\\wenda\\\\AppData\\\\Local\\\\Temp\\\\tmp5qmv_t67', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x00000265AAA0AAC8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "\n",
      "---- Starting training ------\n",
      "WARNING:tensorflow:From C:\\Users\\wenda\\AppData\\Local\\conda\\conda\\envs\\idp\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the retry module or similar alternatives.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into C:\\Users\\wenda\\AppData\\Local\\Temp\\tmp5qmv_t67\\model.ckpt.\n",
      "INFO:tensorflow:loss = 2.4108741, step = 0\n",
      "INFO:tensorflow:global_step/sec: 255.48\n",
      "INFO:tensorflow:loss = 0.53534526, step = 100 (0.395 sec)\n",
      "INFO:tensorflow:global_step/sec: 240.16\n",
      "INFO:tensorflow:loss = 0.4098211, step = 200 (0.417 sec)\n",
      "INFO:tensorflow:global_step/sec: 287.098\n",
      "INFO:tensorflow:loss = 0.28915066, step = 300 (0.343 sec)\n",
      "INFO:tensorflow:global_step/sec: 295.293\n",
      "INFO:tensorflow:loss = 0.40237498, step = 400 (0.339 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 500 into C:\\Users\\wenda\\AppData\\Local\\Temp\\tmp5qmv_t67\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.11637541.\n",
      "---- Training Done ------\n",
      "\n",
      "---- Starting training with noise ------\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from C:\\Users\\wenda\\AppData\\Local\\Temp\\tmp5qmv_t67\\model.ckpt-500\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 501 into C:\\Users\\wenda\\AppData\\Local\\Temp\\tmp5qmv_t67\\model.ckpt.\n",
      "INFO:tensorflow:loss = 0.13959834, step = 500\n",
      "INFO:tensorflow:global_step/sec: 224.979\n",
      "INFO:tensorflow:loss = 1.8319739, step = 600 (0.446 sec)\n",
      "INFO:tensorflow:global_step/sec: 290.121\n",
      "INFO:tensorflow:loss = 2.2940233, step = 700 (0.345 sec)\n",
      "INFO:tensorflow:global_step/sec: 292.355\n",
      "INFO:tensorflow:loss = 3.5564454, step = 800 (0.340 sec)\n",
      "INFO:tensorflow:global_step/sec: 290.183\n",
      "INFO:tensorflow:loss = 4.7566943, step = 900 (0.345 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1000 into C:\\Users\\wenda\\AppData\\Local\\Temp\\tmp5qmv_t67\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 4.542456.\n",
      "---- Training Done ------\n"
     ]
    }
   ],
   "source": [
    "# Let's use our hook to mess things up.\n",
    "estimator = tf.estimator.Estimator(\n",
    "    model_fn=model_fn,\n",
    "    params={\n",
    "        'learning_rate': lambda: tf.train.inverse_time_decay(\n",
    "            0.1, tf.train.get_or_create_global_step(),\n",
    "            decay_steps=10,\n",
    "            decay_rate=1,\n",
    "            staircase=True)\n",
    "    })\n",
    "print('\\n---- Starting training ------')\n",
    "estimator.train(make_input_fn(), steps=500)\n",
    "print('---- Training Done ------')\n",
    "\n",
    "print('\\n---- Resuming training with noise ------')\n",
    "estimator.train(make_input_fn(), steps=500, hooks=[AddNoiseHook()])\n",
    "print('---- Training Done ------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High-Performance models in tensorflow\n",
    "\n",
    "Deep learning applications can be quite sensitive to performance, and there are numerous techniques used to enable faster training. A good overview is available on the [tensorflow website](https://www.tensorflow.org/performance/performance_guide). We discuss a couple here.\n",
    "\n",
    "## MKL-Optimized Tensorflow on CPU\n",
    "\n",
    "If you are using tensorflow on CPU, it can be very advantageous to obtain a version that has been compiled with Intel's MKLDNN library. In addition of substantially speeding up inference and training, it implements the NCHW convolutions on CPU (whereas default tensorflow can only perform NHWC convolutions on CPU)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-GPU and distributed training\n",
    "\n",
    "Deep-learning training is often easily parallelizable by splitting the minibatch computation across different devices. This may be several GPUs on the same machine, or even different machines. Tensorflow provides several strategies to do this in an efficient manner. Beyond doing it manually, the most recent (and easiest to use) API is the `tf.contrib.distribute` API, coming out in the next version (r1.8). The external library [horovod](https://github.com/uber/horovod) is also a good choice and leverages MPI on the cluster. Note that when you are using multi-gpu training, you should be careful of rescaling the learning rate and total steps as you are usually implicitly modifying the batch size."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Half-precision floating point\n",
    "\n",
    "On the recent P100 and V100 gpus, the performance of half-precision linear algebra is substantially higher than that of normal 32-bit single precision floating point operations. There are some subtleties to making use of half-precision floating point numbers without affecting performance, discussed [here](https://devblogs.nvidia.com/mixed-precision-training-deep-neural-networks/). Currently, implementing these in tensorflow is a manual process, although the developers plan to add automated tools for mixed precision training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: C:\\Users\\wenda\\AppData\\Local\\Temp\\tmph77t56tf\n",
      "INFO:tensorflow:Using config: {'_model_dir': 'C:\\\\Users\\\\wenda\\\\AppData\\\\Local\\\\Temp\\\\tmph77t56tf', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x00000265BC311908>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "\n",
      "---- Starting training ------\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into C:\\Users\\wenda\\AppData\\Local\\Temp\\tmph77t56tf\\model.ckpt.\n",
      "INFO:tensorflow:loss = 2.365325, step = 0\n",
      "INFO:tensorflow:global_step/sec: 216.46\n",
      "INFO:tensorflow:loss = 1.0045742, step = 100 (0.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 268.584\n",
      "INFO:tensorflow:loss = 0.96850574, step = 200 (0.372 sec)\n",
      "INFO:tensorflow:global_step/sec: 280.977\n",
      "INFO:tensorflow:loss = 0.8936647, step = 300 (0.361 sec)\n",
      "INFO:tensorflow:global_step/sec: 262.224\n",
      "INFO:tensorflow:loss = 0.8897525, step = 400 (0.376 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 500 into C:\\Users\\wenda\\AppData\\Local\\Temp\\tmph77t56tf\\model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 0.91317457.\n",
      "---- Training Done ------\n"
     ]
    }
   ],
   "source": [
    "# We need some helpers for mixed-precision training.\n",
    "\n",
    "def mixed_precision_getter(dtype=tf.float16):\n",
    "    \"\"\" A custom getter which implements mixed precision by creating\n",
    "    the variables and casting to mixed precision.\n",
    "    \n",
    "    \"\"\"\n",
    "    def custom_getter(getter, *args, **kwargs):\n",
    "        variable = getter(*args, **kwargs)\n",
    "        return tf.cast(variable, dtype)\n",
    "\n",
    "def model_fn_mixed(features, labels, mode, params):\n",
    "    # Here, the params parameter is passed in from tensorflow\n",
    "    images = features\n",
    "    \n",
    "    with tf.variable_scope('Network', custom_getter=mixed_precision_getter()):\n",
    "        images = tf.layers.flatten(images)\n",
    "        # Convert the input to float16\n",
    "        images = tf.cast(images, tf.float16)\n",
    "        \n",
    "        # Compute the network function\n",
    "        logits = tf.layers.dense(images, units=10)\n",
    "        \n",
    "        # Convert the output back to float32 for precision\n",
    "        logits = tf.to_float(logits)\n",
    "\n",
    "        predictions = tf.argmax(logits, axis=1, output_type=tf.int32)\n",
    "        accuracy = tf.metrics.accuracy(labels=labels, predictions=predictions)\n",
    "\n",
    "        metrics = {'accuracy': accuracy}\n",
    "\n",
    "        loss = tf.losses.sparse_softmax_cross_entropy(labels, logits)\n",
    "\n",
    "        optimizer = tf.train.GradientDescentOptimizer(\n",
    "            learning_rate=_evaluate(params['learning_rate']))\n",
    "        \n",
    "        # To avoid loss of precision, we scale up the loss and then scale down the gradients\n",
    "        LOSS_SCALE = 128\n",
    "        grads_and_vars = optimizer.compute_gradients(LOSS_SCALE * loss)\n",
    "        grads_and_vars = [(g / LOSS_SCALE, v) for (g, v) in grads_and_vars]\n",
    "        train_op = optimizer.apply_gradients(grads_and_vars, global_step=tf.train.get_or_create_global_step())\n",
    "\n",
    "    # This gives tensorflow the description of what must be done\n",
    "    # to construct our model.\n",
    "    return tf.estimator.EstimatorSpec(\n",
    "        loss=loss,\n",
    "        mode=mode,\n",
    "        train_op=train_op,\n",
    "        predictions=predictions,\n",
    "        eval_metric_ops=metrics)\n",
    "\n",
    "# Let's do mixed precision training.\n",
    "# In this simple case, guaranteed to be slower\n",
    "estimator = tf.estimator.Estimator(\n",
    "    model_fn=model_fn_mixed,\n",
    "    params={\n",
    "        'learning_rate': lambda: tf.train.inverse_time_decay(\n",
    "            0.1, tf.train.get_or_create_global_step(),\n",
    "            decay_steps=10,\n",
    "            decay_rate=1,\n",
    "            staircase=True)\n",
    "    })\n",
    "print('\\n---- Starting training ------')\n",
    "estimator.train(make_input_fn(), steps=500)\n",
    "print('---- Training Done ------')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TPUs\n",
    "\n",
    "Google has developed custom hardware, called [TPU](https://cloud.google.com/tpu/) available on GCP, for fast computation and large-scale inference. They are accesible through a similar API as the estimator API, but is somewhat more tricky and cumbersome. In particular, they require a careful separation of the host and TPU operations, which makes operations such as recording summaries somewhat more challenging."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Eager execution\n",
    "\n",
    "Eager execution is a relatively new tensoflow features which allows us to use tensorflow to do direct computations instead of computing a graph. It has some trade-offs in terms of performance but can offer a lot of flexibility. It is much more similar to pytorch. On the other hand, it has some drawbacks (for example, it does not work on TPU), and not all APIs work in the eager mode (you may often find newer APIs in `tf.contrib` which are compatible with eager).\n",
    "\n",
    "Note that once you enable eager mode, you cannot disable eager in the python session. You will need to restart your python kernel to restore the default mode. You must also enable eager execution before anything else you do. To run this last part of the notebook, you will need to restart your kernel now."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\wenda\\AppData\\Local\\conda\\conda\\envs\\idp\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(-0.8435967, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "x = tf.random_normal(shape=[])\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(-1.6871934, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(2 * x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Most of the dataset is Eager mode compatible.\n",
    "def get_mnist_dataset():\n",
    "    \"\"\" This function creates a dataset which can be used to load data from the MNIST dataset. \"\"\"\n",
    "    def _format_image(raw_data):\n",
    "        image = tf.decode_raw(raw_data, tf.uint8)\n",
    "        image = tf.to_float(image)\n",
    "        image = tf.reshape(image, [28, 28, 1])\n",
    "        image = image / 255\n",
    "        return image\n",
    "    \n",
    "    def _format_label(raw_data):\n",
    "        label = tf.decode_raw(raw_data, tf.uint8)\n",
    "        label = tf.reshape(label, [])\n",
    "        return tf.to_int32(label)\n",
    "    \n",
    "    dataset_img = tf.data.FixedLengthRecordDataset('data/train-images-idx3-ubyte', 28 * 28, header_bytes=16)\n",
    "    dataset_img = dataset_img.map(_format_image)\n",
    "    \n",
    "    dataset_label = tf.data.FixedLengthRecordDataset('data/train-labels-idx1-ubyte', 1, header_bytes=8)\n",
    "    dataset_label = dataset_label.map(_format_label)\n",
    "    \n",
    "    return tf.data.Dataset.zip((dataset_img, dataset_label))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dataset = get_mnist_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "iterator = tf.contrib.eager.Iterator(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(next(iterator))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}