ResNet50V2_MP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.metrics import classification_report\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.applications import ResNet50V2\n",
    "from tensorflow.keras.layers import GlobalAveragePooling2D\n",
    "from tensorflow.keras.layers import Input\n",
    "from tensorflow.keras.layers import Dense\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.datasets import cifar10\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.0.0\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "# Verify if you are on TensorFlow 2.0 and \n",
    "# you have a GPU\n",
    "print(tf.__version__)\n",
    "print(tf.test.is_gpu_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load CIFAR10 dataset\n",
    "((trainX, trainY), (testX, testY)) = cifar10.load_data()\n",
    "\n",
    "# Scale the data to the range [0, 1]\n",
    "trainX = trainX.astype(\"float32\") / 255.0\n",
    "testX = testX.astype(\"float32\") / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert the labels from integers to vectors\n",
    "lb = LabelBinarizer()\n",
    "trainY = lb.fit_transform(trainY)\n",
    "testY = lb.transform(testY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize the label names for the CIFAR-10 dataset\n",
    "labelNames = [\"airplane\", \"automobile\", \"bird\", \"cat\", \"deer\", \"dog\",\n",
    "\t\"frog\", \"horse\", \"ship\", \"truck\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct the image generator for data augmentation\n",
    "aug = ImageDataGenerator(rotation_range=18, zoom_range=0.15,\n",
    "\twidth_shift_range=0.2, height_shift_range=0.2, shear_range=0.15,\n",
    "\t horizontal_flip=True, fill_mode=\"nearest\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Network surgery\n",
    "base_model  = ResNet50V2(include_top=False, weights='imagenet',\n",
    "                        input_tensor=Input(shape=(32, 32, 3)))\n",
    "x = GlobalAveragePooling2D()(base_model.output)\n",
    "x = Dense(len(labelNames), activation='softmax')(x)\n",
    "model = Model(inputs=base_model.input, outputs=x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_compiled_model(model, mp=False, loss=\"categorical_crossentropy\", opt=Adam()):\n",
    "    if mp:\n",
    "        opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, \"dynamic\")\n",
    "        model.compile(loss=loss, optimizer=opt, metrics=[\"accuracy\"])\n",
    "        return model\n",
    "    else:\n",
    "        model.compile(loss=loss, optimizer=opt, metrics=[\"accuracy\"])\n",
    "        return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "390/390 [==============================] - 122s 312ms/step - loss: 2.2830 - accuracy: 0.1727 - val_loss: 2.3745 - val_accuracy: 0.0974\n",
      "Epoch 2/10\n",
      "390/390 [==============================] - 120s 308ms/step - loss: 2.2140 - accuracy: 0.1927 - val_loss: 2.1835 - val_accuracy: 0.1971\n",
      "Epoch 3/10\n",
      "390/390 [==============================] - 118s 303ms/step - loss: 2.0187 - accuracy: 0.2366 - val_loss: 1.9088 - val_accuracy: 0.2614\n",
      "Epoch 4/10\n",
      "390/390 [==============================] - 118s 303ms/step - loss: 1.8463 - accuracy: 0.3078 - val_loss: 1.6410 - val_accuracy: 0.3730\n",
      "Epoch 5/10\n",
      "390/390 [==============================] - 118s 303ms/step - loss: 1.7361 - accuracy: 0.3574 - val_loss: 1.5417 - val_accuracy: 0.4205\n",
      "Epoch 6/10\n",
      "390/390 [==============================] - 118s 302ms/step - loss: 1.5846 - accuracy: 0.4290 - val_loss: 2.1784 - val_accuracy: 0.3349\n",
      "Epoch 7/10\n",
      "390/390 [==============================] - 121s 311ms/step - loss: 1.6669 - accuracy: 0.4289 - val_loss: 1.6821 - val_accuracy: 0.3843\n",
      "Epoch 8/10\n",
      "390/390 [==============================] - 123s 315ms/step - loss: 2.1562 - accuracy: 0.2380 - val_loss: 9.6379 - val_accuracy: 0.0956\n",
      "Epoch 9/10\n",
      "390/390 [==============================] - 121s 311ms/step - loss: 1.9744 - accuracy: 0.2603 - val_loss: 2.7121 - val_accuracy: 0.2434\n",
      "Epoch 10/10\n",
      "390/390 [==============================] - 122s 312ms/step - loss: 1.7970 - accuracy: 0.3238 - val_loss: 1.7978 - val_accuracy: 0.3333\n",
      "Training took 1201.7255418300629 seconds\n"
     ]
    }
   ],
   "source": [
    "# Train the model with mixed precision policy\n",
    "compiled_model = get_compiled_model(model, mp=True)\n",
    "start = time.time()\n",
    "H = compiled_model.fit_generator(\n",
    "\taug.flow(trainX, trainY, batch_size=128),\n",
    "\tvalidation_data=(testX, testY),\n",
    "\tsteps_per_epoch=trainX.shape[0] // 128,\n",
    "\tepochs=10,\n",
    "\tverbose=1)\n",
    "print('Training took {} seconds'.format(time.time()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "390/390 [==============================] - 114s 293ms/step - loss: 1.6858 - accuracy: 0.3705 - val_loss: 1.5552 - val_accuracy: 0.4233\n",
      "Epoch 2/10\n",
      "390/390 [==============================] - 113s 291ms/step - loss: 1.5919 - accuracy: 0.4078 - val_loss: 1.5426 - val_accuracy: 0.4395\n",
      "Epoch 3/10\n",
      "390/390 [==============================] - 113s 289ms/step - loss: 1.5044 - accuracy: 0.4440 - val_loss: 1.4609 - val_accuracy: 0.4591\n",
      "Epoch 4/10\n",
      "390/390 [==============================] - 115s 294ms/step - loss: 1.4244 - accuracy: 0.4779 - val_loss: 1.8710 - val_accuracy: 0.3954\n",
      "Epoch 5/10\n",
      "390/390 [==============================] - 114s 293ms/step - loss: 1.3605 - accuracy: 0.5058 - val_loss: 1.3105 - val_accuracy: 0.5408\n",
      "Epoch 6/10\n",
      "390/390 [==============================] - 121s 309ms/step - loss: 1.2926 - accuracy: 0.5291 - val_loss: 1.3165 - val_accuracy: 0.5358\n",
      "Epoch 7/10\n",
      "390/390 [==============================] - 113s 290ms/step - loss: 1.2344 - accuracy: 0.5531 - val_loss: 2.2528 - val_accuracy: 0.4126\n",
      "Epoch 8/10\n",
      "390/390 [==============================] - 112s 286ms/step - loss: 1.1772 - accuracy: 0.5748 - val_loss: 1.2994 - val_accuracy: 0.5536\n",
      "Epoch 9/10\n",
      "390/390 [==============================] - 109s 280ms/step - loss: 1.1272 - accuracy: 0.5948 - val_loss: 1.1036 - val_accuracy: 0.6098\n",
      "Epoch 10/10\n",
      "390/390 [==============================] - 111s 285ms/step - loss: 1.0885 - accuracy: 0.6105 - val_loss: 1.1919 - val_accuracy: 0.6050\n",
      "Training took 1134.8909366130829 seconds\n"
     ]
    }
   ],
   "source": [
    "# Train the model without mixed precision policy\n",
    "compiled_model = get_compiled_model(model, mp=False)\n",
    "start = time.time()\n",
    "H = compiled_model.fit_generator(\n",
    "\taug.flow(trainX, trainY, batch_size=128),\n",
    "\tvalidation_data=(testX, testY),\n",
    "\tsteps_per_epoch=trainX.shape[0] // 128,\n",
    "\tepochs=10,\n",
    "\tverbose=1)\n",
    "print('Training took {} seconds'.format(time.time()-start))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}

Note that just calling opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, "dynamic") alone does not activate mixed precision. Moving forward, you should call tf.keras.mixed_precision.experimental.set_policy('mixed_float16') before constructing the model. To fix the current "dtype" error, set the dtype of the softmax layer to tf.float32 manually.

Thanks for passing this along @tlkh. Will incorporate your suggestions and report back.

@sayakpaul can you also test with larger images? CIFAR's 32x32 is "too small".

@tlkh will do! Your materials here (https://github.com/NVAITC/pycon-sg19-tensorflow-tutorial) are just fantastic. Do you mind if I used them in my decks with a proper citation?

@sayakpaul no problem! Please feel free to modify and use for your own purposes.