CIFAR-10 CNN multi-GPU.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import datasets, layers, models\n",
    "from matplotlib import pyplot\n",
    "\n",
    "print(\"Tensorflow Version: \", tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Downlad the Data and inspect the Shape 🕵️‍</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Download the data:\n",
    "(train_images, train_labels), (test_images, test_labels) = datasets.cifar10.load_data()\n",
    "\n",
    "# Parse data as floats and normalize pixel values to be between 0 and 1\n",
    "train_images, test_images = train_images.astype(\"float32\") / 255.0, test_images.astype(\"float32\") / 255.0\n",
    "\n",
    "print(\"-------------------------\")\n",
    "print(\"train_images shape:\", train_images.shape)\n",
    "print(\"test_images shape:\", test_images.shape)\n",
    "print(\"-------------------------\")\n",
    "input_shape = (train_images.shape[1], train_images.shape[2], train_images.shape[3])\n",
    "print(\"image input_shape:\", input_shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Let's have a look at the Data 📷 </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "verbal_labels = ['airplane','automobile','bird','cat','deer','dog','frog','horse','ship','truck']\n",
    "print('Train: X=%s, y=%s' % (train_images.shape, train_labels.shape))\n",
    "print('Test: X=%s, y=%s' % (test_images.shape, test_labels.shape))\n",
    "# plot first few images\n",
    "pyplot.figure(figsize=(14,14))\n",
    "for i in range(36):\n",
    "    # define subplot\n",
    "    pyplot.subplot(6,6,i+1)\n",
    "    pyplot.grid(False)\n",
    "    # plot raw pixel data\n",
    "    pyplot.imshow(train_images[i])\n",
    "    pyplot.title(verbal_labels[train_labels[i][0]])\n",
    "# show the figure\n",
    "pyplot.subplots_adjust(hspace=0.35)\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Define the Model and it's Configuration. 🧠 </h1>\n",
    "\n",
    "<p>\n",
    "When training a model with multiple GPUs, you can use the extra computing power effectively by increasing the batch size. In general, use the largest batch size that fits the GPU memory, and tune the learning rate accordingly. See our article here how to do that: https://medium.com/@genesiscloud/caf8105ab6d7\n",
    "</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model configuration\n",
    "no_classes = 10\n",
    "no_epochs = 60\n",
    "batch_size = 2200\n",
    "\n",
    "\n",
    "# Build the model\n",
    "def get_compiled_model():\n",
    "    model = models.Sequential()\n",
    "    model.add(layers.Conv2D(128, (3, 3), activation='relu', input_shape=input_shape))\n",
    "    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    \n",
    "    model.add(layers.Conv2D(128, (3, 3), activation='relu'))\n",
    "    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    \n",
    "    model.add(layers.Conv2D(128, (3, 3), activation='relu'))\n",
    "    model.add(layers.MaxPooling2D(pool_size=(2, 2)))\n",
    "    \n",
    "    model.add(layers.Flatten())\n",
    "    \n",
    "    model.add(layers.Dense(256, activation='relu'))\n",
    "    model.add(layers.Dense(128, activation='relu'))\n",
    "    model.add(layers.Dense(no_classes, activation='softmax'))\n",
    "    \n",
    "    # Compile the model:\n",
    "    model.compile(optimizer='adam',\n",
    "                   loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "                   metrics=['accuracy'])\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Set up for multiple GPUs </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a MirroredStrategy:\n",
    "strategy = tf.distribute.MirroredStrategy()\n",
    "\n",
    "print('Number of devices: {}'.format(strategy.num_replicas_in_sync))\n",
    "print(\"-------------------------\")\n",
    "\n",
    "#nvidia-smi cmd call:\n",
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open a strategy scope.\n",
    "with strategy.scope():\n",
    "    # Everything that creates variables should be under the strategy scope.\n",
    "    # In general this is only model construction & `compile()`.\n",
    "    model = get_compiled_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Start Training 🏋️ </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train the model on all available devices: \n",
    "history = model.fit(train_images, train_labels, batch_size=batch_size, epochs=no_epochs, \n",
    "                        validation_data=(test_images, test_labels))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1> Visualize the Training History 📈 </h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Plot history: Loss\n",
    "pyplot.plot(history.history['val_loss'])\n",
    "pyplot.title('Validation loss')\n",
    "pyplot.ylabel('Loss value')\n",
    "pyplot.xlabel('No. epoch')\n",
    "pyplot.show()\n",
    "\n",
    "# Plot history: Val Accuracy\n",
    "pyplot.plot(history.history['val_accuracy'])\n",
    "pyplot.title('Validation accuracy')\n",
    "pyplot.ylabel('Accuracy value (%)')\n",
    "pyplot.xlabel('No. epoch')\n",
    "pyplot.show()\n",
    "\n",
    "test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "print(\"Final test accuracy: \", test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}