sampathweb/cifar10-fastai-custom-model-scratch.ipynb Secret

## cifar10-fastai-custom-model-scratch.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Put these at the top of every notebook, to get automatic reloading and inline plotting\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This file contains all the main external libs we'll use\n",
    "from fastai.imports import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.transforms import *\n",
    "from fastai.conv_learner import *\n",
    "from fastai.model import *\n",
    "from fastai.dataset import *\n",
    "from fastai.sgdr import *\n",
    "from fastai.plots import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_path = \"/workspace/datasets/cifar-10/\"\n",
    "label_csv = os.path.join(folder_path, \"trainLabels.csv\")\n",
    "sz = 32\n",
    "bs=32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n",
    "stats = np.array([0.5, 0.5, 0.5]), np.array([0.5, 0.5, 0.5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_from_csv(stats, sz, bs, folder_path, label_csv, tfms, skip_header=True, suffix=\".jpg\",\n",
    "                      train_folder=\"train\", test_folder=None, val_pct=0.2,\n",
    "                      num_workers=8):\n",
    "\n",
    "    n = len(list(open(label_csv)))\n",
    "    if skip_header:\n",
    "        n -= 1\n",
    "    val_idxs = get_cv_idxs(n, val_pct=val_pct)\n",
    "\n",
    "    return ImageClassifierData.from_csv(folder_path, \n",
    "                                        train_folder, \n",
    "                                        label_csv, \n",
    "                                        tfms=tfms,\n",
    "                                        skip_header=skip_header,\n",
    "                                        suffix=suffix, \n",
    "                                        val_idxs=val_idxs, \n",
    "                                        test_name=test_folder,\n",
    "                                        bs=bs,\n",
    "                                        num_workers=num_workers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "aug_tfms = [RandomFlipXY()]\n",
    "tfms = tfms_from_stats(stats, sz, aug_tfms=aug_tfms, max_zoom=1.05, pad=0, crop_type=CropType.NO, tfm_y=None)\n",
    "\n",
    "dataset = get_data_from_csv(stats, sz, bs, folder_path, label_csv, tfms, suffix=\".png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Instantiate the Model and Learner"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Cifar10Net(nn.Module):\n",
    "    \"\"\"Basic Custom Conv Net\n",
    "    \"\"\"\n",
    "    def __init__(self, *args, **kwargs):\n",
    "\n",
    "        super().__init__(*args, **kwargs)\n",
    "\n",
    "        # 3 x 32 x 32 -> 64 x 32 x 32\n",
    "        self.conv1 = nn.Conv2d(3, 64, 3, padding=1)\n",
    "        # 64 x 32 x 32 -> 128 x 16 x 16\n",
    "        self.conv2 = nn.Sequential(\n",
    "            nn.Conv2d(64, 128, 3, padding=1), \n",
    "            nn.MaxPool2d(2)\n",
    "        )\n",
    "        # 128 x 16 x 16 -> 256 x 8 x 8\n",
    "        self.conv3 = nn.Sequential(\n",
    "            nn.Conv2d(128, 256, 3, padding=1),\n",
    "            nn.MaxPool2d(2)\n",
    "        )\n",
    "        # 256 x 8 x 8 -> 10 x 8 x 8\n",
    "        self.conv_out = nn.Conv2d(256, 10, 3, padding=1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        \n",
    "        x = F.relu(self.conv1(x))  # Relu on Conv\n",
    "        x = F.relu(self.conv2(x))\n",
    "        x = F.relu(self.conv3(x))\n",
    "        x = F.relu(self.conv_out(x))\n",
    "        x = F.avg_pool2d(x, 8)  # Global Average pooling on 10 x 8 x 8\n",
    "        x = x.view(x.size(0), -1)\n",
    "        return F.log_softmax(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Cifar10Net()\n",
    "cifar_model = BasicModel(to_gpu(m), name='cifar10')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner(dataset, cifar_model)\n",
    "learn.unfreeze()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LR Finder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d7246497a514b22a34fa7bd3ae5c9aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       2.30271  2.30259  0.10154]                         \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYwAAAEOCAYAAACaQSCZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3Xl8XNV58PHfMzMa7ZK1eZN3bGMD\nBts4xMSsISELNGShCWlCQkJC6JsESElKljZNk74pfdvSrEAoJCUtSwMYwhbCEiAQVtt4ww7GNjaW\nLduSrH2b7Xn/uFfyWJbkkT1Xd5bn+/nMR3fuPXfmObrSPHPuufccUVWMMcaYIwn4HYAxxpjsYAnD\nGGNMSixhGGOMSYklDGOMMSmxhGGMMSYlljCMMcakxBKGMcaYlFjCMMYYkxJLGMYYY1JiCcMYY0xK\nQn4HkE61tbU6a9Ysv8MwxpissXr16mZVrUulbE4ljFmzZrFq1Sq/wzDGmKwhIjtTLWunpIwxxqTE\nEoYxxpiUWMIwxhiTEksYxhhjUmIJwxhjTEosYRhjjEmJJQxjjAGaOvt5fU87Nm31yHLqPgxjjDla\nf//ARh57fS+TK4o4b+FEzjl+IstmVlFVGvY7tIxhCcMYY4ADPRGmVxdz4pRK7n9tN3e8/DYA8yaW\ncerMKhZOqeD4yeUsmFzOhJL8TCKWMIwxBuiPJZhTW8bNl55KXzTO+oZ2Xt1xgFd3HODRDY3c/equ\nwbK1ZYVMry5mWlUJ9ROKmVZVzOSKIqrLwlSXhKkuC1NeGEJEfKxR+nmWMERkOvBrYDKQAG5R1R8P\nKXMR8AN3ewy4RlWfd7d9Fvg7t+g/qertXsV62/NvcfqcGk6YWpFS+URCiSWUcOjQLiBVpb03SkKh\nuz9GTyRONJ4gHAoQiSUoLQzR1hMhodDWE+GVHQcoLghSGg4xsaKQ+gnFFBUEae2JEBAhEk/Q3hOl\nqjRMNJYgllAi8QRBESqKQyQUZlaXEAwICVWKw0HiCWVPWx8Hup1vS6rQF42zq7WX9p4IFcUFdPfH\nicTilBSGqCgKMbOmlLryQuIJpS8aH6xPeVEBVSUFOfdHb8xw+qNxiioKASgqCHLa7GpOm10NOP/b\n+zr6+fPeDrbs6+TNfV3sbutlfUMbj21sJBo/vN+jIChUlYQpLwpRWhiiNByitDBISXjgeZCigiAF\nwQAFISEcDDjLwQAFQSEcOvg8FBACASEoQiAAQRGCSevCoQALp6T2+XUsvGxhxIBrVXWNiJQDq0Xk\nCVXdlFTmKeBBVVURORn4DbBARKqBfwCWAeru+6CqtqY7yO7+GD96Ygs/6I9x5rxaeiNx3j7QQ2lh\niHAwwPzJ5exu7aGqJMy+zj76ognebulBUU6dWUVNaSGFBQEa2/rY1NhBe2803SH6rqY0TF25849U\nV15IKCBMriymNBwkoTB1QhEzqkuYUVPC5IoiygpDhIJ2PYXJLv2xBIWh4LDbRITJlUVMrizinOMn\nHrItnlCaOvvZ1+F8UWvpjtDq/jzQ3U93f5zuSIye/jh72vroicTojsTp7o8Rcb8IHqvaskJW/d17\njvl1jsSzhKGqjUCju9wpIpuBemBTUpmupF1KcZIDwPuAJ1T1AICIPAG8H7gr3XGWFoZ46utnc83d\na1mzs5UFUyo4cWoFsYTS0Rdj3a42JlcUsb25m+KCIDOqS1gyfQLhUIDX93SwdlcbqkpdRRHvO3ES\nM2tKCQWEqtIwpeEQoaAQjScIiNDS1c/kyuLBbw+nzqxC1flDberso6G1l/beKFMnFJNIKCJCSThI\nbzROOBggFBQKQwHae6PEE5BQpaG1l0RCCQaEnkiMQECYWllMRXEBe9p6CQWc95pWVcKEkgK6+mOo\nQmlhkP5ogs6+GG8f6KGps49QMEBJOOgeG9jb0Ud/NE5TV4Smzj5A2N/ZR380wbqG9sHWSE8kftjv\ndfH0CZxUX8GUymKi8QSqMLGikANdERQIBYXF0ydwyrQJlBbamVHjv75onMLQ2L/oBAMHk8nRSLhn\nDqLxBNG4Eo0niMQOfR6NJ0ioEk84CcpZVuKqJBI6bl/QxuU/VURmAUuAl4fZ9hHgn4GJwAXu6npg\nV1KxBnfdcK99BXAFwIwZM44qvonlRdz5xeVHtW86FBUEqSwuYO7E8rS+7qkzqw5bN2mYcoumVR71\ne6gqbT1R3j7Qw67WHrY3ddPWE2XjnnZ++9oeOvtjo+5fGApw4tQK5k8qp6jAaaIXhgKoKpMri6mv\nKuads6spKhj+m58x6dIfS/jydxYICEWBYFb8jXueMESkDLgPp3+iY+h2Vb0fuF9EzsLpz3gPMNxJ\n82Hbbap6C3ALwLJly+wC6nEm4rSmqkrDnDJ9wiHbBr45BURo743SF40zsaKQjt4YCVU2N3bw7JYm\n1u1qY+VruykIiPtNSxFxWjngnAsuKghSVRKmICiUFYZo6uxn7qRy3jm7muVzaoZNjsakYmdLN79/\nfS+dfdGjamHkE08ThogU4CSLO1R15WhlVfWPInKciNTitCjOSdo8DXjGqziNNwa+OQGDfSDOsrNu\nUsXB88GRWIKCoJBQiMYTBAPC/s5+Nu3pYNXOAzR3RuiLxunoi9LWE2ViRRGb9rTzxy1NAMysKWH5\n7BrOOb6Od86podqunTejaOnq597VDTy4bg+v73G+x86bWMZ5C4drg5sB4tVdjeJcWnM7cEBVrxmh\nzFxgm9vpvRR4CCc5VAGrgaVu0TXAqQN9GiNZtmyZ2gRK+SOeUN5q7ubZLU3c+fJOtjV1D24LBYRz\njp/IBxdN5iNL6onGFUXp6I1RWxa2K7/y1Ot72rntubd4eH0jkXiCxdMncMGiKXxg0WSmVZX4HZ4v\nRGS1qi5LqayHCeMM4DlgA85lswDfBmYAqOrNInId8BkgCvQC30i6rPbzbnmA/6uqvzrSe1rCyG99\n0ThP/3k/L25v4e5XdhGJJw7ZPnCa69zj67hsxWzOnp/SrJQmB2zd38kNT2zh0Q17KQ0H+ejSaVx6\n+kzmT0pvv2E2yoiE4QdLGGZAPKEIcPuLO7jrlbc5fnIFdWWFrNp5gPUN7QBc9e65fO298621kcN6\nIjH+44kt/PJPOygKBbj8jNlcfuYcKosL/A4tY4wlYdj1jCYnBQNOEvjcitl8bsXsQ7btbOnmuvvW\n85M/bOWtlh5++sklfoRoPPbS9hau/c06drf18snTpvP184+npqzwyDuaEVnCMHlnZk0pP/urpVz2\nq1d4aN0eKopCfOms46gsLqCyxL55ZrtEQrnxma3c8MQWZtaUcs+Vp/OOWdV+h5UT7JSUyVttPRG+\nfs96nty8b3Ddl86aw7c+uNDHqMyx6IvGuequ13h80z4+dMpUfvjRRZTZjaGjslNSxqRgQkmYWz+7\njJe2t/Dzp7eyemcrv/jjdna39XLa7Go+c/osv0M0Y9DWE+Hy21ex5u1WvnvhCXxuxSzrn0ozSxgm\n7y2fU8PyOTW090Y55R8f5+H1jTy8vpELT55q93NkiY6+KJ++7WW27Ovi53+1lA8umuJ3SDnJbms0\nxlVZXMBX3z2XUnc8raeSTlWZzNUTifG5X73KG3s7+cWnT7Vk4SFLGMYkufb849nwvfcB8I171/Po\nhkafIzKjSSSUq+5ay2tvt/KTS5Zw7oKJR97JHDVLGMYMEQgIHzplKgA/eepNn6Mxo/npH7by5OZ9\n/MNfnMgHrGXhOUsYxgzj3z9+Cu89YRJ/3ttprYwMtb6hjR89tYX3LJzEZ06f6Xc4ecEShjHDKAgG\nuPq8eQB8a+WGQ2YiNP7rjcS58r9XowrXf2yRXQ01TixhGDOCk+orufUzy2jvjbLg7x9jfUOb3yEZ\n1/+8tJM97X1cfd48au3u7XFjCcOYUZx+XM3g8sU3vchv1+4mnoYpNc3Ra+2O8JM/vMm7jqvha++d\n73c4ecUShjGjKC0MccHJTmdqJJ7g6rvXcvntr/ocVX679fntdPbF+Mb7jvc7lLxjCcOYI7jh46fw\n8FfPGHz+zBtN1qfhozf2drJgcjlLZtgsi+PNEoYxR1AYCnJSfSU7rr+A2z7rDLlz63PbfY4qf3X0\nxWx4cp9YwjBmDM49fiJzJ5bxb49vYePudr/DyUsdvVHKiyxh+MEShjFjEAgIN37KmTn4pme3+RxN\nfursi1FRZMPg+cEShjFjNH9SOV86aw6PrG+ksb3X73DyTmdflAo7JeULSxjGHIWPLp0GwI+ftKFD\nxlMioXT1xyi3FoYvLGEYcxSOn1zOwikVbN3f5XcoeaU7EiOhWMLwiSUMY47S4ukT2NZkCWM8dfbF\nAKzT2yeWMIw5SsfVldLaE+VAd8TvUPLGwJVpdTYciC8sYRhzlI6rKwOw01Lj6MZntjGjuoSzj6/z\nO5S8ZAnDmKO0cEoFIvDcm01+h5IX+qJx1je08eHFUykI2keXH+y3bsxRmlxZxAlTKnh2iyWM8fCH\nP+8noXDytAl+h5K3PEsYIjJdRJ4Wkc0i8rqIXD1MmU+JyHr38YKInJK0bYeIbBCRtSKyyqs4jTkW\nJ06tYH1DO7/b0GjjS3lo6/4urr77NebUldo0rD7ysoURA65V1YXAcuDLInLCkDJvAWer6snAD4Bb\nhmw/V1UXq+oyD+M05qhd9/4FAPz1HWu47Fev+BxN7npsYyPRuPIfH19MMGCTJfnFs4uZVbURaHSX\nO0VkM1APbEoq80LSLi8B07yKxxgv1CRdrfPS9gM+RpLb1je0M7OmhFOm2+koP41LH4aIzAKWAC+P\nUuxy4HdJzxV4XERWi8gV3kVnzLF57m/P5aLFUwF4YVuzz9HknkgswZ+2NnPG3Fq/Q8l7nicMESkD\n7gOuUdWOEcqci5MwrktavUJVlwIfwDmdddYI+14hIqtEZFVTk3U+mvE3vbqE71ywEIAtezt9jib3\nbN3fRXckzvI5NUcubDzlacIQkQKcZHGHqq4coczJwK3ARaraMrBeVfe4P/cD9wOnDbe/qt6iqstU\ndVldnV2bbfxRV1ZIcUGQXa02GGG6PbFpHwBz6kp9jsR4eZWUALcBm1X1hhHKzABWApeq6pak9aUi\nUj6wDJwPbPQqVmOOlYhQX1XMM2/sR9Xm/E6nP21rpro0zILJFX6Hkve8HMFrBXApsEFE1rrrvg3M\nAFDVm4HvAjXAjU5+IeZeETUJuN9dFwLuVNXHPIzVmGNWWxbmpe0HeH5rM2fOs9ZuOsQTysbd7fzl\nqdPs6qgM4OVVUs8Dox5hVf0C8IVh1m8HTjl8D2My1z99eBHvueFZNjd2WMJIk/2dffRE4syfXO53\nKAa709uYtJk7sYzq0jBvNXf7HUrOiMWd03thGwokI9hRMCaNZteWWsJIo1jCSRihoJ2OygSWMIxJ\no1k1ljDSKe4mjGDAPqoygR0FY9Jodm0J+zr66YnE/A4lJwwmDLEWRiawhGFMGs2udebI2NHc43Mk\nuSGWSADYFVIZwhKGMWk0q7YEgHUNbT5HkhvcfEHIEkZGsIRhTBrNqnHuRv7Wyg12A18aWAsjs1jC\nMCaNSgtDTKpwRrDd1mRTtx6rg53eljAygSUMY9Lsxk+dCsDX71nvcyTZbyBh2CmpzGAJw5g0Wzpj\nArVlYdbuamNfR5/f4WS1gYQRsISRESxhGJNmIsKFJzvzYzy5eZ/P0WS3mLUwMoolDGM88PcXOrMR\n7zpgw50fi7haH0YmsYRhjAeCAWFyRRHNXf1+h5LVHl7XCFjCyBSWMIzxSHlRiHtXN9hd30eppauf\n+9Y0AJYwMoUlDGM8cvGp0wC4+dntPkeSnXoi8cHlgA0NkhEsYRjjkS+dfRzLZlbxp63NfoeSlfpj\nicHl5ORh/GMJwxgP1ZSF6eyL+h1GVookJYzSwqCPkZgBljCM8VBFUQGdfdaHcTQicSdhXHXePJvP\nO0NYwjDGQxXFBXT0WgvjaPRHndNQp82q9jkSM8AShjEeqi4N0x2J02vn4MdsoIURDtnHVKawI2GM\nh6ZUFgHQ2G438I3VQB+GJYzMYUfCGA/NnehMqLRqR6vPkWSfwYQRtI+pTGFHwhgPLaqvZHZtKb/b\n2Oh3KFmn3e37KS8K+RyJGWAJwxgPiQiTKgp5+o0muvvtaqmxeKu5m3AwwNQJxX6HYlyWMIzxmODc\npXzv6gafI8ku25q6mFVbYsOCZBBLGMZ47MZPLQVgzdvWjzEW25u6mVNb5ncYJolnCUNEpovI0yKy\nWUReF5GrhynzKRFZ7z5eEJFTkra9X0TeEJGtIvJNr+I0xmtVpWFOn1PD7la7UipV0XiCtw/0cNzE\nUr9DMUm8bGHEgGtVdSGwHPiyiJwwpMxbwNmqejLwA+AWABEJAj8HPgCcAHxymH2NyRrVZWFauiN+\nh5E13j7QQyyh1sLIMJ4lDFVtVNU17nInsBmoH1LmBVUdaKe/BExzl08DtqrqdlWNAHcDF3kVqzFe\nqykN02JzY6Rs2/4uAObUWQsjk4xLH4aIzAKWAC+PUuxy4Hfucj2wK2lbA0OSjTHZpLo0TEdfjGg8\nceTChu3N3QDMqbMWRibxPGGISBlwH3CNqnaMUOZcnIRx3cCqYYrpCPteISKrRGRVU1NTOkI2Ju1q\nSsMAtNppqZRsb+qitqyQyuICv0MxSTxNGCJSgJMs7lDVlSOUORm4FbhIVVvc1Q3A9KRi04A9w+2v\nqreo6jJVXVZXV5e+4I1Jo5qyQgDrx0jR9qZuOx2Vgby8SkqA24DNqnrDCGVmACuBS1V1S9KmV4F5\nIjJbRMLAJcCDXsVqjNeq3RbGge4I96zaxdIfPMGb+zp9jipztXRHmFhe6HcYZggvWxgrgEuBd4vI\nWvfxQRG5UkSudMt8F6gBbnS3rwJQ1RjwFeD3OJ3lv1HV1z2M1RhPDZySau7q5xv3rudAd4SP3PiC\nz1Flrp5IjNKwDQmSaTw7Iqr6PMP3RSSX+QLwhRG2PQo86kFoxoy7gRbG828enK61qz9GU2c/dfZN\n+jA9/XFKbJa9jGN3ehszDiaUhBGBe4YMD/LMG/t9iihzqSrdkRglYUsYmcYShjHjIBgQqkqcVkZJ\nOMgr3z6PYEDY0dLtc2SZpz+WIKFQYqekMo4lDGPGSVWJc4no0hlVTKwoYkplkQ0XMozWHudKsgq7\npDbjWMIwZpwMTNO6t6MPgPoJxTRYwjjMG3udq8fmTbSb9jKNJQxjxsmPLlkCwBVnzQFgWlUJu9ss\nYQw1kDAWTC73ORIzlJ0kNGacnDa7mh3XXzD4vL6qmMb2Ptp6Ikxw+zcMrN7ZyrSqYvudZCBrYRjj\nk1k1JQBcfPOLfPWu1+iPxX2OKDNsb+5mUX2l32GYYVjCMMYnF548FYCt+7t4aN0etuzt8jmizNDR\nG7UxpDKUJQxjfBIOBfj4smmDz7c1WcIA6OyLUV5kZ8szkSUMY3z0w48s4uz5zqCZNuc37G7rpTca\np6zQWhiZyBKGMT4KBQPc/vnTAHh+azOv7jjgc0T+evUtp/7Tq4t9jsQMJ6WEISJXi0iFOG4TkTUi\ncr7XwRmTb7bn+WmpZndWwvMWTPI5EjOcVFsYn3cnPzofqAM+B1zvWVTG5JmHvnIGAB29MZ8j8VdL\nd4SCoFBRbH0YmSjVhDEw6uwHgV+p6jqOMBKtMSZ1J9VXEBBo7436HYqvmjv7qSktxJlOx2SaVBPG\nahF5HCdh/F5EygGbnNiYNBEREgo/e3ors775CB19+Zk4Wroj1JbbDXuZKtWEcTnwTeAdqtoDFOCc\nljLGpMmSGRMGl+9ZlZ9XTDV3OS0Mk5lSTRinA2+oapuIfBr4O6Ddu7CMyT93fXE575xdDThzQuSj\nlq4ItWWWMDJVqgnjJqBHRE4B/hbYCfzas6iMyUNFBUHuvmI5BUGhuSvidzjjTlVp6uqntsxOSWWq\nVBNGTJ2vPBcBP1bVHwM2lKQxaSYi1JQWDl5emk96o3EisYQNOpjBUr12rVNEvgVcCpwpIkGcfgxj\nTJrVlofzMmFE485puHDI7ifOVKkemU8A/Tj3Y+wF6oF/9SwqY/LYxPIidjR3510/RizuXHhZELRL\najNVSgnDTRJ3AJUiciHQp6rWh2GMB1bMrWVHS8/gzHz5IpZwEmQoYC2MTJXq0CAfB14B/hL4OPCy\niFzsZWDG5Kv5k5ypSXc09/gcyfiKui2MkLUwMlaqfRjfwbkHYz+AiNQBTwL3ehWYMflqVk0pADta\nujn9uBqfoxk/sfhAC8MSRqZKte0XGEgWrpYx7GuMGYOpE4oJBwPsaOn2O5RxFUsMtDDsoyVTpXpk\nHhOR34vIZSJyGfAI8OhoO4jIdBF5WkQ2i8jrInL1MGUWiMiLItIvIl8fsm2HiGwQkbUisirVChmT\n7YIBYVp1MbsO5NcpqYE+jAJrYWSslE5Jqeo3RORjwAqcQQdvUdX7j7BbDLhWVde4Y0+tFpEnVHVT\nUpkDwFXAh0d4jXNVtTmVGI3JJdOqSmho7fU7jHE1eErKWhgZK+UxhFX1PuC+MZRvBBrd5U4R2Yxz\nOe6mpDL7gf0ickHKERuTB2pKw7zVnF9zYwx2elsLI2ONmjBEpBMY7mJwAVRVK1J5ExGZBSwBXh5D\nbAo8LiIK/EJVbxnDvsZkteJwkF0Heunqj1FWmB9zQwxeVmtXSWWsUdt+qlquqhXDPMrHkCzKcFom\n17iTMKVqhaouBT4AfFlEzhrh9a8QkVUisqqpqWkML29M5lq9oxWAHz2xxedIxs/3H3JOPth9GJnL\n0yMjIgU4yeIOVV05ln1VdY/7cz9wP3DaCOVuUdVlqrqsrq7uWEM2JiP88KOLAOjsy58Z+DbsdgbA\n7urPnzpnG88ShjhTZt0GbFbVG8a4b6nbUY6IlOJMDbsx/VEak5lOnVnF8ZPKae3Jj1FrW7sP1rPQ\nxpLKWF6eHF2BM1jhBhFZ6677NjADQFVvFpHJwCqgAkiIyDXACUAtcL87TWMIuFNVH/MwVmMyTm15\nmKY8GYTwmv9dO7h85rxaHyMxo/EsYajq8xxh3m93jKppw2zqAE7xIi5jskVtWSFr3m71O4xxsWVf\nJwDL51TbfN4ZzNp+xmSourJCmjtz/5RUIqG09kQ4c14tP75kid/hmFFYwjAmQ1WXhemNxumLxv0O\nxVMb97TTF03wsaXTmFRR5Hc4ZhSWMIzJUAP3X+T6VUO7Djh3tC+YYpN4ZjpLGMZkqJKwkzB6+nO7\nhdHS7XTs15QW+hyJORJLGMZkqLLCIJD7LYzmzn5EoLrU5vLOdJYwjMlQgy2MSI4njO4I1SVhgjaG\nVMazhGFMhip1WxjdkRw/JdXVT02ZtS6ygSUMYzJUqdvp3Z3jp6RauiLUlln/RTawhGFMhioN50fC\naO7qp8YSRlawhGFMhioJO6ek3tjb6XMk3mrpilBjHd5ZwRKGMRlq4JTUrc+/5XMk3umLxunsj1FX\nbi2MbGAJw5gMlTxqa67e7d3ijlJrLYzsYAnDmAwlInz13XMB+PIda3h2SxPtvVGfo0qvFnc0XuvD\nyA6WMIzJYOctnATAU3/ez2d/+QpX/vdqnyNKr50tPQBMqbQxpLKBJQxjMtjsmtJDnq/emVvDnf92\n7R4KgsJxdWV+h2JSYAnDmAxWWVJwyPNcmypi0552zp5fR7F7RZjJbJYwjMlwT/7N2YPL/bEE//Wn\nt4jEEj5GlB7d/TH2tPexePoEv0MxKbKEYUyGO66ulHDSFVPfe2gTd73yto8RpcfW/V0AzJ1op6Oy\nhSUMYzKciPDGD97PPVeePriuqTP75/pev7sdgBOnVvociUmVJQxjsoCIsKj+4AfrwBwS2Wzt223U\nlIaZVlXsdygmRZYwjMkSRQVBnvvbc6krL+SuV3bxzfvW+x3SMdm6v5MTplYgudaTn8MsYRiTRaZX\nl9Dq3h1996u7SCSUWDyBqvoc2dg1tPZa6yLLWMIwJsv8+vOnDS7/9A9bmfud3/EPD77uY0Rj1xuJ\n09IdYVpVid+hmDGwhGFMlnnX3Fru+uJyAP7jyS0A/PrFnX6GNGZ3v+pc5VU/wVoY2cQShjFZ6B2z\nqg55Hg4GiGbRqanG9j4A3jW3xudIzFhYwjAmC4WCAcqLnOHP33fiJCLxBKt2tDL7W4+yck2Dz9Ed\nWX80TmVxARPLbQypbOJZwhCR6SLytIhsFpHXReTqYcosEJEXRaRfRL4+ZNv7ReQNEdkqIt/0Kk5j\nstU/ffgkvnvhCVx93nwAntq8D4DvP7zJz7BScvuLO3Nu5N18EPLwtWPAtaq6RkTKgdUi8oSqJv81\nHwCuAj6cvKOIBIGfA+8FGoBXReTBIfsak9cuWlwPQCSWIBQQVrkDE7b1RFHVjLxc9YWtzVy3Mrsv\nB85nnrUwVLVRVde4y53AZqB+SJn9qvoqMPSrxmnAVlXdrqoR4G7gIq9iNSabhUMBjqsrY+2utsF1\nK9fszsj+jO8++Dq7DvQC8MUzZ/scjRmrcenDEJFZwBLg5RR3qQd2JT1vYEiyMcYctGBK+SHPr71n\nHX/1n86/W28kzr889mc6+vw/BTTQ7wLwl8um+xiJORqeJwwRKQPuA65R1Y5Udxtm3bBfl0TkChFZ\nJSKrmpqajjZMY7LagskVh617cXsL3/3tRm56dhs3PbONX7+wY/wDGyJ5lN1J1uGddbzsw0BECnCS\nxR2qunIMuzYAyV8/pgF7hiuoqrcAtwAsW7Ys89rgxoyD+ZOGH/E1+f6Mjr7YeIUzoraeg62coXN9\nmMzn5VVSAtwGbFbVG8a4+6vAPBGZLSJh4BLgwXTHaEyuOHNeHXNqS7nirDkjlhkYTtwvb+7rZHdb\nr68xmGPjZQtjBXApsEFE1rrrvg3MAFDVm0VkMrAKqAASInINcIKqdojIV4DfA0Hgl6qaXWMfGDOO\nwqEAf/j6OQBcunwmB7ojXPTzPx1SZnervx/Wz245eMr4l5ct8zESc7Q8Sxiq+jzD90Ukl9mLc7pp\nuG2PAo96EJoxOW16dQnTq0v47ZdXsH53O3//wEYA9nf2+RrXtqYuAgLbfvjBjLzk1xyZp30Yxhj/\nnDJ9AgunVNDeE2F7czcr1+wmEkscMnvfeLrrFefCR0sW2cuGBjEmh4VDAb7y7nm8Y1Y1AM1d2T/x\nkvGPJQxj8kBdWSHg79SupeFwdXuyAAAQd0lEQVSg3ayX5SxhGJMHplU7w4jvaOn2LYZYQgkG7CMn\nm9nRMyYPzK0rozAUYH1Du28xxBJKKGD9F9nMEoYxeSAUDHDi1AqeeWM/8cT439+qqsQTStASRlaz\nhGFMnpg/qZxtTd0s/sfHx31gwoEkZS2M7GYJw5g88e4FEwHo7I/R3BUZ1/eOuQkjGLSEkc0sYRiT\nJ84/cTLXf3QRAOsb2o5QOr2shZEbLGEYk0c+srSeYNJkS+NlsIVhV0llNTt6xuSRwlCQssIQNz2z\njb5ofNze11oYucEShjF5ZmAu7RXX/4FoPHGE0ukRSzjvY1dJZTdLGMbkmTl1pQC0dEf4xj3rxuU9\nrYWRGyxhGJNn7r3yXYPLD6zdw+t7vL+ZLxYf6MOwhJHNLGEYk2eqS8P8y8cWDT7/4u2rPH/PwRaG\nXVab1SxhGJOHPvGOGfz4ksUA7Gnv46q7XvP0/TY3dgB2lVS2s6NnTJ66aHH94PKD6/Z48h7NXf3M\n+uYj/PUdawAIWwsjq1nCMMYA8PD69CeNjbsP7R+pKgmn/T3M+LGEYUweWzpjwuDyV+5M/2mpgc7u\nAdWlljCymSUMY/LYzZ8+le9fdCIA5YVHP2Pz2l1ttPUcPj7V0Bn+JlYUHfV7GP/ZnN7G5LGJFUV8\n5vRZPPdmM7sO9Ix5/2g8wXX3rWflmt2cVF/B/1z+Tpo6+5k3qRw4NGGcc3wdlcUFaYvdjD9rYRhj\nKC8K0dkXG/N+T27ax8o1uwHYuLuDxd9/gvf+xx8H7+1IHhX3Xz52cnqCNb6xFoYxhoqiAjr6omPa\n56nN+wavfhpqIFE0d/Uzs6aE+//PCuu/yAHWwjDGDLYw9nf0pbzPb1btGlw+5/i6Q7b1RpyBDZu7\n+qkrK7RkkSMsYRhjKC9yTjac9sOnUt6n1O0k/8Sy6dz86VOpn1A8uK3J7bto7opQW1aYxkiNnyxh\nGGMIBw9+FBxp+tau/hgbd7fT3hNl4ZQKrv/YIooKgoODGgLcv6aBWDzBrgM91FcVj/JqJpt4ljBE\nZLqIPC0im0XkdRG5epgyIiI/EZGtIrJeRJYmbYuLyFr38aBXcRpjYMGUisHljiN0fl9912tc+NPn\n2binndqyMCLO3duFIefjZFF9JWvebuNnT2+lP5bgxKkVo72cySJetjBiwLWquhBYDnxZRE4YUuYD\nwDz3cQVwU9K2XlVd7D4+5GGcxuS95XNqBgck/N2GxlHLvri9BYB9HU6H9oDCUBA4eDPgj558E4CT\n6ivTHq/xh2cJQ1UbVXWNu9wJbAbqhxS7CPi1Ol4CJojIFK9iMsaM7MKTpwLw9ij3Y0RiCXoiB2fq\ne8/CSYPLl54+E4DPnzF7cF15YYh5E8vSHarxybj0YYjILGAJ8PKQTfXArqTnDRxMKkUiskpEXhKR\nD3sepDF5rrQwxKyaEnaOkjB+t/HQ1sfZ8w9eHbV8Tg07rr+AmTWlVJU4N+jdfOmpg6esTPbz/D4M\nESkD7gOuUdWOoZuH2WWgx22Gqu4RkTnAH0Rkg6puG+b1r8A5ncWMGTPSGLkx+WdGTSk7W7pH3L7J\nHab8osVT+ey7Zo2YDGbUlNLa00ZB0K6rySWeHk0RKcBJFneo6sphijQA05OeTwP2AKjqwM/twDM4\nLZTDqOotqrpMVZfV1dUNV8QYk6KTplawubGTliFjQA1o6YowtbKIH1+yhKUzqkZ8nfefOBmA2jK7\n/yKXeHmVlAC3AZtV9YYRij0IfMa9Wmo50K6qjSJSJSKF7uvUAiuATV7FaoxxrJhbSzyhvLG3c9jt\nLV391KRwX8WVZ8/h+evOZU6d9V/kEi9PSa0ALgU2iMhad923gRkAqnoz8CjwQWAr0AN8zi23EPiF\niCRwktr1qmoJwxiPDVz1tPNAD+8aZntLd4SaFFoNIsK0qpIjljPZxbOEoarPM3wfRXIZBb48zPoX\ngEWH72GM8dLAXdkHug8fqry9J8r6hnaWJM2hYfKL9UgZYwYVFQQpDQdp6To8Yazf3QbAzpaxD4Nu\ncoMlDGPMISZWFLGnrfew9cGAc8LgXy+2YcrzlSUMY8whTpxawYbd7by4rYUP/Pg5+qLOjXqRWAKA\nCTYvd96yhGGMOcTi6RPY3dbLJ//zJTY3dvBWs3NfxkDCGBgzyuQfO/LGmEOcOPXQsZ/aepyJlaJx\n557asCWMvGVH3hhziGWzDr0hb+DO7wfWOlOxhu3u7bxlR94Yc4iCYIDzFkwcfP7c1mZW72zliU37\nnO3WwshbNqe3MeYwyVOqPrK+kUfWHxx00FoY+cuOvDHmMBPc0WaHY30Y+cuOvDHmMOVFTsIoCB4+\nWINdJZW/7MgbYw5TWuicrb5g0ZRDJkB6/rpzKSoI+hWW8ZklDGPMYcrdhBEMBHjib84e7NOwAQXz\nm3V6G2MOU1bkfDR098cA+OPfnkssnvAzJJMBLGEYYw5zxrxaTqqv4Or3zAOgrNA+KowlDGPMMCqK\nCnj4q2f6HYbJMNaHYYwxJiWWMIwxxqTEEoYxxpiUWMIwxhiTEksYxhhjUmIJwxhjTEosYRhjjEmJ\nJQxjjDEpEVX1O4a0EZEmoA1od1dVHmG5Fmg+hrdMfs2xlhlu/dB1oz0fWE5edyz1OZa6jLQtlfhH\nWs6lY5MLdUletr+z1GI9UplM+QyYqap1KZVU1Zx6ALekugysStd7jbXMcOuHrhvteVIdktcddX2O\npS5HU598Oja5UJd01cf+zrz7O0tHfY70yMVTUg+NcTld7zXWMsOtH7putOcPjVDmaB1LXUbalkr8\noy0fCzs2I68/mrqkGseR2N/Z6Ov9/Ds7opw6JTVWIrJKVZf5HUe65FJ9rC6ZK5fqk0t1Ae/rk4st\njLG4xe8A0iyX6mN1yVy5VJ9cqgt4XJ+8bmEYY4xJXb63MIwxxqTIEoYxxpiUWMIwxhiTEksYIxCR\nc0TkORG5WUTO8TueYyUipSKyWkQu9DuWYyUiC93jcq+I/LXf8RwLEfmwiPyniPxWRM73O55jJSJz\nROQ2EbnX71iOhvt/crt7TD7ldzzHwotjkZMJQ0R+KSL7RWTjkPXvF5E3RGSriHzzCC+jQBdQBDR4\nFeuRpKkuANcBv/EmytSloz6qullVrwQ+Dvh2SWSa6vKAqn4RuAz4hIfhHlGa6rNdVS/3NtKxGWO9\nPgrc6x6TD417sEcwlrp4ciy8vCvQrwdwFrAU2Ji0LghsA+YAYWAdcAKwCHh4yGMiEHD3mwTckeV1\neQ9wCc6H0oXZfmzcfT4EvAD8VbbXxd3v34GluXBs3P3u9bMux1CvbwGL3TJ3+h37sdTFi2MRIgep\n6h9FZNaQ1acBW1V1O4CI3A1cpKr/DIx2mqYVKPQizlSkoy4ici5QivMP0Ssij6pqwtPAR5CuY6Oq\nDwIPisgjwJ3eRTyyNB0bAa4Hfqeqa7yNeHRp/r/JGGOpF87ZhGnAWjLwDMwY67Ip3e+fcb8QD9UD\nu5KeN7jrhiUiHxWRXwD/DfzM49jGakx1UdXvqOo1OB+s/+lXshjFWI/NOSLyE/f4POp1cGM0proA\nX8VpAV4sIld6GdhRGuuxqRGRm4ElIvItr4M7BiPVayXwMRG5iXEccuMYDVsXL45FTrYwRiDDrBvx\nrkVVXYnzx5OJxlSXwQKq/5X+UNJirMfmGeAZr4I5RmOty0+An3gXzjEba31agExMfEMNWy9V7QY+\nN97BHKOR6pL2Y5FPLYwGYHrS82nAHp9iOVa5VBfIrfrkUl0g9+ozIJfqNW51yaeE8SowT0Rmi0gY\npxP4QZ9jOlq5VBfIrfrkUl0g9+ozIJfqNX518bvX36MrCe4CGoEoTva93F3/QWALzhUF3/E7znyr\nS67VJ5fqkov1ycV6+V0XG3zQGGNMSvLplJQxxphjYAnDGGNMSixhGGOMSYklDGOMMSmxhGGMMSYl\nljCMMcakxBKG8Y2IdI3De3woxeHf0/me54jIu45ivyUicqu7fJmIZMQYZiIya+hw2sOUqRORx8Yr\nJuMPSxgm64lIcKRtqvqgql7vwXuONg7bOcCYEwbwbeCnRxWQz1S1CWgUkRV+x2K8YwnDZAQR+YaI\nvCoi60XkH5PWPyDOTIGvi8gVSeu7ROT7IvIycLqI7BCRfxSRNSKyQUQWuOUGv6mLyH+5o9y+ICLb\nReRid31ARG503+NhEXl0YNuQGJ8RkR+KyLPA1SLyFyLysoi8JiJPisgkd+jpK4GvichaETnT/fZ9\nn1u/V4f7UBWRcuBkVV03zLaZIvKU+7t5SkRmuOuPE5GX3Nf8/nAtNnFmkHtERNaJyEYR+YS7/h3u\n72GdiLwiIuVuS+I593e4ZrhWkogEReRfk47Vl5I2PwBk9Sx15gj8vtXdHvn7ALrcn+cDt+CMuhnA\nmYznLHdbtfuzGNgI1LjPFfh40mvtAL7qLv8f4FZ3+TLgZ+7yfwH3uO9xAs4cAgAX4wyTHgAm48yB\ncvEw8T4D3Jj0vAoGR0v4AvDv7vL3gK8nlbsTOMNdngFsHua1zwXuS3qeHPdDwGfd5c8DD7jLDwOf\ndJevHPh9Dnndj+EMaT/wvBJnkp3twDvcdRU4I1eXAEXuunnAKnd5Fu6EPcAVwN+5y4XAKmC2+7we\n2OD335U9vHvk0/DmJnOd7z5ec5+X4Xxg/RG4SkQ+4q6f7q5vAeLAfUNeZ2A4+tU4U20O5wF15gPZ\nJCKT3HVnAPe46/eKyNOjxPq/ScvTgP8VkSk4H8JvjbDPe4ATRAZHoa4QkXJV7UwqMwVoGmH/05Pq\n89/A/0ta/2F3+U7g34bZdwPwbyLyL8DDqvqciCwCGlX1VQBV7QCnNQL8TEQW4/x+5w/zeucDJye1\nwCpxjslbwH5g6gh1MDnAEobJBAL8s6r+4pCVIufgfNierqo9IvIMzhzrAH2qGh/yOv3uzzgj/233\nJy3LkJ+p6E5a/ilwg6o+6Mb6vRH2CeDUoXeU1+3lYN2OJOUB4FR1i4icijM43T+LyOM4p46Ge42v\nAfuAU9yY+4YpIzgtud8Ps60Ipx4mR1kfhskEvwc+LyJlACJSLyITcb69trrJYgGw3KP3fx5nlrWA\n2+o4J8X9KoHd7vJnk9Z3AuVJzx8HvjLwxP0GP9RmYO4I7/MCzpDV4PQRPO8uv4Rzyomk7YcQkalA\nj6r+D04LZCnwZ2CqiLzDLVPuduJX4rQ8EsClOHNFD/V74K9FpMDdd77bMgGnRTLq1VQmu1nCML5T\n1cdxTqm8KCIbgHtxPnAfA0Iish74Ac4HpBfuwxkqeiPwC+BloD2F/b4H3CMizwHNSesfAj4y0OkN\nXAUsczuJNzHMLGiq+meg0u38Huoq4HPu7+FS4Gp3/TXA34jIKzintIaLeRHwioisBb4D/JOqRoBP\nAD8VkXXAEzitgxuBz4rISzgf/t3DvN6tOHNFr3Evtf0FB1tz5wKPDLOPyRE2vLkxgIiUqWqXiNQA\nrwArVHXvOMfwNaBTVW9NsXwJ0KuqKiKX4HSAX+RpkKPH80fgIlVt9SsG4y3rwzDG8bCITMDpvP7B\neCcL103AX46h/Kk4ndQCtOFcQeULEanD6c+xZJHDrIVhjDEmJdaHYYwxJiWWMIwxxqTEEoYxxpiU\nWMIwxhiTEksYxhhjUmIJwxhjTEr+P0HXYH01Wbi5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f86970605c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the Learning Rate\n",
    "lr=1e-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ac0e33439534f838703bf1a53e7be2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       1.62316  1.59802  0.40665]                         \n",
      "\n",
      "CPU times: user 41.1 s, sys: 27.4 s, total: 1min 8s\n",
      "Wall time: 1min 7s\n"
     ]
    }
   ],
   "source": [
    "%time learn.fit(lr, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f0fc15c1dd24106982dac057cd171c3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.       1.35955  1.34674  0.52626]                         \n",
      "[ 1.       1.3159   1.25054  0.55691]                         \n",
      "[ 2.       1.17125  1.15738  0.59525]                         \n",
      "[ 3.       1.18631  1.18037  0.58826]                         \n",
      "[ 4.       1.08554  1.12625  0.59774]                         \n",
      "[ 5.       0.96763  0.96539  0.66494]                          \n",
      "[ 6.       0.91771  0.93701  0.67722]                          \n",
      "\n"
     ]
    }
   ],
   "source": [
    "learn.fit(lr, 3, cycle_len=1, cycle_mult=2)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}