georgehc/94-775, 95-865 Basic neural nets and convnets.ipynb Secret

## 94-775, 95-865 Basic neural nets and convnets.ipynb
{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 94-775/95-865: Handwritten Digit Recognition with Neural Nets\n",
    "\n",
    "Author: George H. Chen (georgechen [at symbol] cmu.edu)\n",
    "\n",
    "This demo shows how to train and evaluate four neural net models using PyTorch:\n",
    "\n",
    "1. Flatten -> fully connected -> softmax activation*\n",
    "\n",
    "2. Flatten -> fully connected -> ReLU -> fully connected -> softmax activation*\n",
    "\n",
    "3. Conv2d -> ReLU -> MaxPool2d -> flatten -> fully connected -> softmax activation*\n",
    "\n",
    "4. Conv2d -> ReLU -> MaxPool2d -> Conv2d -> ReLU -> MaxPool2d -> flatten -> fully connected -> softmax activation*\n",
    "\n",
    "*In PyTorch, softmax activation is automatically done as part of using the cross entropy loss."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x130739530>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# the next two lines are needed on my Intel-based MacBook Air to get the code to run; you likely don't need these two lines...\n",
    "# (in fact I used to not need these two lines)\n",
    "import os\n",
    "os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torchsummaryX import summary\n",
    "\n",
    "from UDA_pytorch_utils import UDA_pytorch_classifier_fit, \\\n",
    "        UDA_plot_train_val_accuracy_vs_epoch, UDA_pytorch_classifier_predict, \\\n",
    "        UDA_compute_accuracy\n",
    "\n",
    "np.random.seed(0)\n",
    "torch.manual_seed(0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading in the data and a quick data inspection¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = torchvision.datasets.MNIST(root='data/',\n",
    "                                           train=True,\n",
    "                                           transform=transforms.ToTensor(),\n",
    "                                           download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_images = torch.tensor([image.numpy() for image, label in train_dataset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_labels = torch.tensor([label for image, label in train_dataset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([60000, 1, 28, 28])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_images.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([60000])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We first take a look at the data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x120424090>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAN80lEQVR4nO3df6hcdXrH8c+ncf3DrBpTMYasNhuRWBWbLRqLSl2RrD9QNOqWDVgsBrN/GHChhEr6xyolEuqP0qAsuYu6sWyzLqgYZVkVo6ZFCF5j1JjU1YrdjV6SSozG+KtJnv5xT+Su3vnOzcyZOZP7vF9wmZnzzJnzcLife87Md879OiIEYPL7k6YbANAfhB1IgrADSRB2IAnCDiRxRD83ZpuP/oEeiwiPt7yrI7vtS22/aftt27d281oAesudjrPbniLpd5IWSNou6SVJiyJia2EdjuxAj/XiyD5f0tsR8U5EfCnpV5Ku6uL1APRQN2GfJekPYx5vr5b9EdtLbA/bHu5iWwC61M0HdOOdKnzjND0ihiQNSZzGA03q5si+XdJJYx5/R9L73bUDoFe6CftLkk61/V3bR0r6kaR19bQFoG4dn8ZHxD7bSyU9JWmKpAci4o3aOgNQq46H3jraGO/ZgZ7ryZdqABw+CDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUii4ymbcXiYMmVKsX7sscf2dPtLly5tWTvqqKOK686dO7dYv/nmm4v1u+66q2Vt0aJFxXU///zzYn3lypXF+u23316sN6GrsNt+V9IeSfsl7YuIs+toCkD96jiyXxQRH9TwOgB6iPfsQBLdhj0kPW37ZdtLxnuC7SW2h20Pd7ktAF3o9jT+/Ih43/YJkp6x/V8RsWHsEyJiSNKQJNmOLrcHoENdHdkj4v3qdqekxyTNr6MpAPXrOOy2p9o++uB9ST+QtKWuxgDUq5vT+BmSHrN98HX+PSJ+W0tXk8zJJ59crB955JHF+nnnnVesX3DBBS1r06ZNK6577bXXFutN2r59e7G+atWqYn3hwoUta3v27Cmu++qrrxbrL7zwQrE+iDoOe0S8I+kvauwFQA8x9AYkQdiBJAg7kARhB5Ig7EASjujfl9om6zfo5s2bV6yvX7++WO/1ZaaD6sCBA8X6jTfeWKx/8sknHW97ZGSkWP/www+L9TfffLPjbfdaRHi85RzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlrMH369GJ948aNxfqcOXPqbKdW7XrfvXt3sX7RRRe1rH355ZfFdbN+/6BbjLMDyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJM2VyDXbt2FevLli0r1q+44opi/ZVXXinW2/1L5ZLNmzcX6wsWLCjW9+7dW6yfccYZLWu33HJLcV3UiyM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTB9ewD4JhjjinW200vvHr16pa1xYsXF9e9/vrri/W1a9cW6xg8HV/PbvsB2zttbxmzbLrtZ2y/Vd0eV2ezAOo3kdP4X0i69GvLbpX0bEScKunZ6jGAAdY27BGxQdLXvw96laQ11f01kq6uuS8ANev0u/EzImJEkiJixPYJrZ5oe4mkJR1uB0BNen4hTEQMSRqS+IAOaFKnQ287bM+UpOp2Z30tAeiFTsO+TtIN1f0bJD1eTzsAeqXtabzttZK+L+l429sl/VTSSkm/tr1Y0u8l/bCXTU52H3/8cVfrf/TRRx2ve9NNNxXrDz/8cLHebo51DI62YY+IRS1KF9fcC4Ae4uuyQBKEHUiCsANJEHYgCcIOJMElrpPA1KlTW9aeeOKJ4roXXnhhsX7ZZZcV608//XSxjv5jymYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9knulFNOKdY3bdpUrO/evbtYf+6554r14eHhlrX77ruvuG4/fzcnE8bZgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtmTW7hwYbH+4IMPFutHH310x9tevnx5sf7QQw8V6yMjIx1vezJjnB1IjrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcHUVnnnlmsX7PPfcU6xdf3Plkv6tXry7WV6xYUay/9957HW/7cNbxOLvtB2zvtL1lzLLbbL9ne3P1c3mdzQKo30RO438h6dJxlv9LRMyrfn5Tb1sA6tY27BGxQdKuPvQCoIe6+YBuqe3XqtP841o9yfYS28O2W/8zMgA912nYfybpFEnzJI1IurvVEyNiKCLOjoizO9wWgBp0FPaI2BER+yPigKSfS5pfb1sA6tZR2G3PHPNwoaQtrZ4LYDC0HWe3vVbS9yUdL2mHpJ9Wj+dJCknvSvpxRLS9uJhx9sln2rRpxfqVV17ZstbuWnl73OHir6xfv75YX7BgQbE+WbUaZz9iAisuGmfx/V13BKCv+LoskARhB5Ig7EAShB1IgrADSXCJKxrzxRdfFOtHHFEeLNq3b1+xfskll7SsPf/888V1D2f8K2kgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSKLtVW/I7ayzzirWr7vuumL9nHPOaVlrN47eztatW4v1DRs2dPX6kw1HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2SW7u3LnF+tKlS4v1a665plg/8cQTD7mnidq/f3+xPjJS/u/lBw4cqLOdwx5HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2w0C7sexFi8abaHdUu3H02bNnd9JSLYaHh4v1FStWFOvr1q2rs51Jr+2R3fZJtp+zvc32G7ZvqZZPt/2M7beq2+N63y6ATk3kNH6fpL+PiD+X9FeSbrZ9uqRbJT0bEadKerZ6DGBAtQ17RIxExKbq/h5J2yTNknSVpDXV09ZIurpXTQLo3iG9Z7c9W9L3JG2UNCMiRqTRPwi2T2ixzhJJS7prE0C3Jhx229+W9Iikn0TEx/a4c8d9Q0QMSRqqXoOJHYGGTGjozfa3NBr0X0bEo9XiHbZnVvWZknb2pkUAdWh7ZPfoIfx+Sdsi4p4xpXWSbpC0srp9vCcdTgIzZswo1k8//fRi/d577y3WTzvttEPuqS4bN24s1u+8886WtccfL//KcIlqvSZyGn++pL+V9LrtzdWy5RoN+a9tL5b0e0k/7E2LAOrQNuwR8Z+SWr1Bv7jedgD0Cl+XBZIg7EAShB1IgrADSRB2IAkucZ2g6dOnt6ytXr26uO68efOK9Tlz5nTUUx1efPHFYv3uu+8u1p966qli/bPPPjvkntAbHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIk04+znnntusb5s2bJiff78+S1rs2bN6qinunz66acta6tWrSque8cddxTre/fu7agnDB6O7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQRJpx9oULF3ZV78bWrVuL9SeffLJY37dvX7FeuuZ89+7dxXWRB0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEVF+gn2SpIcknSjpgKShiPhX27dJuknS/1ZPXR4Rv2nzWuWNAehaRIw76/JEwj5T0syI2GT7aEkvS7pa0t9I+iQi7ppoE4Qd6L1WYZ/I/Owjkkaq+3tsb5PU7L9mAXDIDuk9u+3Zkr4naWO1aKnt12w/YPu4FusssT1se7irTgF0pe1p/FdPtL8t6QVJKyLiUdszJH0gKST9k0ZP9W9s8xqcxgM91vF7dkmy/S1JT0p6KiLuGac+W9KTEXFmm9ch7ECPtQp729N425Z0v6RtY4NefXB30EJJW7ptEkDvTOTT+Ask/Yek1zU69CZJyyUtkjRPo6fx70r6cfVhXum1OLIDPdbVaXxdCDvQex2fxgOYHAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HvK5g8k/c+Yx8dXywbRoPY2qH1J9NapOnv7s1aFvl7P/o2N28MRcXZjDRQMam+D2pdEb53qV2+cxgNJEHYgiabDPtTw9ksGtbdB7Uuit071pbdG37MD6J+mj+wA+oSwA0k0Enbbl9p+0/bbtm9toodWbL9r+3Xbm5uen66aQ2+n7S1jlk23/Yztt6rbcefYa6i322y/V+27zbYvb6i3k2w/Z3ub7Tds31Itb3TfFfrqy37r+3t221Mk/U7SAknbJb0kaVFEbO1rIy3YflfS2RHR+BcwbP+1pE8kPXRwai3b/yxpV0SsrP5QHhcR/zAgvd2mQ5zGu0e9tZpm/O/U4L6rc/rzTjRxZJ8v6e2IeCcivpT0K0lXNdDHwIuIDZJ2fW3xVZLWVPfXaPSXpe9a9DYQImIkIjZV9/dIOjjNeKP7rtBXXzQR9lmS/jDm8XYN1nzvIelp2y/bXtJ0M+OYcXCarer2hIb7+bq203j309emGR+YfdfJ9OfdaiLs401NM0jjf+dHxF9KukzSzdXpKibmZ5JO0egcgCOS7m6ymWqa8Uck/SQiPm6yl7HG6asv+62JsG+XdNKYx9+R9H4DfYwrIt6vbndKekyjbzsGyY6DM+hWtzsb7ucrEbEjIvZHxAFJP1eD+66aZvwRSb+MiEerxY3vu/H66td+ayLsL0k61fZ3bR8p6UeS1jXQxzfYnlp9cCLbUyX9QIM3FfU6STdU92+Q9HiDvfyRQZnGu9U042p43zU+/XlE9P1H0uUa/UT+vyX9YxM9tOhrjqRXq583mu5N0lqNntb9n0bPiBZL+lNJz0p6q7qdPkC9/ZtGp/Z+TaPBmtlQbxdo9K3ha5I2Vz+XN73vCn31Zb/xdVkgCb5BByRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ/D+f1mbtgJ8kQQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(train_images[0][0], cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(5)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basics of working with neural nets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_model = nn.Sequential(nn.Flatten(),\n",
    "                             nn.Linear(in_features=784, out_features=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================================\n",
      "      Kernel Shape Output Shape Params Mult-Adds\n",
      "Layer                                           \n",
      "0_0              -     [1, 784]      -         -\n",
      "1_1      [784, 10]      [1, 10]  7.85k     7.84k\n",
      "--------------------------------------------------\n",
      "                      Totals\n",
      "Total params           7.85k\n",
      "Trainable params       7.85k\n",
      "Non-trainable params     0.0\n",
      "Mult-Adds              7.84k\n",
      "==================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Kernel Shape</th>\n",
       "      <th>Output Shape</th>\n",
       "      <th>Params</th>\n",
       "      <th>Mult-Adds</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Layer</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0_0</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 784]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_1</th>\n",
       "      <td>[784, 10]</td>\n",
       "      <td>[1, 10]</td>\n",
       "      <td>7850.0</td>\n",
       "      <td>7840.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Kernel Shape Output Shape  Params  Mult-Adds\n",
       "Layer                                             \n",
       "0_0              -     [1, 784]     NaN        NaN\n",
       "1_1      [784, 10]      [1, 10]  7850.0     7840.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary(simple_model, torch.zeros((1, 1, 28, 28)))   # (batch size, num channels, height, width)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "proper_train_size = int(0.8 * len(train_dataset))\n",
    "val_size = len(train_dataset) - proper_train_size\n",
    "proper_train_dataset, val_dataset = torch.utils.data.random_split(train_dataset,\n",
    "                                                                  [proper_train_size,\n",
    "                                                                   val_size])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.8920\n",
      "  Validation accuracy: 0.8883\n",
      "Epoch 2 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9057\n",
      "  Validation accuracy: 0.9033\n",
      "Epoch 3 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9138\n",
      "  Validation accuracy: 0.9114\n",
      "Epoch 4 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9179\n",
      "  Validation accuracy: 0.9127\n",
      "Epoch 5 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9207\n",
      "  Validation accuracy: 0.9153\n",
      "Epoch 6 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9233\n",
      "  Validation accuracy: 0.9173\n",
      "Epoch 7 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9240\n",
      "  Validation accuracy: 0.9173\n",
      "Epoch 8 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9258\n",
      "  Validation accuracy: 0.9187\n",
      "Epoch 9 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9264\n",
      "  Validation accuracy: 0.9193\n",
      "Epoch 10 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9277\n",
      "  Validation accuracy: 0.9185\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 10  # during optimization, how many times we look at training data\n",
    "batch_size = 128  # during optimization, how many training data to use at each step\n",
    "learning_rate = 0.001  # during optimization, how much we nudge our solution at each step\n",
    "\n",
    "train_accuracies, val_accuracies = \\\n",
    "    UDA_pytorch_classifier_fit(simple_model,\n",
    "                               torch.optim.Adam(simple_model.parameters(),\n",
    "                                                lr=learning_rate),\n",
    "                               nn.CrossEntropyLoss(),  # includes softmax\n",
    "                               proper_train_dataset, val_dataset,\n",
    "                               num_epochs, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3yV9dn48c+VRQIhCSTMBAhgZIUdhorMimCdaBXcVkUtjmq1ah9XbX2kj/5aq1jrAmdBxQFOtIDiZsgGIQEZGUAYSRgJZFy/P+47cJKchBPIycm43q9XXuc+97xOlHPlu0VVMcYYY3wVFOgAjDHG1C+WOIwxxlSLJQ5jjDHVYonDGGNMtVjiMMYYUy0hgQ6gNsTFxWliYmKgwzDGmHpl2bJlu1W1Vfn9jSJxJCYmsnTp0kCHYYwx9YqIbPW236qqjDHGVIslDmOMMdViicMYY0y1NIo2Dm8KCwtJT0+noKAg0KE0GOHh4SQkJBAaGhroUIwxftRoE0d6ejrNmzcnMTEREQl0OPWeqrJnzx7S09Pp3LlzoMMxxvhRo00cBQUFljRqkIgQGxtLdnZ2oEMxptH7YHkGT8zbQGZOPu1jIrjn7G5c2D++xu7faBMHYEmjhtnv05jA+2B5Bve/t5r8wmIAMnLyuf+91QA1ljyscdwYYxqQ/5v389GkUSq/sJgn5m2osWc06hJHIO3Zs4cxY8YAsGPHDoKDg2nVyhmguXjxYsLCwo57j+uuu4777ruPbt26VXrOs88+S0xMDFdccUXNBG6MqTOKS5Rfdh9kTUYuazJyWZ2RS2aO9w4/mTn5NfZcSxw+quk6w9jYWFasWAHAI488QmRkJHfffXeZc1QVVSUoyHvBcMaMGcd9zpQpU044RmNM3VFcomzKPsDq9FzWZDqJYl1mHgePOKWLJiFB9GwfRbOw4KP7PLWPiaixWCxx+KA26gxLpaWlceGFFzJs2DB+/PFHPvroI/785z/z008/kZ+fz2WXXcZDDz0EwLBhw5g2bRrJycnExcVx88038+mnn9K0aVPmzJlD69ateeCBB4iLi+P3v/89w4YNY9iwYSxYsIDc3FxmzJjB6aefzsGDB7n66qtJS0ujZ8+epKam8tJLL9GvX78a/WzGGN8UFZeQuutAmZLE+qz9R7+DIkKD6dU+it+kdCA5Pprk+ChOaRVJSHBQhe+r0vPvObvymonqssQB/PnDtazLzKv0+PJtORwpLimzL7+wmD/OXsXMxdu8XtOzfRQPn9frhOJZt24dM2bM4N///jcAU6dOpWXLlhQVFTFq1CguueQSevbsWeaa3NxcRowYwdSpU7nrrruYPn069913X4V7qyqLFy9m7ty5PProo3z22Wc888wztG3blnfffZeVK1cyYMCAE4rbGFN9R4pK2LhzP2sznQSxJiOP9Vl5HC5yvnOahQXTq300kwZ3pHdCFMnto+nSKpLgIO+dUUr/mLVeVQFWPmkcb//J6tq1K4MGDTr6fubMmbz88ssUFRWRmZnJunXrKiSOiIgIxo8fD8DAgQP5+uuvvd57woQJR8/ZsmULAN988w333nsvAH379qVXrxNLeMaYqqu1DxcVs3HHAVa7pYi1mbn8nLX/6HdJ8yYh9IqP4urTOrkliWg6xzYjqJIkUZkL+8fXeG2IJ0sccNySwRlTF5DhpWEpPiaCt246rcbjadas2dHt1NRU/vnPf7J48WJiYmK48sorvY5292xMDw4OpqioyOu9mzRpUuEcVa3J8I1ptLxVa98zeyWzFm9j/+EiNu7cT2Gx8+8tKjyE3gnRXHdGIsnx0fSOj6Zjy6bVThKBYInDB/ec3c3vdYaVycvLo3nz5kRFRZGVlcW8efMYN25cjT5j2LBhvP3225x55pmsXr2adevW1ej9jWkoVJW8giL2HjzCngOH2XPwSJntWYu3V+gKW1isLN6ylzNOieOGM7vQ200SCS0i6u3YJ0scPqiNOsPKDBgwgJ49e5KcnEyXLl0444wzavwZt912G1dffTV9+vRhwIABJCcnEx0dXePPMcafTqTn4/ESgbN9xN0+zN6DR46WGMpr3iSkQtI49hx4/fohJ/0Z6wppDNUUKSkpWn4hp/Xr19OjR48ARVS3FBUVUVRURHh4OKmpqYwdO5bU1FRCQqr/d4X9Xk0geOtJFBYSxJVDOpLUpvkJJ4KWkWG0bBZGbLMmxDYLo2VkGLHNwoiNDKOluy/WPadJSHCV1drf3jfab5/fX0RkmaqmlN9vJQ7DgQMHGDNmDEVFRagqzz///AklDWMCITMnn0fmrq3w1/6RohKmf7vl6PvIJiFHv+TjY8LpEx9daSJo0TSM8NDgascSyGrt2mTfDoaYmBiWLVsW6DCM8UlBYTE/bN7Doo27WZSaTdquA5WeK8C3942mZbMTSwTVFchq7drk18QhIuOAfwLBwEuqOrXc8U7AdKAVsBe4UlXTRaQf8BwQBRQDj6nqW+41rwAjgFz3Nteq6gp/fg5jTOCoKqm7DrBoYzZfbczmx1/2cqSohCYhQQzpEsvEQR14YdFmdu0/XOHa9jERNTpi2hf+7gpbF/gtcYhIMPAscBaQDiwRkbmq6tll50ngNVV9VURGA48DVwGHgKtVNVVE2gPLRGSequa4192jqrP9FbsxJrByDxXyTdpuFm3MZlFqNlm5Thf0pNaRXDW0E8NPbcWQzi2PliLiIps0iiqiusKfJY7BQJqqbgYQkVnABYBn4ugJ3OluLwQ+AFDVjaUnqGqmiOzCKZXkYIxpcIpLlBXbc44mipXbcyhRZ6zDsKQ47khqxfBTW1VaemgsVUR1hT8TRzyw3eN9OlC+P9pK4GKc6qyLgOYiEquqe0pPEJHBQBiwyeO6x0TkIWA+cJ+qViijishkYDJAx44dT/7TGGNqVFZuvpMoNu7mm7Td5OYXIgJ9E2K4dXQSI05tRd+EaEKCfVv9oTFUEdUV/lyPw9vIlvL93u4GRojIcpx2iwzg6JBnEWkHvA5cp6ql83vcD3QHBgEtgXu9PVxVX1DVFFVNKZ2uvC4ZOXIk8+bNK7Pvqaee4ne/+12l10RGRgKQmZnJJZdcUul9y3c9Lu+pp57i0KFDR9+fc8455ORYYc74V0FhMYs2ZvPXj9Yx9h9fcdrjC7j33dUs3bqXsT3bMO3y/ix/8Cw+mHIGd511KgM7tfA5aZja5c8SRzrQweN9ApDpeYKqZgITAEQkErhYVXPd91HAx8ADqvqDxzVZ7uZhEZmBk3xqz8LHYdT9J32bSZMmMWvWLM4+++yj+2bNmsUTTzxx3Gvbt2/P7Nkn3sTz1FNPceWVV9K0aVMAPvnkkxO+lzGVDbxTdaYB/2qj01bxw+Y9HC4qISwkiCGdW/KbgR0YfmorTm0TWW9HUDdW/kznS4AkEeksImHARGCu5wkiEicipTHcj9PDCvf893Eazt8pd00791WAC4E1fvwMFX019fjn+OCSSy7ho48+4vBhp5Zty5YtZGZm0q9fP8aMGcOAAQPo3bs3c+bMqXDtli1bSE5OBiA/P5+JEyfSp08fLrvsMvLzjw0+uuWWW0hJSaFXr148/PDDADz99NNkZmYyatQoRo0aBUBiYiK7d+8G4O9//zvJyckkJyfz1FNPHX1ejx49uPHGG+nVqxdjx44t8xzTeJUOvMvIyUdx5mb64+yVTHz+e86YuoBf/X0Rf/loHen7DnH5kI7MuG4QKx8ay+vXD+HG4V3o1ra5JY16yG8lDlUtEpFbgXk43XGnq+paEXkUWKqqc4GRwOMiosAioHTVoUuB4UCsiFzr7ivtdvumiLTCqQpbAdx80sF+eh/sWO37+TN+ffxz2vaG8ZUnmdjYWAYPHsxnn33GBRdcwKxZs7jsssuIiIjg/fffJyoqit27dzN06FDOP//8Sv9xPffcczRt2pRVq1axatWqMlOiP/bYY7Rs2ZLi4mLGjBnDqlWruP322/n73//OwoULiYuLK3OvZcuWMWPGDH788UdUlSFDhjBixAhatGhBamoqM2fO5MUXX+TSSy/l3Xff5corr/Tt92UalPwjxezaX8DOvMP8+UMvA++KlR9/2cvZvdpy6+hWDD81joQWTQMUrfEHv47jUNVPgE/K7XvIY3s2UKHORVXfAN6o5J61P24/ZyvkerTzb/3GeY3uADGdTvi2pdVVpYlj+vTpqCp/+tOfWLRoEUFBQWRkZLBz507atm3r9R6LFi3i9ttvB6BPnz706dPn6LG3336bF154gaKiIrKysli3bl2Z4+V98803XHTRRUdn550wYQJff/01559/Pp07dz66sJPnlOym4ThSVEL2gcPszCtgV56TGHbmFbAjr4Bd7vbOvALyCrzPvFzev68a6OeITaDYyHGosmRQwSPR8Eju8c/zwYUXXshdd911dHW/AQMG8Morr5Cdnc2yZcsIDQ0lMTHR6zTqnryVRn755ReefPJJlixZQosWLbj22muPe5+q5i0rnY4dnCnZraoq8Hyd1K+4RNlz4PDRRLDTLS3szD22vSuvgD0Hj1S4NiRIaBMVTuuoJnRtFcnpXWNpHRVOm6hw2kQ14Q9vr6x04J1puCxxBFBkZCQjR47kt7/9LZMmTQKclfxat25NaGgoCxcuZOvWrVXeY/jw4bz55puMGjWKNWvWsGrVKsCZjr1Zs2ZER0ezc+dOPv30U0aOHAlA8+bN2b9/f4WqquHDh3Pttddy3333oaq8//77vP766zX/wc1Jq2zdh8/X7qBlZBg7cg+71UkFZO8/TEm5vwlEnEFzbaKaEB8TTv+OMbRp7iSD0kTRJiqclk3Dqlwf4k/n9LCBd42QJY7qGlFxOdaTMWnSJCZMmMCsWbMAuOKKKzjvvPNISUmhX79+dO/evcrrb7nlFq677jr69OlDv379GDx4MOCs5Ne/f3969epVYTr2yZMnM378eNq1a8fChQuP7h8wYADXXnvt0XvccMMN9O/f36ql6hhV5bGP13td9+GTNTto0TTU/fIPp1ub5rSNdrbbNG/ilhTCiYsMq5GurjbwrnGyadVNjbLfq//syivg/eUZzF6WTmolE/sJ8MtUHzpvGOMDm1bdmHqooLCY/67fyexl6SzamE2JwoCOMcREhJKTX1jhfGtbMLXBEocxdYyqsjI9l9nLtjN3RSZ5BUW0iw7nlpFdmTAgga6tIr0uXGRtC6a2NOrEoao2+KgGNYZqT3/akVtaFbWdTdkHaRISxPjktlwysAOndY0l2KOR2toWTCA12sQRHh7Onj17iI2NteRRA1SVPXv2EB4eHuhQ6pWCwmI+X+dURX2T6lRFDUpswY1nduGcPu2ICg+t9Fqb1M8ESqNNHAkJCaSnp5OdnR3oUBqM8PBwEhISAh1GnaeqLN+ew+xl6Xy4MpP9BUW0jw5nyqhTmDAggc5xzQIdojFVarSJIzQ0lM6dOwc6DNOIZOXm895PGby7LJ3Nuw8SHhrEOcntuGRgAkO7xFY5XsKYuqTRJg5jakP+kWI+X7fDqYpK240qDO7ckptHdGV877Y0r6Iqypi6yhKHMTVMVflp2z5mL0vno5VZ7D9cRHxMBLeNTuLiAfF0irWqKFO/WeIwppoqmyMqIyef939K592fMvhl90GahgUz3q2KGtK5pVVFmQaj0Y4cN+ZEeBs/ERosdI5tRmr2AVRhaJeWXDKwA+OT29Ksif1tZuovGzluTA14Yt4Gr3NEbdp9kN+POZUJA+Lp0NLWnjANmyUOY3x0pKiEjBzv08mXlCh3/CqpliMyJjD8uhK8iIwTkQ0ikiYiFaaVFZFOIjJfRFaJyJcikuDu7yci34vIWvfYZR7XdBaRH0UkVUTecpeZNcZvduUV8I8vNnL61AWVnmNzRJnGxG+JQ0SCgWeB8UBPYJKI9Cx32pM464r3AR4FHnf3HwKuVtVewDjgKRGJcY/9DfiHqiYB+4Dr/fUZTOOlqizbuo/bZy7n9KkLeHpBKn0SorlpRBciQsv+s7E5okxj48+qqsFAmqpuBhCRWcAFwDqPc3oCd7rbC4EPAFR1Y+kJqpopIruAViKSC4wGLncPvwo8Ajznv49hGpPDRcV8tDKLV77bwuqMXJqHh3DN6YlcfVqno91oe7SNsjmiTKPmz8QRD3gs1E06MKTcOSuBi4F/AhcBzUUkVlX3lJ4gIoOBMGATEAvkqGrposfp7nMqEJHJwGSAjh07nvSHMQ3bjtwC3vhhKzMXb2PPwSMktY7krxcmc1H/+Ao9o2yOKNPY+TNxeOu0Xr7v793ANBG5FlgEZAClSQERaQe8DlyjqiXifTZCr/2JVfUF4AVwuuNWO3rT4KkqS7fu45XvtvDZmh2UqPKrHm249vRETu9qk1+ak7TwcRh1f6Cj8At/Jo50oIPH+wQg0/MEVc0EJgCISCRwsarmuu+jgI+BB1T1B/eS3UCMiIS4pY4K9zTmeAoKi5m7IpNXvtvCuqw8oiNCuX5YZ64a2sm60pqa89VUSxwnYAmQJCKdcUoSEznWNgGAiMQBe1W1BLgfmO7uDwPex2k4f6f0fFVVEVkIXALMAq4B5vjxM5gGJCMnn9e/38pbS7ax71Ah3ds25/EJvbmwXzwRYcGBDs/UdcVFcGgPHMx2f3Z7bJf7yc9xrpk+HqLjIao9RCU4r9HxEBUPTeMgyK8dW/3Gb4lDVYtE5FZgHhAMTFfVtSLyKLBUVecCI4HHRURxqqqmuJdfCgwHYt1qLIBrVXUFcC8wS0T+CiwHXvbXZzD1n6ryw+a9vPrdFj5ftwOAsT3bcu0ZiQzp3NKqoxoiX6uIVOHw/uMkAY/9h/bitWY8KASatYJmcc79crYdO7btu9KTgJKy1wWHQfN2EO0mlCg3oXgmmmZxUAf/H7UpR0yDlH+kmPeXZ/Da91v4ecd+WjQNZeLgjlw5tBPxNuai4SouhL/EwU1fV50EDrivxYe93yc8xk0GbkLw3I5sXfZ9eIz3L/dHouGRXGe7pAQO7Ya8DMjNgLxMyEt3XnMznP15mVBSbh354LCySSWqfcVEU1VyOcl2FptyxDQK2/ce4vUftvLWku3k5hfSs10U/3dxH87v157wUKuOalBKSmBPGmQsO/azc41z7Pkzy54bHAbNWh9LAq16QGQr78mhaRyE1PC44qAgJ+FEtob2/Sv/PId2Q66bUPIyyiaa7T9WklyaQFS7ilVhUfFOO8uwOyG0ZlfmtMRh6j1V5btNe5jx7Rbm/7yTIBHGJbfl2tMTSenUwqqjGoq8TI8k8RNkLofDec6xoNCKX6gAgyfD6AehSfPar/IZUWGyjKp5Jpf4Ad7POW5y+QHWZpX9XezbAq27n/DH8Maqqky9UX4689tHn8KREuW177aQuusAsc3CuHxIR64Y0om20bb2eb2Wn+Mkhoxlx173ZznHgkKgTTLEDzz2E5cEQW6J0rOKqDFa8L+w6G8V94+4r9rVVpVVVVniMPWCt+nMS/VJiOaa0xL5dZ92Vh1VHxUWOFVMGT8dK1HsST12PPaUYwmi/QBo27vqqpfGnjg8neTvwto4TL3mbTpzgFaRTZgz5QyrjqovSkpg90bI9EgSO9Ycq1qJbAPxKdB3olNd074/RLSo3jOqW0Vkqs0Sh6nzSkq00unMdx84bEmjroxQLh+Harl2iWWQuQKO7HeOhzWH9v3gtCnHShRR7U++LaIu/C7qCj8lUUscpk5bn5XHgx+sqfS4TWdO3Rmh/NVU6DjkWON1xjI4sNM5FhQKbZOh72XHkkRsUr0dAFdv+On/C0scpk7aX1DIP75I5dXvtxAdEcrEwR2YszyD/MJjg6ga7XTmqrBzLWz8DFI/d/b9LTGgIVHaVvr6Rc5rbBJ0GXUsSbRNhpAmgYvP1ChLHKZOUVXmrszksY/Xk33gMJcP7sg9Z3cjpmkYQzvHNt7pzAvz4ZevnWSxcZ4zeMxT/j7ntf0ASKjQluk/6Uud9oryki+uG6Ug4xeWOEydkbZrPw9+sJbvN++hT0I0L16dQt8OMUePN7rpzPMynSSxcR5s/hKK8iG0GXQdBSPvhaSx0Lxt3elFVFfiMH5nicME3KEjRTw9P42Xv9lMRGgwf70wmUmDOxIc1MgavUtKnDELGz9zfnascvbHdIQBV8GpZ0OnYTU+CtiY6rLEYQJGVZm3dgePfriOzNwCLhmYwH3juxMX2Yjqwg/vh00LnVJF6jxn/iQJgg5D4Vd/dpJFq+5V9zSqK91P60ocxu8scZiA2LL7II98uJYvN2TTvW1znp7Un5TEloEOq3bs3QwbP3dKFVu+ccYwhEfDKWfBqePglDHQtBq/i7rSllBX4jB+Z4nD1KqCwmL+9eUm/v3VJsKCg3jw3J5cc1onQoIbcLfM4iJnDqHS9ordG5z9cd1g6C1OsugwBILtn6OpH+z/VFNrFv68i4fnrmXb3kOc37c9//PrHrSJaqD19Yf2Qtp/nVJF2n+hINcZy5A4DFJ+C6eOhZZdAh2lMSfEEofxu/R9h3j0w3V8vm4nXVs14z83DOH0U+ICHdbJ8xwprQrZPx/rLrv9R9ASZ5ru7uc5bRVdRzmztBpTz/k1cYjIOOCfOCsAvqSqU8sd74SzXGwrYC9wpaqmu8c+A4YC36jquR7XvAKMAEr7/ZWuDGjqmCNFJbz49WaeWZCKINw7rjvXD+tMWEgDqZb6aip0GORWQX12bOW3tn3gzLudKqj2/W10tGlw/JY4RCQYeBY4C0gHlojIXFVd53Hakzjrir8qIqOBx4Gr3GNPAE2Bm7zc/h5Vne2v2M3J+zZtNw/OWcPm7IOc3asND53Xq2GsvFdSAlu/heVvOO/fuBhCIqDLSBh2lzO2IroRjTUxjZI/SxyDgTRV3QwgIrOACwDPxNETuNPdXgh8UHpAVeeLyEg/xmf8YGdeAX/5aB0frcqiU2xTZlw3iFHdWgc6rJOXmw4rZsL3zzjtFZ6K8qFdX0i5LjCxGVPL/Jk44oHtHu/TgSHlzlkJXIxTnXUR0FxEYlV1z3Hu/ZiIPATMB+5T1QoLB4vIZGAyQMeOHU/sExifFRaX8Op3W/jHFxspLFF+/6skbh7RtX6vj1F0GH7+2CldbFoAKCSeCf2vgh7nwf+2s5HSplHyZ+LwNmKp/KpRdwPTRORaYBGQARQd5773AzuAMOAF4F7g0QoPUn3BPU5KSkrDX60qgBb/speH5qzh5x37GdWtFY+c34tOsc0CHdaJ27HaSRar3nLmgIpKgOH3QL/LoWXnQEdnTMD5M3GkAx083icAmZ4nqGomMAFARCKBi1W1yj/hVNVdP5LDIjIDJ/mYANh94DCPf/Iz7/6UTnxMBM9fNZCxPdvUz/Ux8vfB6tmw/HXIWgnBYdD9XOh/pdN+EeSl5GQjpU0j5c/EsQRIEpHOOCWJicDlnieISBywV1VLcEoS0493UxFpp6pZ4nw7XQhUvliDqTGe6323iwnn9C6xfL5uJ/mFxdwysiu3jT6FpmH1rHd3SQn88qVTulj/ERQfdpYlHf8E9L7k+KO3baS0aaT89i9dVYtE5FZgHk533OmqulZEHgWWqupcYCTwuIgoTlXVlNLrReRroDsQKSLpwPWqOg94U0Ra4VSFrQBu9tdnMI7y631n5hQw+6cMklo347krUzildWSAI6ymfVthxX9gxZuQux3CY2DgNU7pol3fQEdnTJ3n1z8RVfUT4JNy+x7y2J4NeO1Wq6pnVrJ/dE3GaI6vsvW+Dx0prj9JozDfKVUsfx1++QoQZ0DeWX+Gbr+2GWeNqYZ6VrdgAiGzkvW+M3MKajmSalJ1pilf/obTfnE415mifNT/QN9JENPh+PcwxlRgicMcV1REKLn5hRX219n1vg/ugdVvOwlj5xoICYce5ztVUYln2khuY06SJQ5TpW9Sd5OXX0iQQIlHp+Y6t953SbEz1mL56/DzJ85U5e0HwK//7ixjGhFz/HsYY3xiicNUanP2AX735jKS2kTy2zMSeWbBprqx3rfn5IJ7NjmN3Ctmwv5MiGgJg290ShdtegUmPmMaOEscxqvcQ4Vc/+pSQoKDePmaQXRo2ZSJgzsFOizHV1OhRaJTFbX1G2fFvFN+BeOnwqnjISQs0BEa06BZ4jAVFBaX8Lv/LCN93yH+c+NQOrRsGsBg8mHXeqetYsdq2OEO2/ngZmc9izEPOQ3dUe0DF6MxjYwlDlOGqvLI3LV8m7aHJy7pw6DaXM71wC43Oaw+lih2p4JW7AoMOEuwFh2xpGFMLbPEYcp47futvPnjNm4a0YXfpPipu2pxEexJc5PDKqcUsWM1HNx17JzoDtAm2ekN1TbZGdEdk+j0iHok2iYXNCaALHGYoxZtzObPH67lVz3a8Mezu9fMTQvyYOdatxThliZ2rYcidwxIUCi07g5JZzmJom1vp1H7eNN9GGMC5riJw5025E1V3VcL8ZgASdu1nyn/+YlT2zTnqYn9CA6qZKJCzx5NnlSdFfB2rnFLEKuc7X1bjp0T0dJJDINucBNEMsSdWv3GbJtc0JiA8qXE0RZn9b6fcCYhnKeqNk15A7Lv4BGuf3UpTUKCeOmaFCKbVPG/xVdTYdidzvranm0RO9d4LHAkENvVWTa1/1VOkmjbG5q3g5qYOdcmFzQmoI6bOFT1ARF5EBgLXIezfsbbwMuqusnfARr/OlJUws1vLCMrp4CZk4eS0KKSHlSFBfD5A872/7Y/1mAd2sypWkq+2C1F9IY2PSGsHq/HYYypkk9tHKqqIrIDZwGlIqAFMFtEvlDVP/ozQOM/qsrDc9fw4y97+cdlfRnYqYX3Exf+L3z1N48L3aQxeDKM+5tN4WFMI+NLG8ftwDXAbuAl4B5VLRSRICAVsMRRT03/dgszF29nyqiuXNQ/ofITQ5o4r6MfgAV/tR5NxjRyvvypGAdMUNWzVfUdVS0EcBdfOtev0Rm/WfjzLh77eB1n92rDH86qYs6p9R/C/Eeh92/gTFts0RjjW+L4BNhb+kZEmovIEABVXe+vwIz/bNy5n9tmLqd72yj+cVk/girrQZW1Et6bDPEpcP4zTsO29WgyptHzJXE8BxzweH/Q3XdcIjJORDaISJqIVPjGEZFOIjJfRFaJyJcikuBx7CzSLwcAAB53SURBVDMRyRGRj8pd01lEfhSRVBF5S0RsYqJq2HPgMNe/uoSIsGBeuial8uVe9++EmZMgogVM/A+EulOoW48mYxo9XxKHeHa/dauofGkbCQaeBcYDPYFJItKz3GlPAq+pah/gUeBxj2NPAFd5ufXfgH+oahKwD7jeh89ggMNFxdz8xjJ25R3mxatTKl9Po7AAZl0O+ftg0kxo3qZ2AzXG1Gm+JI7NInK7iIS6P3cAm324bjCQpqqbVfUIMAu4oNw5PYH57vZCz+OqOh/Y73myiAgwmmPLzb4KXOhDLI2eqvLA+2tYsmUfT/ymL/06VLI+hSrMvRUylsJFz9sa3MaYCnxJHDcDpwMZQDowBJjsw3XxwHaP9+nuPk8rgYvd7YuA5iISW8U9Y4EcVS2q4p4AiMhkEVkqIkuzs7N9CLdhe/HrzbyzLJ3bxyRxft8qJgX8+v/B6necHlQ9z6+9AI0x9cZxE4eq7lLViaraWlXbqOrlqrrreNcB3lpcy484vxsYISLLgRE4yamowlXVu2dp3C+oaoqqprRq1cqHcBuu/67byeOf/syve7fj92OSKj9x3VxY8BfrQWWMqZIvbRXhOO0IvYDw0v2q+tvjXJoOeE6vmgBkep6gqpnABPc5kcDFqlrVIIHdQIyIhLiljgr3NGWtz8rjjlnLSW4fzZO/6Vt1D6r3b3J7UE2rmalBjDENki9VVa/jzFd1NvAVzpf1/iqvcCwBktxeUGHARGCu5wkiEucOJAS4H2curEq5jfQLgUvcXdcAc3yIpVHK3n+YG15dSmR4CC9enUJEWLD3E4/2oGrp9qAK936eMcbgW+I4RVUfBA6q6qvAr4Hex7vILRHcCswD1gNvq+paEXlUREorz0cCG0RkI9AGeKz0ehH5GngHGCMi6SJytnvoXuAuEUnDafN42YfP0OgUFDo9qPYcdHpQtY2uJBlYDypjTDX5MldVofuaIyLJOPNVJfpyc1X9BGcAoee+hzy2Z3Osh1T5a8+sZP9mnB5bphKqyp/eW82yrft49vIB9EnwoQfVpa9Duz61G6gxpl7yJXG8ICItgAdwqpoigQf9GpU5Kc99tYn3lmdw11mn8us+7So/8esn3R5UD1oPKmOMz6pMHG77Q567iNMioEutRGVO2GdrdvB/n23gvL7tuW30KZWfuG6uM2Fh70vhzD/UXoDGmHqvyjYOd5T4rbUUizlJazJyufOtFfTtEMMTl/RBKusZVaYH1TPWg8oYUy2+NI5/ISJ3i0gHEWlZ+uP3yEy17NpfwI2vLSWmaSgvXjWQ8NDKelDtsB5UxpiT4ksbR+l4jSke+xSrtqozCgqLmfzaMnIOFfLOzafROqqyHlT5x3pQ/Xae9aAyxpwQX5aO7VwbgZgTo6r8cfYqVmzP4d9XDiQ5PrqyE2HOrZCxDC57w3pQGWNOmC8jx6/2tl9VX6v5cEx1PbMgjbkrM7nn7G6MS25b+YlfPwlrZsOYh6DHebUXoDGmwfGlqmqQx3Y4MAb4CbDEEWAfr8ri719s5KL+8fxuZNfKT1w3x+lB1ecyGHZX7QVojGmQfKmqus3zvYhE40xDYgJoVXoOf3hnBQM6xvD4hN6V96DKXAHv3wwJg+C8p60HlTHmpPnSq6q8Q0AVU6waf9uR6/Sgim3WhOevSvGtB9Vlb1oPKmNMjfCljeNDjk1dHoSz+NLb/gzKVC7/SDE3vraU/QVFvHvL6bRq3sT7iaU9qApyrAeVMaZG+dLG8aTHdhGwVVXT/RSPqUJJifKHd1awJjOXF65KoUe7KO8nqsKcKW4PqjetB5Uxpkb5kji2AVmqWgAgIhEikqiqW/wamangqfmpfLJ6B/eP785ZPasoQSx6Eta86/agOrf2AjTGNAq+JI53cJaOLVXs7hvk/XRTkz5YnsET8zaQkZMPwODEFkweXsXYy3VzYKH1oDLG+I8vjeMhqnqk9I27Hea/kEypD5ZncP97q48mDYBVGbnMWVHJooeZK+C9m6wHlTHGr3xJHNkeCy8hIhfgLOFq/OyJeRvILywus6+gsIQn5m2oeHJpD6qmsTYHlTHGr3xJHDcDfxKRbSKyDWcFvpt8ubmIjBORDSKSJiL3eTneSUTmi8gqEflSRBI8jl0jIqnuzzUe+79077nC/WntSyz1UaZHSaPK/YX5TtIoyIXLZ0Fkg/2VGGPqAF8GAG4ChopIJCCq6st644hIMPAscBaQDiwRkbmqus7jtCeB11T1VREZDTwOXOXOvvswkILTFXiZe+0+97orVHWpj5+x3moXE05mTkGF/e1jIo69Ke1BlfmT04Oq7XFX9TXGmJNy3BKHiPyviMSo6gFV3S8iLUTkrz7cezCQpqqb3XaRWcAF5c7pCcx3txd6HD8b+EJV97rJ4gtgnC8fqCEZntSqwr6I0GDuObvbsR2LnnB7UD1sPaiMMbXCl6qq8aqaU/rG/SI/x4fr4oHtHu/T3X2eVgIXu9sXAc1FJNaHa2e41VQPSiVzbYjIZBFZKiJLs7OzfQi3bikqLuG7TXtIiIkgPiYcAeJjInh8Qm8u7O/+KtZ+AAsfgz4TYdidAY3XGNN4+NIdN1hEmqjqYXDGcQCVDFcuw9sXupZ7fzcwTUSuxVmaNgNnkGFV116hqhki0hx4F7gKLxMuquoLwAsAKSkp5Z9b581dmcm2vYd44aqBjO3lZdbbzOXuHFSD4bx/Wg8qY0yt8aXE8QYwX0SuF5HrcaqNXvXhunSgg8f7BKBMP1JVzVTVCaraH/gfd19uVdeqaob7uh/4D06VWINSXKJMW5BG97bNvQ/0y8uCmZdDsziYaHNQGWNq13ETh6r+H/BXoAdOm8RnQCcf7r0ESBKRziISBkwE5nqeICJxIlIaw/3AdHd7HjDWbU9pAYwF5olIiIjEudeGAucCa3yIpV75eHUWm3cf5PYxSRVnvT06B1UuTJppPaiMMbXO19lxdwAlOO0RY4D1x7tAVYuAW3GSwHrgbVVdKyKPeowLGQlsEJGNQBvgMffavcBfcJLPEuBRd18TnASyCliBU7X1oo+foV4oKVGmLUglqXUk48pXUR3tQbUcLn7RelAZYwKi0jYOETkVp5QwCdgDvIXTHXeUrzdX1U+AT8rte8hjezYwu5Jrp3OsBFK67yAw0Nfn10fz1u5g484D/HNiP4KCypU2SntQ/eoR6P7rQIRnjDFVNo7/DHwNnKeqaQAiYl13/EhVeXpBGl3imnFun/ZlD759tTMPVd9JcMbvAxOgMcZQdVXVxThVVAtF5EURGYP33k6mhvx3/S7WZ+UxZdQpBHuWNjKXO0kjYTCc+5T1oDLGBFSliUNV31fVy4DuwJfAnUAbEXlORMbWUnyNhqry9PxUOrZsygX9PEobxYXwtjvjivWgMsbUAb70qjqoqm+q6rk43WJXABXmnTIn58uN2azOyGXKqK6EBLv/WRY+Dn+Jg5ytzvsnk+CRaGe/McYEiC8DAI9yezY97/6YGlJa2oiPieCi/gnHDoy8DzZ+BkcOwJ40eCQ3cEEaY4zL1+64xo++TdvD8m053DyyK2EhHv9Jtn4LWStg6O8CF5wxxpRTrRKH8Y+n56fSNiqcS1MSyh74bhpEtHR6Uh3YFZjgjDGmHCtxBNgPm/eweMtebhrRhSYhwccO7E6FjZ/CoBsgrCmMuj9wQRpjjAdLHAH29PxU4iKbMGlwx7IHvn8WgpvA4BsDE5gxxlTCEkcALd2yl+827eGm4V0ID/UobRzcDStnQp9LbS4qY0ydY4kjgJ5ekEbLZmFcMbRcaWPJy1BUAKfdGpjAjDGmCpY4AmTF9hwWbczmhjM70zTMo49CYQEseRGSxkLr7oEL0BhjKmGJI0CmLUglpmkoV5+WWPbAqrfgYLaVNowxdZYljgBYk5HLf9fv4rdndCayiUdpo6TEaRRv2xs6Dw9cgMYYUwVLHAEwbUEazcNDuOb0xLIH0v4LuzfAabfZRIbGmDrLEkct+3lHHp+t3cF1pycSHRFa9uD3z0Dz9pA8ITDBGWOMD/yaOERknIhsEJE0EakwMaKIdBKR+SKySkS+FJEEj2PXiEiq+3ONx/6BIrLavefTUmFt1bpt2oI0moUF89thncseyFoFvyyCITdBcKj3i40xpg7wW+IQkWDgWWA8zlrlk0SkZ7nTngReU9U+wKPA4+61LYGHgSHAYOBhd+1xgOeAyUCS+zPOX5+hpqXt2s/Hq7O4+vREYpqGlT34/TQIi4SB1wYkNmOM8ZU/SxyDgTRV3ayqR4BZwAXlzukJzHe3F3ocPxv4QlX3quo+4AtgnIi0A6JU9XtVVeA14EI/foYa9ezCTYSHBHND+dJGboazJGz/qyAiJjDBGWOMj/yZOOKB7R7v0919nlbirDQIcBHQXERiq7g23t2u6p4AiMhkEVkqIkuzs7NP+EPUlF92H2TOigyuHNqR2MgmZQ8ufh60BIbeHJjgjDGmGvyZOLy1PWi593cDI0RkOTACyACKqrjWl3s6O1VfUNUUVU1p1aqV71H7yb8WphEaHMSNw7uUPXB4Pyx9BXqcDy0SAxGaMcZUiz8TRzrQweN9ApDpeYKqZqrqBFXtD/yPuy+3imvT3e1K71kXbd97iPeWZzBpcEdaNy+39OvyN+BwLpx+W2CCM8aYavJn4lgCJIlIZxEJAyYCcz1PEJE4ESmN4X5gurs9DxgrIi3cRvGxwDxVzQL2i8hQtzfV1cAcP36GGvGvLzcRLMLNI7qWPVBcBD/8CzoMhYSUwARnjDHV5LfEoapFwK04SWA98LaqrhWRR0XkfPe0kcAGEdkItAEec6/dC/wFJ/ksAR519wHcArwEpAGbgE/99RlqQkZOPrOXbefSQQm0jS5X2vj5Q8jZBqfb9CLGmPrDrysAquonwCfl9j3ksT0bmF3JtdM5VgLx3L8USK7ZSP3n+a82AXDLyFPKHlCF756BFp2h2zkBiMwYY06MjRz3o515Bcxasp2LByQQHxNR9uC2HyBjGZw2BYKCvd/AGGPqIEscfvT8V5spLlF+V760Ac6Av/AY6Hd57QdmjDEnwRKHn2TvP8ybP27lwn7xdIxtWvbgnk3w88cw6HoIaxaYAI0x5gRZ4vCTl77eTGFxCVNGda148Id/OfNRDZ5c+4EZY8xJssThB3sPHuH1H7ZyXt/2dGkVWfbgob2w/E3o/Rto3jYwARpjzEmwxOEHL3+zmfzCYm4d5aVtY+nLUJTvNIobY0w9ZImjhuUcOsKr323lnOR2JLVpXvZg0WFY/CJ0HQ1tegUmQGOMOUmWOGrYjG+3cOBwEbeO9lLaWP0OHNhp64kbY+o1Sxw1KK+gkOnf/sLYnm3o0S6q7EFVZz3x1r2cEocxxtRTljhq0GvfbWF/QRG3j0mqeHDTfNi1zplepH4tWmiMMWVY4qghBw4X8dI3vzC6e2uS46MrnvDdNIhsC8mX1H5wxhhTgyxx1JA3fthKzqFCbvPWtrFjDWxeCEMmQ0hYxePGGFOPWOKoAYeOFPHios2cmRRH/44tKp7w/bMQ2hQGXlf7wRljTA2zxFED/vPjNvYcPMId3to28rKc3lT9r4SmLWs/OGOMqWGWOE5SQWExzy/azGldYklJ9JIYFr8AJUUw9JbaD84YY/zAEsdJemvJdrL3H/bek+rIQVg6HXqcCy27VDxujDH1kF8Th4iME5ENIpImIvd5Od5RRBaKyHIRWSUi57j7w0RkhoisFpGVIjLS45ov3XuucH9a+/MzVOVwUTHPfbmJQYktGNrFS2lj+ZtQkAOn2XrixpiGw28rAIpIMPAscBaQDiwRkbmqus7jtAdwlpR9TkR64qwWmAjcCKCqvd3E8KmIDFLVEve6K9yVAANq9rJ0duQV8MRv+iDlx2aUFMMPz0LCIOg4JDABGmOMH/izxDEYSFPVzap6BJgFXFDuHAVKh1hHA5nudk9gPoCq7gJygBQ/xlpthcUl/GvhJvp1iGHYKXEVT/j5Y9i3xaYXMcY0OP5MHPHAdo/36e4+T48AV4pIOk5po7ROZyVwgYiEiEhnYCDQweO6GW411YNS4U99h4hMFpGlIrI0Ozu7Bj5OWe//lEFGTj53jEmqWNoAZ4W/mE7Q47waf7YxxgSSPxOHty90Lfd+EvCKqiYA5wCvi0gQMB0n0SwFngK+A4rca65Q1d7Ame7PVd4erqovqGqKqqa0atXqpD+Mp6LiEqYtTKN3fDQju3m59/YlsP1HGPo7W0/cGNPg+DNxpFO2lJDAsaqoUtcDbwOo6vdAOBCnqkWqeqeq9lPVC4AYINU9L8N93Q/8B6dKrFbNXZnJtr2HuG30KZWUNp6B8Ghn7IYxxjQw/kwcS4AkEeksImHARGBuuXO2AWMARKQHTuLIFpGmItLM3X8WUKSq69yqqzh3fyhwLrDGj5+hguISZdqCNLq3bc5ZPdtUPGHfFlj/oTNKvElkxePGGFPP+a1XlaoWicitwDwgGJiuqmtF5FFgqarOBf4AvCgid+JUY12rqur2pJonIiVABseqo5q4+0Pde/4XeNFfn8Gbj1dnsXn3Qf51xQDvpY0fngMJgiE31WZYxhhTa/yWOABU9ROcRm/PfQ95bK8DzvBy3Ragm5f9B3EaygOipESZtiCVpNaRjOvlZb3w/H3w0+vODLhR7Ws/QGOMqQU2crwa5q3dwcadB7h19CkEBXkpbSx7BQoPOmtuGGNMA2WJw0eqytML0ugS14xz+3gpTRQdgR+fh84joG3v2g/QGGNqiSUOH/13/S7WZ+UxZdQpBHsrbax9D/Znwek2vYgxpmGzxOEDVeWZBal0bNmUC/p5KW2oOiv8teoOp/yq9gM0xphaZInDB19uzGZVei5TRnUlJNjLr+yXr2Dnajhtiq0nboxp8CxxHIeq8vT8VOJjIriof4L3k76bBs1aQe9Lazc4Y4wJAEscx/Ft2h6Wb8vhlpFdCQvx8uva9TOkfQGDJ0NoeO0HaIwxtcyv4zjqsw+WZ/DEvA1k5OQTJBDuLWmAM5lhSASkXF+7ARpjTIBY4vDig+UZ3P/eavILiwEoUXhwzlpCgoO4sL/HBL8HdsGqt5w5qZrFBihaY4ypXVZV5cUT8zYcTRql8guLeWLehrInLn4Rigth6JRajM4YYwLLEocXmTn5x99/5BAseQm6nQNxp9RSZMYYE3iWOLxoHxNx/P0rZ0L+XptexBjT6Fji8OKes7sREVp2AaaI0GDuOdudd7GkBH74F7QfAB1PC0CExhgTONY47kVpA/gT8zaQmZNP+5gI7jm727GG8Y2fwZ40uGS6DfgzxjQ6ljgqcWH/+LI9qDx9Pw2iO0KPC2o3KGOMqQOsqqq6MpbB1m9h6M0QbHnXGNP4+DVxiMg4EdkgImkicp+X4x1FZKGILBeRVSJyjrs/TERmiMhqEVkpIiM9rhno7k8TkafF6zJ8fvTdNGgSBf2vOv65xhjTAPktcYhIMPAsMB7oCUwSkZ7lTnsAeFtV++OsSf4vd/+NAKraGzgL+H8iUhrrc8BkIMn9Geevz1BBzjZYNwcGXgPhUbX2WGOMqUv8WeIYDKSp6mZVPQLMAso3CihQ+g0cDWS62z2B+QCqugvIAVJEpB0Qparfq6oCrwEX+vEzlPXDv53G8CE319ojjTGmrvFn4ogHtnu8T3f3eXoEuFJE0nHWJi9dBWklcIGIhIhIZ5x1xju416cf557+UZALP70GvS6C6EpmyTXGmEbAn4nDW9uDlns/CXhFVROAc4DX3Sqp6ThJYSnwFPAdUOTjPZ2Hi0wWkaUisjQ7O/sEP4KHZa/Ckf1wmg34M8Y0bv7sFpSOU0oolcCxqqhS1+O2Uajq9yISDsS51VN3lp4kIt8BqcA+9z5V3RP3fi8ALwCkpKR4TS4+Ky6EH/8NiWdC+34ndStjjKnv/FniWAIkiUhnEQnDafyeW+6cbcAYABHpAYQD2SLSVESaufvPAopUdZ2qZgH7RWSo25vqamCOHz+DY+0HkJdhpQ1jjMGPJQ5VLRKRW4F5QDAwXVXXisijwFJVnQv8AXhRRO7EqXK6VlVVRFoD80SkBMgAPPu+3gK8AkQAn7o//qMK3z8DsUmQNNavjzLGmPrAryPYVPUTnEZvz30PeWyvA87wct0WoFsl91wKJNdooFXZ8g1krYRzn4IgGy9pjDH2TXg8H94BTeOg78RAR2KMMXWCJY6qZG+EvZtg0A0Q6n2qdWOMaWwscVTlh2ed10E3BDYOY4ypQ2yWPm8WPg5fTT32/kl3hb8R98Go+wMTkzHG1BGWOLwZdf+xBPFINDySG9h4jDGmDrGqKmOMMdViieN4RlSYDd4YYxo1SxzHY20axhhThiUOY4wx1WKJwxhjTLVY4jDGGFMtljiMMcZUiyUOY4wx1SLO0t0Nm4hkA1tP8PI4YHcNhnOiLI66FQNYHOVZHGXVhThONoZOqtqq/M5GkThOhogsVdUUi6PuxFEXYrA4LI76EIe/YrCqKmOMMdViicMYY0y1WOI4vhcCHYDL4jimLsQAFkd5FkdZdSEOv8RgbRzGGGOqxUocxhhjqsUShzHGmGqxxFEJEZkuIrtEZE0AY+ggIgtFZL2IrBWROwIUR7iILBaRlW4cfw5EHB7xBIvIchH5KIAxbBGR1SKyQkSWBjCOGBGZLSI/u/+fnBaAGLq5v4fSnzwR+X0A4rjT/f9zjYjMFJHw2o7BjeMON4a1tfl78PadJSItReQLEUl1X1vUxLMscVTuFWBcgGMoAv6gqj2AocAUEekZgDgOA6NVtS/QDxgnIkMDEEepO4D1AXx+qVGq2i/AffX/CXymqt2BvgTg96KqG9zfQz9gIHAIeL82YxCReOB2IEVVk4FgYGJtxuDGkQzcCAzG+e9xrogk1dLjX6Hid9Z9wHxVTQLmu+9PmiWOSqjqImBvgGPIUtWf3O39OF8K8QGIQ1X1gPs21P0JSK8KEUkAfg28FIjn1yUiEgUMB14GUNUjqpoT2KgYA2xS1ROdqeFkhAARIhICNAUyAxBDD+AHVT2kqkXAV8BFtfHgSr6zLgBedbdfBS6siWdZ4qgnRCQR6A/8GKDnB4vICmAX8IWqBiQO4Cngj0BJgJ5fSoHPRWSZiEwOUAxdgGxghlt195KINAtQLKUmAjNr+6GqmgE8CWwDsoBcVf28tuMA1gDDRSRWRJoC5wAdAhBHqTaqmgXOH6JA65q4qSWOekBEIoF3gd+ral4gYlDVYrcqIgEY7BbJa5WInAvsUtVltf1sL85Q1QHAeJwqxOEBiCEEGAA8p6r9gYPUUFXEiRCRMOB84J0APLsFzl/XnYH2QDMRubK241DV9cDfgC+Az4CVOFXODYoljjpOREJxksabqvpeoONxq0K+JDDtP2cA54vIFmAWMFpE3ghAHKhqpvu6C6c+f3AAwkgH0j1Kf7NxEkmgjAd+UtWdAXj2r4BfVDVbVQuB94DTAxAHqvqyqg5Q1eE4VUepgYjDtVNE2gG4r7tq4qaWOOowERGc+uv1qvr3AMbRSkRi3O0InH+kP9d2HKp6v6omqGoiTpXIAlWt9b8qRaSZiDQv3QbG4lRR1CpV3QFsF5Fu7q4xwLrajsPDJAJQTeXaBgwVkabuv5sxBKgDhYi0dl87AhMI3O8EYC5wjbt9DTCnJm4aUhM3aYhEZCYwEogTkXTgYVV9uZbDOAO4Cljtti8A/ElVP6nlONoBr4pIMM4fG2+rasC6wtYBbYD3ne8nQoD/qOpnAYrlNuBNt5poM3BdIIJw6/PPAm4KxPNV9UcRmQ38hFM1tJzATfnxrojEAoXAFFXdVxsP9fadBUwF3haR63GS629q5Fk25YgxxpjqsKoqY4wx1WKJwxhjTLVY4jDGGFMtljiMMcZUiyUOY4wx1WKJw5gaICLF5WaIrbER3CKSGMhZmo0pz8ZxGFMz8t0pWYxp8KzEYYwfuet2/M1dz2SxiJzi7u8kIvNFZJX72tHd30ZE3nfXPlkpIqXTZgSLyIvuGg+fuyP4jQkISxzG1IyIclVVl3kcy1PVwcA0nNl9cbdfU9U+wJvA0+7+p4Gv3LVPBgBr3f1JwLOq2gvIAS728+cxplI2ctyYGiAiB1Q10sv+LTiLYG12J6zcoaqxIrIbaKeqhe7+LFWNE5FsIEFVD3vcIxFnKvsk9/29QKiq/tX/n8yYiqzEYYz/aSXblZ3jzWGP7WKsfdIEkCUOY/zvMo/X793t7zi2tOkVwDfu9nzgFji6eFZUbQVpjK/srxZjakaExwzG4KwDXtolt4mI/Ijzh9okd9/twHQRuQdnFb/SWW3vAF5wZzMtxkkiWX6P3phqsDYOY/zIbeNIUdXdgY7FmJpiVVXGGGOqxUocxhhjqsVKHMYYY6rFEocxxphqscRhjDGmWixxGGOMqRZLHMYYY6rl/wN0sg79rN1aqAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "UDA_plot_train_val_accuracy_vs_epoch(train_accuracies, val_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================\n",
      "      Kernel Shape Output Shape   Params Mult-Adds\n",
      "Layer                                             \n",
      "0_0              -     [1, 784]        -         -\n",
      "1_1     [784, 512]     [1, 512]  401.92k  401.408k\n",
      "2_2              -     [1, 512]        -         -\n",
      "3_3      [512, 10]      [1, 10]    5.13k     5.12k\n",
      "----------------------------------------------------\n",
      "                        Totals\n",
      "Total params           407.05k\n",
      "Trainable params       407.05k\n",
      "Non-trainable params       0.0\n",
      "Mult-Adds             406.528k\n",
      "====================================================\n",
      "Epoch 1 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9528\n",
      "  Validation accuracy: 0.9458\n",
      "Epoch 2 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9650\n",
      "  Validation accuracy: 0.9565\n",
      "Epoch 3 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9796\n",
      "  Validation accuracy: 0.9693\n",
      "Epoch 4 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9833\n",
      "  Validation accuracy: 0.9723\n",
      "Epoch 5 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9912\n",
      "  Validation accuracy: 0.9768\n",
      "Epoch 6 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9924\n",
      "  Validation accuracy: 0.9789\n",
      "Epoch 7 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9949\n",
      "  Validation accuracy: 0.9782\n",
      "Epoch 8 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9955\n",
      "  Validation accuracy: 0.9774\n",
      "Epoch 9 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9975\n",
      "  Validation accuracy: 0.9791\n",
      "Epoch 10 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9980\n",
      "  Validation accuracy: 0.9797\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEICAYAAABF82P+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3xV9fnA8c+TAUkgJCEJe6rsjYgDZIiKOBDRFrAurLNWa61WqNaqrYUWa9Xqz9YBilWQqiAu0NIIosjeyJ5JGGEkEDJv8vz+ODdwCTcQyLm5Se7zfr3yumfd73kS5TznfM93iKpijDHGlBYW7ACMMcZUTZYgjDHG+GUJwhhjjF+WIIwxxvhlCcIYY4xfliCMMcb4FbAEISITRWSfiKwpY7+IyMsisllEVolIT599t4vIJu/P7YGK0RhjTNkkUP0gRKQfkA1MVtXOfvZfDTwIXA1cCLykqheKSH1gCdALUGApcL6qHjrV+ZKSkrRVq1bu/hLGGFPDLV26dL+qJvvbFxGok6rqPBFpdYpDrsdJHgr8ICLxItIYGAB8raoHAUTka+AqYMqpzteqVSuWLFniRujGGBMyRGRHWfuC+Q6iKbDLZz3Vu62s7cYYYypRMBOE+Nmmp9h+cgEi94jIEhFZkpGR4WpwxhgT6oKZIFKB5j7rzYD0U2w/iaq+rqq9VLVXcrLfKjRjjDFnKWDvIMphJvBLEZmK85I6S1V3i8hs4M8ikuA97kpg7NmcoLCwkNTUVPLy8tyJ2AAQFRVFs2bNiIyMDHYoxpgACliCEJEpOC+ck0QkFfgDEAmgqv8EvsBpwbQZyAFGe/cdFJE/Aou9RT1b8sL6TKWmphIbG0urVq0Q8VdzZc6UqnLgwAFSU1Np3bp1sMMxxgRQIFsxjTrNfgUeKGPfRGBiRWPIy8uz5OAyESExMRF752NM8M1YnsaE2RtIz8ylSXw0jw1ux7Ae7rXpCWYVU6Ww5OA++5saE3wzlqcx9uPV5BYWAZCWmcvYj1cDuJYkanyCMMYYtwX6zt2Xp6iY7HwPR/I8HM4r5Eies/zMp2uPJYcSuYVFTJi9wRJEdXDgwAEGDRoEwJ49ewgPD6ektdWiRYuoVavWacsYPXo0Y8aMoV27dmUe8+qrrxIfH8/PfvYzdwI3xpTpTO7cC4uKvRf0wpMu8EeOLftsyz95e05B0UkxnEp6Zq47vyiWIE7g9l1BYmIiK1asAODpp5+mbt26PProoycco6qoKmFh/lscT5o06bTneeABv69yjDEB8NfZ6/3euT/+0SomfbfthAt9XmHxacuLjgynblQEsVERxEZFUi8qgsZxUcTWjjy2LbbU/tioSO6avJi9h/NPKq9JfLRrv6slCK/KqM8rsXnzZoYNG0bfvn1ZuHAhn332Gc888wzLli0jNzeXESNG8NRTTwHQt29fXnnlFTp37kxSUhL33XcfX375JTExMXzyySc0aNCAJ598kqSkJB5++GH69u1L3759+d///kdWVhaTJk3ikksu4ejRo9x2221s3ryZjh07smnTJt588026d+/u6u9mTE2TlVvIil2ZLNtxiGU7D5Ge6b/ZfL6nmLiYWjSrH3PsIh5bO6LURd75rOf9rBsVQWT42XVHGzukwwnXLHCSzWODy65tOFMhkyCe+XQt69IPl7l/+c5MCopOzPa5hUX89sNVTFm00+93Ojapxx+u63RW8axbt45Jkybxz3/+E4Dx48dTv359PB4PAwcO5KabbqJjx44nfCcrK4v+/fszfvx4HnnkESZOnMiYMWNOKltVWbRoETNnzuTZZ59l1qxZ/OMf/6BRo0Z89NFHrFy5kp49e570PWNCXXGxsnX/0WPJYNnOQ2zal40qhAm0bRhLTK1wv9U+TeOjmXxn70qLteTG1VoxVYLSyeF02yvq3HPP5YILLji2PmXKFN566y08Hg/p6emsW7fupAQRHR3NkCFDADj//PP59ttv/ZY9fPjwY8ds374dgPnz5/P4448D0K1bNzp1OrvEZkxNkp3vYaX36WDpzkMs35lJVm4hAHHRkfRoEc+1XZtwfssEujWPp27tiJNqG8D9O/fyGtajacBejkMIJYjT3en3Gf8/0vy83GkaH80H917sejx16tQ5trxp0yZeeuklFi1aRHx8PLfccovf3t++L7XDw8PxeDx+y65du/ZJxwRqWHdjqgtVZfuBnGNPB0t3HGLj3iMUe/9ptG1YlyGdG9GzRQI9WyZwTlIdwsJObtJdGXfuVUXIJIjTeWxwu6DdFRw+fJjY2Fjq1avH7t27mT17NldddZWr5+jbty/Tpk3j0ksvZfXq1axbt87V8o2panIKPKxKzWLpjkMs33mIZTszOXi0AIDY2hF0bxHP4E6N6Nkyge7N44mLLv/QMYG+c68qLEF4BfOuoGfPnnTs2JHOnTtzzjnn0KdPH9fP8eCDD3LbbbfRtWtXevbsSefOnYmLi3P9PMYEUlktDVWV1EO5znsDb3XRj7uPUOR9PDgnuQ6XtW9AzxYJnN8ygfMa1CXcz9OBOVHAZpSrbL169dLSEwb9+OOPdOjQIUgRVS0ejwePx0NUVBSbNm3iyiuvZNOmTUREnN09gv1tTWXzV/cfESZ0aBzL7qx89mc7TT5jaoXTrVk857dMoGfLeHo0TyChzun7HIUqEVmqqr387bMniBCRnZ3NoEGD8Hg8qCr/+te/zjo5mNAUqN7DRcVKZk4Bh3IKj30eyik4vny0gEM5BaSszzip0YinWFm3+whDuzWhZ8sEeraIp13DWCLOsumoOZFdIUJEfHw8S5cuDXYYppoqbz+hvMIiDnov6JneC/2hnEIyj/omgONJ4NDRAg7n+W9sAc4TQnxMLerXiSyzRWFxsfL3EdafJxAsQRhjTmvCKXoP/2ve1mMX/lP1HK5TK5z4mFok1IkkIaYWzevHkBAT6SSAmEgS6tRy9sc4++NjIqlbO+LY4JBltTR0s+ewOZElCGOMX1m5hXy3eT9zN2SQdorew03jo+jUpN6xi32C9yJfkgzqx9QiLiaS2hHhFYonmC0NQ5UlCGMM4LwLWJ2WxbyNGczdmMGKXZkUFSuxURFERYb5fTpoGh/Nm7df4Kc094VS/4OqwhKEMSFs35E85m3cz7yNGXy7KYNDOYWIQNemcfxiwLn0b5tM9+bxfLZqd5W4ew+V/gdVhb3qD7ABAwYwe/bsE7a9+OKL/OIXvyjzO3Xr1gUgPT2dm266qcxySzfrLe3FF18kJyfn2PrVV19NZmZmeUM3NVCBp5gFWw4w/sv1XP3St/R+bg6P/mcl3285wMD2DXhpZHeWPHE5n/yyL7+5sh29WtUnIjyMYT2aMm54F5rGRyM4Tw7jhnexi3UNZ08Q/qSMg4FjXSlq1KhRTJ06lcGDBx/bNnXqVCZMmHDa7zZp0oQPP/zwrM/94osvcssttxATEwPAF198cdZlmepr54Ec5m7KYO6GDBZs2c/RgiIiwoTzWybw26va0a9NMh0b1/M7rIQvu3sPPZYg/Jk73rUEcdNNN/Hkk0+Sn59P7dq12b59O+np6XTv3p1BgwZx6NAhCgsL+dOf/sT1119/wne3b9/Otddey5o1a8jNzWX06NGsW7eODh06kJt7vDXH/fffz+LFi8nNzeWmm27imWee4eWXXyY9PZ2BAweSlJRESkoKrVq1YsmSJSQlJfHCCy8wcaIz7fddd93Fww8/zPbt2xkyZAh9+/bl+++/p2nTpnzyySdER1srkeokp8DDD1sPMG/jfuZuzGDb/qMANEuIZliPpvRvm8zF5yYSG1X+oSVMaAqdBPHlGNizuvzHT7rm9Mc06gJDxp/ykMTERHr37s2sWbO4/vrrmTp1KiNGjCA6Oprp06dTr1499u/fz0UXXcTQoUPLnO/5tddeIyYmhlWrVrFq1aoThut+7rnnqF+/PkVFRQwaNIhVq1bx0EMP8cILL5CSkkJSUtIJZS1dupRJkyaxcOFCVJULL7yQ/v37k5CQwKZNm5gyZQpvvPEGP/3pT/noo4+45ZZbTv+3MEGjqmzcm83cjfuYt3E/i7YdpKComKjIMC4+J5HbLm5J/7bJtE6qY/OJmzMSOgnidDJ3QNau4+s75jufcc0hvmWFii6pZipJEBMnTkRV+d3vfse8efMICwsjLS2NvXv30qhRI79lzJs3j4ceegiArl270rVr12P7pk2bxuuvv47H42H37t2sW7fuhP2lzZ8/nxtuuOHYiLLDhw/n22+/ZejQobRu3frYJEK+w4Wb4PHXg3lguwbM37z/WFLYc9hphtq2YV1uv6Ql/domc0Gr+kRFVqxpqQltoZMgTnOnf4Kn4+DpLNdOPWzYMB555JFjM8b17NmTt99+m4yMDJYuXUpkZCStWrXyO8S3L393f9u2beP5559n8eLFJCQkcMcdd5y2nFONv1UyVDg4w4X7VmWZyuevB/Ovp62g5D9hbFQEl7ZJon/bZC5tk2ydxoyrQidBBFHdunUZMGAAd955J6NGjQKc2eEaNGhAZGQkKSkp7Nix45Rl9OvXj/fee4+BAweyZs0aVq1aBThDhdepU4e4uDj27t3Ll19+yYABAwCIjY3lyJEjJ1Ux9evXjzvuuIMxY8agqkyfPp13333X/V/clJuqcuBoAWmHckk9lEvqoRzSMnP5YPEu8j3FpY51hqt++84L6NYs3sYdMgFjCcKf/idP41lRo0aNYvjw4UydOhWAn/3sZ1x33XX06tWL7t270759+1N+//7772f06NF07dqV7t2707u3M7Vht27d6NGjB506dTppqPB77rmHIUOG0LhxY1JSUo5t79mzJ3fcccexMu666y569Ohh1UkBpKpkZOd7L/653kSQ4yxnOsulO6LFRkWclBxKZOd7OL9l/coI3YQwG+7bnJVQ+NueyeilxcXKviP5x+78U30TwKFcUjNzKSh1sY+PiaRZQjRN46NplhBzwnLThGjioiNPOdPhd2MuC8jvbUKLDfdtzBnyV/f/+Eer2JKRTeukOj4XficJ7M7MO2m00cQ6tWiWEE37xrFc3rHhSQmgbu3T//Oz8YdMMFmCMMaPCbM3nDR6ab6nmH/8b/Ox9eTY2jSNj6ZL0ziGdG5M04RomiVE0zwhmibx0cTUqvg/Lxt/yARTjU8Qqmptv11WU6olTyXdT7VOiTm/6U/T+OhKa0JqPZhNsAS0+YOIXCUiG0Rks4ic9OZXRFqKyBwRWSUi34hIM599fxGRNd6fEWdz/qioKA4cOBASF7TKoqocOHCAqKioYIcSMLuzcsucr7hpfDTnJte1/gUmJATsCUJEwoFXgSuAVGCxiMxU1XU+hz0PTFbVd0TkMmAccKuIXAP0BLoDtYG5IvKlqh4+kxiaNWtGamoqGRkZbvxKxisqKopmzZqd/sBqaOPeI9w+cRHhAmHhYSe8V7C6fxNqAlnF1BvYrKpbAURkKnA94JsgOgK/9i6nADN8ts9VVQ/gEZGVwFXAtDMJIDIyktatW5/9b2BCyg9bD3DP5CXUjgzn4wf6sGlvttX9m5AWyATRFPAZu4JU4MJSx6wEbgReAm4AYkUk0bv9DyLyAhADDOTExGKMqz5blc4jH6ykef1o3h7dm+b1Y+jUJM4SgglpgXwH4a8St/TLgEeB/iKyHOgPpAEeVf0K+AL4HpgCLABOmtlcRO4RkSUissSqkczZemv+Nh6cspyuzeL46P5LaF4/JtghGVMlBDJBpALNfdabAem+B6hquqoOV9UewBPebVnez+dUtbuqXoGTbDaVPoGqvq6qvVS1V3JycqB+D1NDFRcrz32+jj9+to4rOzbk33ddSHxMrWCHZUyVEcgEsRhoIyKtRaQWMBKY6XuAiCSJSEkMY4GJ3u3h3qomRKQr0BX4KoCxmhCT7yniVx+s4I1vt3H7xS35v5+dby2TjCklYO8gVNUjIr8EZgPhwERVXSsizwJLVHUmMAAYJyIKzAMe8H49EvjW23/hMHCL94W1MRWWlVvIve8u4YetBxkzpD339jvH+soY40eNHovJmNJ2Z+Vyx8TFbN2fzYSbutlLaBPybCwmY4ANe45wx6RFHMnzMOmO3vRtk3T6LxkTwixBmJDww9YD3D15CdGR4Xxw70V0ahIX7JCMqfIsQZgaz7ePwzt39qZZgjVjNaY8LEGYGu3Nb7fyp89/pFfLBN68vZc1YzXmDFiCMDVScbHy5y9+5M3527iqUyNeHNndmrEac4YsQZgaJ99TxG+mreSzVbu545JW/P7ajmWOzmqMKZslCFOj+PZxGDukPfdYHwdjzpolCFNj+PZxeGlkd67vbn0cjKkISxCmRvDt4/D26N70Oc/6OBhTUZYgTLXn28dh2r0X07FJvWCHZEyNYAnCVGslfRxaJMbw9ugLrI+DMS6yBGGqrZI+Dhe0SuCN26yPgzFuswRhqh3fPg5DOjfi7yOsj4MxgWAJwlQr1sfBmMpjCcJUG759HH53dXvuvtT6OBgTSJYgTLWQnpnL6EnWx8GYymQJwlR56/cc5o6Jizma7+Gd0b25xPo4GFMpLEGYKmfG8jQmzN5AemYuiXVrcSSvkPiYWky772I6NLY+DsZUFksQpkqZsTyNsR+vJrewCID92QUIcF//cy05GFPJwoIdgDG+JszecCw5lFDgzW+3BScgY0KYPUGYKqG4WJm7KYO0zFy/+9PL2G6MCRxLECaosnIK+c/SXbz7ww52HMghTKBYTz6uSXx05QdnTIizBGGCYm16Fu8u2MGMFWnkFRbTq2UCv7myHQWFRfz+k7UnVDNFR4bz2OB2QYzWmNBkCcJUmgJPMbPW7mHy99tZsuMQUZFhDOvelFsvbkmnJnHHjosIDzvWiqlJfDSPDW7HsB7W78GYymYJwgTcnqw83l+4g/cX7WJ/dj4tE2N48poO/OT85sTFRJ50/LAeTS0hGFMFWIIwAaGqLNx2kMkLtjN77V6KVRnYrgG3XdySfm2SCbPxk4yp8ixBGFcdzfcwfXka7y7YwYa9R4iLjuTnfVtzy4UtaZFoczUYU51YgjCu2JKRzbsLdvDR0lSO5Hvo1KQef72xK9d1a0J0LRuK25jqyBKEOWtFxcr/1u9j8oLtfLtpP5HhwjVdGnPrxa3o2SLeRlo1ppoLaIIQkauAl4Bw4E1VHV9qf0tgIpAMHARuUdVU776/Atfg9Pb+GviVqvppIW8q28GjBUxdvJP3fthJWmYujepF8eiVbRlxQQuSY2sHOzxjjEsCliBEJBx4FbgCSAUWi8hMVV3nc9jzwGRVfUdELgPGAbeKyCVAH6Cr97j5QH/gm0DFa05v5a5MJi/Ywaer0inwFHPxOYn8/toOXN6hIRHhNmqLMTVNIJ8gegObVXUrgIhMBa4HfBNER+DX3uUUYIZ3WYEooBYgQCSwN4CxGk4cRbWk/8FVnRvx+ardTP5hByt3ZVKnVjgjejXn1otb0rZhbLBDNsYApIyDgWNdLzaQCaIpsMtnPRW4sNQxK4EbcaqhbgBiRSRRVReISAqwGydBvKKqPwYw1pBXehTVtMxcHv3PSp6YvpqjBUWcm1yHZ4Z2YnjPpsRGndx3wRgTRHPHV7sE4e8NZel3CI8Cr4jIHcA8IA3wiMh5QAegmfe4r0Wkn6rOO+EEIvcA9wC0aNHCxdBDj79RVD3FSoQq7911IZecm2gvnY0pzc07d1XIPwJ5mZCXBbmZznKud93vsnc9QAKZIFKB5j7rzYB03wNUNR0YDiAidYEbVTXLe+H/QVWzvfu+BC7CSSK+338deB2gV69e9gK7AsoaLTW/sJg+NoNb1RCgagRTAaXv3IuLvBf3Qydf6E970c8CLSr7XAhExUF0PETFO+fI3HF899Pe4Wr6j3Ht/5NAJojFQBsRaY3zZDASuNn3ABFJAg6qajEwFqdFE8BO4G4RGYfzJNIfeDGAsYa8JvHRfofatlFUq4j8I87FaMAYsCe54MnLgr1rYc8a2LPK2fZa3+MX+oIjp/5+eC3n4l5yoY9JgsTznPWoeO/F33fZ59hasRBWRmOQp+PgafefJAKWIFTVIyK/BGbjNHOdqKprReRZYImqzgQGAONERHGeDh7wfv1D4DJgNU611CxV/TRQsRr4xYBzeWLGmhO22SiqQaIKmTth10LYuQB2/gD7vK/g/tQAoutDTCLE1IfoBO9n/RM/YxKPL0fFQZiLnRVD4UlGFbJSYc9q52ev9/PQ9pOP3bva+WzaC9pcceoLfWR0tUrwAe0HoapfAF+U2vaUz/KHOMmg9PeKgHsDGZs50Zr0LARIjq1NxpF8G0W1MhUXwd41TiIo+TnirY0NrwVFBcePLSqA7D1QO9b5XsYGyD3oVDcUe8o4gTgXqROSyKkSjHdfRBl9WgL0QjRoPAWwf8PxZFDyk5fpPUCg/jnQuDv0uBUadXF+YhvDM/EBuXM/Y/3HBKRY60ltWLrjEFMW7eLuS1vzxDUdgx1OzVdwFFKXOIlg1w+wa/Hxqol6zaDlJdDiIuenQcfjd/+nqkZQhfzDkHPQSRg5h7yfByHnwPHl3INwZDfsW+esFx4tO87IOn6SSKKzb+nbULch1G3gfNZJLjuhVCW5h7zVQz6JIGM9FBc6+yOioWFH6DTMmwi6Ov8NatcNbtynE6CEbQkixHmKinli+moax0Xx8OVtgx1OzXRkr5MIdv7gVBntXuV9GSnQsBN0GwEtLobmF0J889MW55d4X2BGxQGty/+9wjznolk6ieR4n0pK1nevOv5UA/Dpr04uKyr+xKRxwnLy8W0xiRWv8jpdNZeq8wK39FNBlk/L+7oNnSRw3qDjySDx3DOLLUB37lWFJYgQ9/b321m/5wj/vKUndWrb/w4Vpgr7N/pUFy2AQ9ucfRFRTj113187CaFZL6fqp7wCcTGKjILIxlCvcfm/83Qc/HodZO+F7H0nfh7d5yynL3M+C7JP/r6EOU8cvomkTrL/5BIV57/O3reaqzAPMn488clg7xrniarkfIltnAR8wV3QqDM07AKxDc/871VaTapq88OuCCFsd1Yuf/96IwPbJTO4U6Ngh1M9efIhfYWTCHYtdJJC7kFnX0ySU010wc+dhNCoK0TUOvtzVaWLUVxT5+d08rOPJ42TEop3ed9657OkmsdXeG1v0mhwYgIB+PgebxXRhuPNQ2vVdZ7Kuv7UeSpo2AUadIBaNtT82ThtgvC2RHpPVQ9VQjymEj376To8xcozQztbJ7hT8a3OyD0EuxYdb12UtgyK8p19iedBu6u97w8udqorauLf9UyeZGrXdX7qn3Pq41Sdv21ZTyTZeyF1MRzNOP6dVR84ny0uhgvvcxJCQuuym4KaM1aeJ4hGOAPtLcPppzDbRlWt/lLW7+PLNXt4bHA7m8jnVI7sdaozsvc6CSHD29w0LMJp1dL7bichNL/IqWcPBYF4khHxNs+tDw3an/rYIg/8MbFqtB6q4U6bIFT1SRH5PXAlMBpnaIxpwFuquiXQARr35RYU8dTMNZybXIe7Lz3NnV2oOrwb5r/gtNYBWPMRNO8NXW50kkHT863aIljCrWa8spTrL62qKiJ7gD2AB0gAPhSRr1X1t4EM0Ljv1ZTN7DqYy/t3X0itCHscP8GRPTD/77DojROHPcg/DJv/67xkbn1p8OIzjhreeqiqKM87iIeA24H9wJvAY6paKCJhwCbAEkQ1snlfNv+at4UbejTlknNtjKVjjuyF716EJROhqBC6j4J+j0FCq4ANY2AqoCq9sK/ByvMEkQQMV9UdvhtVtVhErg1MWCYQVJXfz1hDdGQ4v7u6Q7DDqRqy98H8F2HJW05i6DYS+j16+peqxoSA8iSIL3CmAwVARGKBjqq60OZoqF5mrEhjwdYD/GlYZ5saNDvDeWJY/JbTCqnrCOeJIfHck4+16gwTosqTIF4DevqsH/WzzVRxWTmFPPf5j3RrHs/NvUN47oyj++G7l2Dxm+DJgy4/gX6/haTzyv6OVWeYEFWeBCG+zVq9VUvWjKCamfDVeg4eLeDt0b0JC6uBbfNP5+gB+P5l5+VzYY6TGPr/FpLaBDsyY6qs8lzot3pfVL/mXf8FsDVwIRm3rdiVyXsLd3LHJa3o3DQu2OFUrpyD8P0/YNHrziB5nW+E/o9Dso07ZczplCdB3Ae8DDyJMzfDHLzTfJqqr2QwvgaxtXnkihC6KOYchAWvwsJ/OeMBdbrBSQyn64RljDmmPB3l9uHMBmeqoXd/2MHa9MO8cnMPYqMigx1O4OUeggX/Bwv/6fRd6DjMSQwNbRhzY85UefpBRAE/BzoBUSXbVfXOAMZlXLD3cB5/+2ojl7ZJ4pouZzBaZ3WUmwk/vOb85GdBh6HO9JwNOwU7MmOqrfJUMb0LrAcGA88CPwOseWs18MfP1lFQVMwfr6/Bg/HlZcEP/4QfXnWW21/rJIZGXYIdmTHVXnkSxHmq+hMRuV5V3xGR93HmmTZV2LyNGXy2aje/vrwtrZLqBDsc9+Uddt4vLHjFmRqy3TUw4HFo3C3YkRlTY5QnQZQM0p4pIp1xxmNqFbCITIXlFRbx1CdraJ1Uh/sG1LAewflHjieG3EPQdojzxNCke7AjM6bGKU+CeF1EEnBaMc0E6gK/D2hUpkJe+2YL2w/k8O+fX0jtiApO7VhV5Gc7TVW//4czIU+bwU5iaGr9NY0JlFMmCO+AfIe9kwXNA2rY7WjNs23/UV77ZgtDuzWhb5tqPhhfyjjo85DTue37l515k8+7AgaMhWbnBzs6Y2q8UyYIb6/pXwLTKikeUwGqylOfrKF2RBhPXlvNB+PLz3Ym6ln8JuTsh3MHOYmh+QXBjsyYkFGeKqavReRR4AOccZgAUNWDZX/FBMNnq3bz7ab9PDO0Ew1io07/haqm4Chs+grWToeNXznbGnWBgb9zJusxxlSq8iSIkv4OD/hsU6y6qUo5nFfIs5+to0vTOG65qGWwwym/ghzY/LU3Kcx2xknytTXF+ek/xgbNM6aSlacndevKCMRUzAtfbWR/dj5v3d6L8Ko+GF9hrjM729rpsGEWFB6FmCToNsoZEqPlJRAWbhP1GBNk5elJfZu/7ao62f1wzNlYk5bF5AXbufWilsQrd24AABk+SURBVHRtFh/scPwrzIMtc7xJ4UtnfKSYROj6U29S6GNzDRtTxZTnX6TvW8EoYBCwDLAEUQUUFStPTF9N/Tq1+c2V7YIdzok8+bDlf05SWP8FFByB6PrOiKqdboBWl546KdhEPcYEVXmqmB70XReROJzhN0wV8P7CHaxMzeKlkd2Ji64Cg/F5Cpx3Bmunw/rPnQHzouKh0zAnKbTuB+HljNPeORgTVGfzTJ8DlGuWFRG5CngJCAfeVNXxpfa3BCYCyTjTmt6iqqkiMhD4u8+h7YGRqjrjLOKtsfYdyeOvszfQ57xEhnZrErxAPAWwba43KXzmjIkUFecMmNfpBjinf/mTgjGmyijPO4hPcVotAYQBHSlHvwgRCQdeBa4AUoHFIjJTVdf5HPY8MNk7xtNlwDjgVlVNAbp7y6kPbAa+KvdvFSL+/PmP5BcW82wwBuMrKjyeFH78zBkPqXYctL/GmxQGQEStyo3JGOOq8jxBPO+z7AF2qGpqOb7XG9isqlsBRGQqcD3gmyA6Ar/2LqcA/p4QbgK+VNUcP/tC1veb9zNjRToPXXYe5ybXrZyTFnlg+zxvUvjUGQupdj1od7WTFM4dCBG1KycWY0zAlSdB7AR2q2oegIhEi0grVd1+mu81BXb5rKcCF5Y6ZiVwI0411A1ArIgkquoBn2NGAi/4O4GI3IN3drsWLVqU41epGfI9RTw5Yw0t6sfwi4HnuX+ClHHH6/+LPLBj/vGkkHMAatX1SQqXQWQ17JRnjDmt8iSI/wCX+KwXebedbswDf3UeWmr9UeAVEbkDZ6ynNJynFKcAkcZAF8oYXlxVXwdeB+jVq1fpsmus1+duZev+o7w9+gKiIgMwGN/c8dCqj5MU1s10hrqoVRfaDfEmhUGWFIwJAeVJEBGqWlCyoqoFIlKeyuVUoLnPejMg3fcAVU0HhgOISF3gRlX17Rn1U2C6qhZiANh5IIdXUjZzTZfGDGjXwP0TbJ7jfL5zHUTWgXZXOUnhvMshMtr98xljqqywchyTISJDS1ZE5Hpgfzm+txhoIyKtvQllJM5w4ceISJJ3xFiAsTgtmnyNAqaU41whQVV5auYaIsKE31/r8hzLKeOcnsv/Hn58W+FRSGwDHa6z5GBMCCrPE8R9wHsi8op3PRXw27val6p6vCPBzsZp5jpRVdeKyLPAElWdCQwAxomI4lQxHRvvSURa4TyBzC33b1PDzVqzh282ZPD7azvSKM7lKp4BY2D3CtiSAkX5NsSFMaZcHeW2ABd5q4BEVY+Ut3BV/QL4otS2p3yWPwQ+LOO723FedBsgO9/DM5+uo2Pjetx+cQAG41v2DmycBVeNh1nWg9kYU44qJhH5s4jEq2q2qh4RkQQR+VNlBGeO+/vXG9l7JI/nbuhMRHh5agbPwIEtMOt30Lo/9L7XhrgwxgDlewcxRFUzS1a8s8tdHbiQTGnr0g/z9vfbGdW7BT1aJLhbeJEHpt/njIk07DUIC7MhLowxQPkSRLiIHOv9JCLRgPWGqiTFxcoTM1YTHx3J44Pbu3+C7/4OqYvgmhcgzmr0jDHHlecl9b+BOSIyybs+GngncCEZX1MX72L5zkz+9pNuxMW4PJ5R2jL4Zjx0vgm63ORu2caYaq88L6n/KiKrgMtxOr/NAqrRlGXV1/7sfP4yaz0Xtq7P8J4u390X5MD0e6FOA7jm+dMfb4wJOeUdzXUPUIzTcW0b8FHAIjLHjPtiPTkFHp67IQCD8f33adi/EW6dAdEuv9cwxtQIZSYIEWmL07ltFHAA+ACnmevASootpC3ceoCPlqXyiwHncl6DWHcL3zwHFv0LLrzfGWDPGGP8ONUTxHrgW+A6Vd0MICK/PsXxxiUFnmKenLGGZgnRPHhZuabeKL+cg/DJA5DcHi7/g7tlG2NqlFO1YroRp2opRUTeEJFB+B+Az7jszflb2bQvm2eGdiK6louD8anC54/A0f0w/HUbPsMYc0plJghVna6qI3Bmc/sGZ96GhiLymohcWUnxhZxdB3N4ec4mBndqyKAODd0tfPV/nBFaB46Fxt3cLdsYU+OUpxXTUeA9nPGY6gM/AcZgM7y5ZsbyNCbM3kB6Zi61I8IoUuUP13Vy9ySZu+DzR6H5hdDnYXfLNsbUSGc0ZoOqHlTVf6nqZYEKKNTMWJ7G2I9Xk5aZiwJ5nmJUYdG2g+6dpLgYZtwPWgQ3/AvCAjCHhDGmxnF5UB9zpibM3kBuYdEJ2zzFyoTZG9w7ycLXYPu3cNU4qN/avXKNMTWaJYggS8/MPaPtZ2zvOvjvM9DuGuhxqztlGmNCgiWIIGsS778lUVnbz4gnHz6+B6LqwXUvgdud7YwxNZoliCC7s0+rk7ZFR4bz2OB2FS885c+wdzUM/QfUTa54ecaYkGIJIogKi4r5dNVuakcIDevVRoCm8dGMG96FYT0qOPbSju/hu5eg5+3Qbogr8RpjQkt5x2IyAfC3rzayYlcmr97ck2u6Nnav4LzDzkB8CS1h8J/dK9cYE1IsQQTJt5sy+OfcLYzq3dzd5AAwayxkpcKds6F2XXfLNsaEDKtiCoL92fk8Mm0lbRrU5alrXe4Q9+OnsOLfcOlvoHlvd8s2xoQUe4KoZMXFym+mrSQrt5B3f97b3bGWjuyFT38FjbtD/8fdK9cYE5LsCaKSTfxuG3M3ZvD7azrQvlE99wpWhZm/hIKjzkB84S7PPmeMCTn2BFGJVqdm8ZdZ6xncqSG3XOTypHxLJ8Gmr2DIXyHZhSayxpiQZ08QlSQ738ODU5aRVLc2f7mxq7szxB3YArOfgHMGwgV3u1euMSak2RNEJXlqxhp2Hsxh6j0XEx9Ty72CizxOb+nwWjDs/yDMcr4xxh2WICrBx8tS+Xh5Gg9f3obereu7W/j8FyBtCdw0Ceo1cbdsY0xIs9vNANu2/yhPzlhD79b13Z8+NG0pfDMeuvwUOg93t2xjTMizBBFABZ5iHpyyjMjwMF4c0Z3wMBffOxTkwMf3QmwjuHqCe+UaY4yXVTEF0F9nrWdN2mFev/V8d0Zn9fX1U3BgE9w2E6Lj3S3bGGMI8BOEiFwlIhtEZLOIjPGzv6WIzBGRVSLyjYg089nXQkS+EpEfRWSdiLQKZKxuS9mwjzfnb+O2i1tyZadG7ha+6b+w+A246AE4p7+7ZRtjjFfAEoSIhAOvAkOAjsAoEelY6rDngcmq2hV4Fhjns28yMEFVOwC9gX2BitVt+w7n8ei0lbRvFMvvru7gbuE5B+GTByC5Awx6yt2yjTHGRyCfIHoDm1V1q6oWAFOB60sd0xGY411OKdnvTSQRqvo1gKpmq2pOAGN1TXGx8si0lRwt8PDKzT2IinRxKA1V+OxhyDng9JaOjHKvbGOMKSWQCaIpsMtnPdW7zddK4Ebv8g1ArIgkAm2BTBH5WESWi8gE7xPJCUTkHhFZIiJLMjIyAvArnLl/ztvC/M37efq6TpzXINbdwldNg3WfwGVPQOOu7pZtjDGlBDJB+Guyo6XWHwX6i8hyoD+QBnhwXp5f6t1/AXAOcMdJham+rqq9VLVXcnLwZ0xbtvMQf/tqI9d0acyIC5q7W3jmLvjiUWhxMVzykLtlG2OMH4FMEKmA71WyGZDue4CqpqvqcFXtATzh3Zbl/e5yb/WUB5gB9AxgrBV2OK+Qh6Ysp1G9KP48vIu7Q2kUF8OM+0GL4YZ/QpiL1VbGGFOGQCaIxUAbEWktIrWAkcBM3wNEJElESmIYC0z0+W6CiJQ8FlwGrAtgrBWiqvzu49Xszsrj5VE9iIt2eSTVH16F7d/CkL9AQit3yzbGmDIELEF47/x/CcwGfgSmqepaEXlWRIZ6DxsAbBCRjUBD4Dnvd4twqpfmiMhqnOqqNwIVa0X9Z0kqn63azSNXtOX8lgnuFr53Lcx5FtpfC91/5m7ZxhhzCqJa+rVA9dSrVy9dsmRJpZ93874jXPeP7+jRIp53f36hu72lPfnwxmWQvQ9+sQDqJLlXtjHGACKyVFV7+dtnPakrIK+wiF++v5zoWuH83e2hNABSnoO9a+DmaZYcjDGVzhJEBYz/cj3r9xxh4h29aFjP5T4J27+D716G80dD28Hulm2MMeVgg/Wdpa/X7eXt77dzZ5/WXNa+obuF52XB9Pugfmu48k/ulm2MMeVkTxBnYXdWLo99uJJOTerx+JAATO/55Rg4nAp3fgW167pfvjHGlIM9QZyhomLl4akrKPAU849RPagd4XKfhHWfwMr34dJHofkF7pZtjDFnwJ4gztCrKZtZuO0gz/+kG+cku3x3P/sJWPE+NOkB/X/rbtnGGHOGLEGcgcXbD/Lifzdyffcm3Niz9LBSFaQKC16BiGgY/gaEu9zZzhhjzpBVMZVTVk4hv5qynOb1Y/jTsM7uDqUBMP/vzucVz0KSy1OTGmPMWbAEUQ6qyuMfrWLfkXxeHtmD2CgX7+5TxsHTcTDnGWf9y8ec9ZRxp/6eMcYEmFUxlcN7C3cya+0exg5pT7fmLk/v2f5q+P5laNAR0pbA01nulm+MMWfJniBOY8OeI/zxs3X0a5vM3Zee427hR/bAlFEQXR9Gvu9u2cYYU0H2BHEKuQVFPDhlGbFREfztJ90Ic3MojcJcmHoz5B6CO2dDbEPof9K03cYYEzSWIE7hj5+vY+PebCbf2Zvk2NruFazqzCudtgxG/Pv47HADx7p3DmOMqSCrYirDl6t38/7Cndzb7xz6tXV5trq5f4U1H8Hlf4AO17pbtjHGuMQShB+ph3J4/KNVdGsWx2+udHkojTUfwzd/hm6joM/D7pZtjDEusgRRiqeomIenrqBY4eVRPagV4eKfKG2pM3Vo84vgupfA7b4UxhjjInsHUcrLczaxZMchXhrZnZaJddwrOCsNptwMdRvAyPcgwsV3GsYYEwCWIHws2HKAf6Rs5qbzm3F9dxeH0ig4ClNHOZ+3TrfJf4wx1YIlCK+DRwt4+IPltE6swzNDO7lXcHExTL8X9qyGUR9Aw47ulW2MMQEU8glixvI0JsxeT1pmHgC/ubIldWq7+GdJeQ5+/BQG/xnaXuleucYYE2Ah/ZJ6xvI0xn68+lhyAPi/lC3MWJ7mzglWfgDfPg89b4OLfuFOmcYYU0lCOkFMmL2B3MKiE7blFhYxYfaGihe+axHM/CW0uhSu/pu1WDLGVDshnSDSM3PPaHu5Ze50htGIawY/nQwRtSpWnjHGBEFIJ4gm8dFntL1c8o/A+yPBU+C8lI6pf/ZlGWNMEIV0gnhscDuiI0+cUzo6MpzHBp9l7+niIvjoLshYDz99G5LbVjxIY4wJkpBuxTSsh9PXYcLsDaRn5tIkPprHBrc7tv2M/fdp2DgLrn4ezr3MvUCNMSYIQjpBgJMkzjoh+Fr+b2finwvuht53V7w8Y4wJspCuYnLN9u/g04fhnAFw1fhgR2OMMa6wBFFRB7fBB7dAQiv4yTsQHvIPZcaYGiKgCUJErhKRDSKyWUROmi5NRFqKyBwRWSUi34hIM599RSKywvszM5BxnrW8LHh/BGgx3PwBRLs8X7UxxgRRwG53RSQceBW4AkgFFovITFVd53PY88BkVX1HRC4DxgG3evflqmr3QMVXYUUe+M9oOLgFbp0BiecGOyJjjHFVIJ8gegObVXWrqhYAU4HrSx3TEZjjXU7xs7/q+upJ2DIHrnkBWl8a7GiMMcZ1gUwQTYFdPuup3m2+VgI3epdvAGJFJNG7HiUiS0TkBxEZ5u8EInKP95glGRkZbsZ+aksmwsLX4KIH4PzbK++8xhhTiQKZIPwNPqSl1h8F+ovIcqA/kAZ4vPtaqGov4GbgRRE5qQ5HVV9X1V6q2is52eV5o8uydS58/ii0uRKu/GPlnNMYY4IgkE1uUoHmPuvNgHTfA1Q1HRgOICJ1gRtVNctnH6q6VUS+AXoAWwIY7+nt3wzTboOktnDjWxAWfvrvGGNMNRXIJ4jFQBsRaS0itYCRwAmtkUQkSURKYhgLTPRuTxCR2iXHAH0A35fblS/3EEwZ4SSFm6dCVL2ghmOMMYEWsAShqh7gl8Bs4EdgmqquFZFnRWSo97ABwAYR2Qg0BJ7zbu8ALBGRlTgvr8eXav1UuYoKYdrtziitI95z+jwYY0wNJ6qlXwtUT7169dIlS5a4X7AqfP6I82J62GvQ/Wb3z2GMMUEiIku973tPYj2pT2fR605y6POwJQdjTEixBHEqm/4Ls8ZA+2th0B+CHY0xxlQqSxBl2bcePhwNDTrBDf+CMPtTGWNCi131/Dl6wGmxFBEFo6ZA7brBjsgYYyqdDT1amqcApt0Kh3fD6C8gvvnpv2OMMTWQJQhfqvD5r2HHd05HuGZ+X+wbY0xIsComXwtecWaG6/db6HJTsKMxxpigsieIEh/+HNZ8BB2HwYCxwY7GGGOCzp4gAPasgTUfQpPuTmc4a7FkjDGWIMjeB1NGOssjp0CtmODGY4wxVURoJ4iUcfB8G8jyTlvxQnt4Os7ZbowxIS6030EMHOv8gJMYns4KbjzGGFOFhPYThDHGmDJZgijRf0ywIzDGmCrFEkSJgda01RhjfFmCMMYY45clCGOMMX5ZgjDGGOOXJQhjjDF+WYIwxhjjl6hqsGNwhYhkADsqUEQSsN+lcKpzDGBxlGZxnKgqxFEVYoCaEUdLVU32t6PGJIiKEpElqhrUCSCqQgwWh8VRHeKoCjGEQhxWxWSMMcYvSxDGGGP8sgRx3OvBDoCqEQNYHKVZHCeqCnFUhRighsdh7yCMMcb4ZU8Qxhhj/Ar5BCEiE0Vkn4isCWIMzUUkRUR+FJG1IvKrIMURJSKLRGSlN45nghGHN5ZwEVkuIp8FKwZvHNtFZLWIrBCRJUGKIV5EPhSR9d7/Ry4OQgztvH+Dkp/DIvJwZcfhjeXX3v8/14jIFBGJClIcv/LGsLYy/xb+rlkiUl9EvhaRTd7PBDfOFfIJAngbuCrIMXiA36hqB+Ai4AER6RiEOPKBy1S1G9AduEpELgpCHAC/An4M0rlLG6iq3YPYnPElYJaqtge6EYS/i6pu8P4NugPnAznA9MqOQ0SaAg8BvVS1MxAOjAxCHJ2Bu4HeOP9NrhWRNpV0+rc5+Zo1Bpijqm2AOd71Cgv5BKGq84CDQY5ht6ou8y4fwbkANA1CHKqq2d7VSO9Ppb+kEpFmwDXAm5V97qpGROoB/YC3AFS1QFUzgxsVg4AtqlqRjqkVEQFEi0gEEAOkByGGDsAPqpqjqh5gLnBDZZy4jGvW9cA73uV3gGFunCvkE0RVIyKtgB7AwiCdP1xEVgD7gK9VNRhxvAj8FigOwrlLU+ArEVkqIvcE4fznABnAJG+V25siUicIcfgaCUwJxolVNQ14HtgJ7AayVPWrIISyBugnIokiEgNcDTQPQhwlGqrqbnBuOIEGbhRqCaIKEZG6wEfAw6p6OBgxqGqRtxqhGdDb+yhdaUTkWmCfqi6tzPOeQh9V7QkMwan661fJ548AegKvqWoP4CguVR+cDRGpBQwF/hOk8yfg3C23BpoAdUTklsqOQ1V/BP4CfA3MAlbiVBXXKJYgqggRicRJDu+p6sfBjsdbjfENlf9+pg8wVES2A1OBy0Tk35UcwzGqmu793IdT5967kkNIBVJ9nuQ+xEkYwTIEWKaqe4N0/suBbaqaoaqFwMfAJcEIRFXfUtWeqtoPp8pnUzDi8NorIo0BvJ/73CjUEkQVICKCU8f8o6q+EMQ4kkUk3rscjfOPcX1lxqCqY1W1maq2wqnK+J+qVvodIoCI1BGR2JJl4EqcqoVKo6p7gF0i0s67aRCwrjJjKGUUQape8toJXCQiMd5/N4MIUmMGEWng/WwBDCe4f5eZwO3e5duBT9woNMKNQqozEZkCDACSRCQV+IOqvlXJYfQBbgVWe+v/AX6nql9UchyNgXdEJBzn5mGaqga1mWmQNQSmO9chIoD3VXVWEOJ4EHjPW72zFRgdhBjw1rVfAdwbjPMDqOpCEfkQWIZTpbOc4PVm/khEEoFC4AFVPVQZJ/V3zQLGA9NE5Oc4SfQnrpzLelIbY4zxx6qYjDHG+GUJwhhjjF+WIIwxxvhlCcIYY4xfliCMMcb4ZQnCmDMgIkWlRjV1rVeziLQK5qjCxpQW8v0gjDlDud6hSIyp8ewJwhgXeOeN+It3Po1FInKed3tLEZkjIqu8ny282xuKyHTv3BsrRaRkuIhwEXnDO8fAV94e7cYEhSUIY85MdKkqphE++w6ram/gFZwRafEuT1bVrsB7wMve7S8Dc71zb/QE1nq3twFeVdVOQCZwY4B/H2PKZD2pjTkDIpKtqnX9bN+OM9nSVu/Ai3tUNVFE9gONVbXQu323qiaJSAbQTFXzfcpohTPEehvv+uNApKr+KfC/mTEnsycIY9yjZSyXdYw/+T7LRdh7QhNEliCMcc8In88F3uXvOT4l5s+A+d7lOcD9cGySpnqVFaQx5WV3J8acmWifEXfBmSu6pKlrbRFZiHPjNcq77SFgoog8hjMzXMlIrL8CXveOvlmEkyx2Bzx6Y86AvYMwxgXedxC9VHV/sGMxxi1WxWSMMcYve4Iwxhjjlz1BGGOM8csShDHGGL8sQRhjjPHLEoQxxhi/LEEYY4zxyxKEMcYYv/4f2k7mJVQuHkAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "deeper_model = nn.Sequential(nn.Flatten(),\n",
    "                             nn.Linear(in_features=784, out_features=512),\n",
    "                             nn.ReLU(),\n",
    "                             nn.Linear(in_features=512, out_features=10))\n",
    "summary(deeper_model, torch.zeros((1, 1, 28, 28)))  # (batch size, num channels, height, width)\n",
    "\n",
    "train_accuracies, val_accuracies = \\\n",
    "    UDA_pytorch_classifier_fit(deeper_model,\n",
    "                               torch.optim.Adam(deeper_model.parameters(),\n",
    "                                                lr=learning_rate),\n",
    "                               nn.CrossEntropyLoss(),  # includes softmax\n",
    "                               proper_train_dataset, val_dataset,\n",
    "                               num_epochs, batch_size)\n",
    "\n",
    "UDA_plot_train_val_accuracy_vs_epoch(train_accuracies, val_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convnets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========================================================\n",
      "        Kernel Shape     Output Shape  Params Mult-Adds\n",
      "Layer                                                  \n",
      "0_0    [1, 32, 3, 3]  [1, 32, 26, 26]   320.0  194.688k\n",
      "1_1                -  [1, 32, 26, 26]       -         -\n",
      "2_2                -  [1, 32, 13, 13]       -         -\n",
      "3_3                -        [1, 5408]       -         -\n",
      "4_4       [5408, 10]          [1, 10]  54.09k    54.08k\n",
      "---------------------------------------------------------\n",
      "                        Totals\n",
      "Total params            54.41k\n",
      "Trainable params        54.41k\n",
      "Non-trainable params       0.0\n",
      "Mult-Adds             248.768k\n",
      "=========================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Kernel Shape</th>\n",
       "      <th>Output Shape</th>\n",
       "      <th>Params</th>\n",
       "      <th>Mult-Adds</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Layer</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0_0</th>\n",
       "      <td>[1, 32, 3, 3]</td>\n",
       "      <td>[1, 32, 26, 26]</td>\n",
       "      <td>320.0</td>\n",
       "      <td>194688.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_1</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 32, 26, 26]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2_2</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 32, 13, 13]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3_3</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 5408]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_4</th>\n",
       "      <td>[5408, 10]</td>\n",
       "      <td>[1, 10]</td>\n",
       "      <td>54090.0</td>\n",
       "      <td>54080.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        Kernel Shape     Output Shape   Params  Mult-Adds\n",
       "Layer                                                    \n",
       "0_0    [1, 32, 3, 3]  [1, 32, 26, 26]    320.0   194688.0\n",
       "1_1                -  [1, 32, 26, 26]      NaN        NaN\n",
       "2_2                -  [1, 32, 13, 13]      NaN        NaN\n",
       "3_3                -        [1, 5408]      NaN        NaN\n",
       "4_4       [5408, 10]          [1, 10]  54090.0    54080.0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_convnet = nn.Sequential(nn.Conv2d(1, 32, 3),\n",
    "                               nn.ReLU(),\n",
    "                               nn.MaxPool2d(2),\n",
    "                               nn.Flatten(),\n",
    "                               nn.Linear(in_features=5408, out_features=10))\n",
    "summary(simple_convnet, torch.zeros((1, 1, 28, 28)))  # (batch size, num channels, height, width)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9555\n",
      "  Validation accuracy: 0.9508\n",
      "Epoch 2 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9733\n",
      "  Validation accuracy: 0.9674\n",
      "Epoch 3 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9812\n",
      "  Validation accuracy: 0.9757\n",
      "Epoch 4 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9837\n",
      "  Validation accuracy: 0.9788\n",
      "Epoch 5 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9870\n",
      "  Validation accuracy: 0.9801\n",
      "Epoch 6 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9876\n",
      "  Validation accuracy: 0.9794\n",
      "Epoch 7 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9893\n",
      "  Validation accuracy: 0.9801\n",
      "Epoch 8 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9910\n",
      "  Validation accuracy: 0.9816\n",
      "Epoch 9 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9899\n",
      "  Validation accuracy: 0.9821\n",
      "Epoch 10 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9920\n",
      "  Validation accuracy: 0.9813\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxU9dX48c/JnpBASAhLiEBYZN9DsG4IqLijSBVa26J1qa21y6Mttj6ttfWRVn99tE9bFRXUVkHEtXXBpQhaKyFhX0SQJWQhCQkJhASSSc7vjzsJQ5hsMEuW83695jV37nzn3pNR7pn7XUVVMcYYYxoKCXYAxhhj2iZLEMYYY7yyBGGMMcYrSxDGGGO8sgRhjDHGq7BgB+ArPXr00AEDBgQ7DGOMaVeysrIOqmqSt/c6TIIYMGAAmZmZwQ7DGGPaFRHZ19h7VsVkjDHGK0sQxhhjvLIEYYwxxqsO0wbhTXV1NTk5ORw7dizYoXQoUVFRpKSkEB4eHuxQjDF+1KETRE5ODnFxcQwYMAARCXY4HYKqUlxcTE5ODqmpqcEOxxjjRx06QRw7dsySg4+JCImJiRQVFQU7FGM6vTfW5/LIih3klVaSHB/NvTOGcu34vj47fodOEIAlBz+w79SY4HtjfS73vbaZyuoaAHJLK7nvtc0APksS1khtjDHt0CMrdtQnhzqV1TU8smKHz85hCcKPiouLGTduHOPGjaN379707du3/nVVVVWLjnHzzTezY0fT/8H/8pe/8OKLL/oiZGNMG6aqfHHgMH9ZuYvc0kqvZfIa2X86OnwVU2v4uj4vMTGRDRs2APDAAw8QGxvLPffcc1IZVUVVCQnxnqsXL17c7Hl+8IMfnHaMxpi27birhjW7S/hoewEfbi+sTwzhoUJ1zakLviXHR/vs3HYH4VZXn5dbWolyoj7vjfW5Pj/Xrl27GDVqFN/73veYMGEC+fn53H777aSlpTFy5EgefPDB+rLnn38+GzZswOVyER8fz/z58xk7dixf+9rXKCwsBOD+++/nscceqy8/f/580tPTGTp0KJ999hkAR48e5frrr2fs2LHMnTuXtLS0+uRlTHvxxvpczlvwL1Lnv815C/7ll3+fbUHJ0Spezcrh+y9mMfG3H/LtRRm8nLmf4X3ieHjWaDJ+MZ1HZo8lOjz0pM9Fh4dy74yhPouj09xB/OYfW9mWd7jR99dnl1JVU3vSvsrqGn62fBNLMrK9fmZEcld+ffXI04pn27ZtLF68mCeffBKABQsWkJCQgMvlYurUqcyePZsRI0ac9JmysjKmTJnCggUL+OlPf8qiRYuYP3/+KcdWVTIyMnjrrbd48MEHee+99/i///s/evfuzauvvsrGjRuZMGHCacVtTLAEolE2WFSVr4rK+XB7IR9uK2Bd9iFqFXrGRXL12GQuHt6Tcwf1IDriREKo+5utF1MANEwOze0/U4MGDWLSpEn1r5csWcKzzz6Ly+UiLy+Pbdu2nZIgoqOjufzyywGYOHEin3zyiddjz5o1q77M3r17Afj000/5+c9/DsDYsWMZOfL0EpsxwfLwu9u9Nso++M9tjE7pRv+EGMJC20+lSHVNLWv3lvDhtkI++qKAfcUVAIxM7spd04Zw8fCejEruRkhI470Grx3f16/JsdMkiOZ+6Z+34F9eG336xkfz8h1f83k8Xbp0qd/euXMnjz/+OBkZGcTHx3PTTTd5Hf0dERFRvx0aGorL5fJ67MjIyFPKqJ5aV2lMW6Wq7CuuYM2eYtbsLmHNnhIKDh/3WrbkaBXT/98qIkJDSO3RhcE9Y+sfQ3rFktqjC5FhoV4/G2hlFdV8/GUhH24v5OMdhRw55iIiLIRzByVy6wUDmT6sp0/bEM5Up0kQzbl3xtCTbl/B9/V5jTl8+DBxcXF07dqV/Px8VqxYwWWXXebTc5x//vksW7aMCy64gM2bN7Nt2zafHt+YM1FXxfK5Oxlk7CmuTwiJXSKYPDCB8uPVlFWe+qMoKTaS+ZcPY2dhObsKj7A1r4x3tuRT95soRKB/opM4htQljp5xDOrZhZgI/18C9xw86m5gLmDt3kPU1Co9YiO4fFRvpg/vxfmDe9Alsm1eiv0alYhcBjwOhALPqOqCBu/3BxYBSUAJcJOq5rjf+z1wpbvob1X1ZX/GGoj6vMZMmDCBESNGMGrUKAYOHMh5553n83P88Ic/5Nvf/jZjxoxhwoQJjBo1im7duvn8PMa0RG2tsqPgCGt2F7sTQgnFR52u3z3jIpk8MJHJqQmcMzCBQUmxiMgpbRDg/Ij75ZXDT/l3eqy6ht1FR9lVVM6ugiPu5FHOyi8KcdWeuJvuGx/NkF6xDE5y7jYG94xjcM9YukU3Pc9YUz0eXTW1rMsu5aPtBXywvYDdRUcBGNY7ju9NGcj04b0YlxLfZNVRWyH+qnoQkVDgS+ASIAdYC8xV1W0eZV4B/qmqz4vINOBmVf2WiFwJ/Bi4HIgEVgHTVLXRVua0tDRtuGDQ9u3bGT58uI//svbJ5XLhcrmIiopi586dXHrppezcuZOwsNP7jWDfrWmNmlple/5hPncnhLV7SyitqAaci/Tk1AQmD0xgcmoi/RNjGh2tf6Zd0atratlXfJRdheXsLChnV5Hz/FVROcddJ9obe8ZF1ieOwb3i6u88ErtE8OaGvFMSVVR4CDemncXhYy5W7iiktKKa8FDhnIGJXDy8F9OG9eSshJjT/Pb8S0SyVDXN23v+vINIB3ap6m53EEuBmYBn3cYI4Cfu7ZXAGx77V6mqC3CJyEbgMmCZH+Pt0MrLy5k+fToulwtV5amnnjrt5GBMc6pratmSW8aaPSWs2V1M5t5DHDnuVA/1T4zh0hG9mJyaSHpqQqsunGfaKBseGuK+S4jjslEn9tfUKrmHKtlZeOJuY2dhOcuzcjhadSIRdI8J5+jxmlM6rxyrruX5/+yje0w404b15OLhvbhgSA/iotr3jMf+vEL0BfZ7vM4BJjcosxG4Hqca6jogTkQS3ft/LSJ/BGKAqZycWAAQkduB2wH69evn6/g7lPj4eLKysoIdhmnHmvr1ftxVw6acsvoqo6x9h6hwX1gHJXXhqrHJnOO+Q+jdLSqYf4ZXoSFCv8QY+iXGMH14r/r9qsqBw8ecuw130mis27sAmfdfQmg7qDpqKX8mCG/fUsP6rHuAP4vIPGA1kAu4VPV9EZkEfAYUAf8BTmmdUtWFwEJwqph8F7oxxpO3MQg/e3UT727O5/AxF+uyD9VX0QztFcfsiSn1dwhJcZHBDP2MiAh9ukXTp1s0F56dBMDqL4u89nhMjo/uUMkB/JsgcoCzPF6nAHmeBVQ1D5gFICKxwPWqWuZ+7yHgIfd7LwE7/RirMaYJv3/vi1PGIFS5almxrYCRyV355uT+TB6YQPqABLp3iWjkKB1DMHs8Bpo/E8RaYIiIpOLcGcwBvuFZQER6ACWqWgvch9Ojqa6BO15Vi0VkDDAGeN+PsRpjPBw+Vk3W3kN87h6HkF/mfVVGAd6++4LABhdkwezxGGh+SxCq6hKRu4AVON1cF6nqVhF5EMhU1beAi4CHRURxqpjqZp0LBz5x92Q4jNP91fuoMGPMGSutqCJjjzMGYc2eYrblHaZWnQnhxqTEExsZRvnxU/8JtqVBXYHk7xHMbYVfx6Wr6juqeraqDnJXGaGqv3InB1R1uaoOcZe5VVWPu/cfU9UR7sc5qtpuZ5W76KKLWLFixUn7HnvsMb7//e83+pnY2FgA8vLymD17dqPHbditt6HHHnuMioqK+tdXXHEFpaWlLQ3ddGAHy4/zzuZ8HnhrK5c9tprxv/2A2/+Wxd8+30eXiDDumjaEl26dzKZfz+DVO8/ld9eO8vvEcKbtsX6O3qx8GKbe55NDzZ07l6VLlzJjxoz6fUuXLuWRRx5p9rPJycksX778tM/92GOPcdNNNxET43QjfOedd077WKZ9Kzh8rL7L6Zo9JewqLAeci/zE/t25cnQf0lMTGHtWPFHhp05L0ZmqVcwJliC8WbXAZwli9uzZ3H///Rw/fpzIyEj27t1LXl4e48aNY/r06Rw6dIjq6mp+97vfMXPmzJM+u3fvXq666iq2bNlCZWUlN998M9u2bWP48OFUVp7oRXHnnXeydu1aKisrmT17Nr/5zW/405/+RF5eHlOnTqVHjx6sXLmSAQMGkJmZSY8ePfjjH//IokWLALj11lv58Y9/zN69e7n88ss5//zz+eyzz+jbty9vvvkm0dGdsxqhPcstrXSSwW6nymiveyK42MgwJvbvzqwJfZmcmsjovt2ICGtZRUJnqVYxJ3SeBPHufDiwueXlF1/ZfJneo+HyBU0WSUxMJD09nffee4+ZM2eydOlSbrzxRqKjo3n99dfp2rUrBw8e5JxzzuGaa65pdATpE088QUxMDJs2bWLTpk0nTdf90EMPkZCQQE1NDdOnT2fTpk3cfffd/PGPf2TlypX06NHjpGNlZWWxePFi1qxZg6oyefJkpkyZQvfu3dm5cydLlizh6aef5oYbbuDVV1/lpptuav67MEGjqmSXVLBmd0l9o3JdN8yuUWGkpybU9zIa0adru5rx1ARX50kQzSndB2Ue4/r2feo8dzsL4vuf0aHrqpnqEsSiRYtQVX7xi1+wevVqQkJCyM3NpaCggN69e3s9xurVq7n77rsBGDNmDGPGjKl/b9myZSxcuBCXy0V+fj7btm076f2GPv30U6677rr6GWVnzZrFJ598wjXXXENqairjxo0DTp4u3ARPwwFq91x6NqNT4lmzp9hpWN5dwoHDTi+jhC4RpA9I4NYLUklPTWBY764drm++CZzOkyCa+aV/kge6wQNlPjv1tddey09/+lPWrVtHZWUlEyZM4LnnnqOoqIisrCzCw8MZMGCA1ym+PXm7u9izZw+PPvooa9eupXv37sybN6/Z4zQ1/1bdVOHgTBfuWZVlAu+VzP3c/8aW+kFouaWV/GTZxvr3e8RGMnlgAuekJjB5YCKDk2LbxSRwpn3oPAkiiGJjY7nooou45ZZbmDt3LuCsDtezZ0/Cw8NZuXIl+/bta/IYF154IS+++CJTp05ly5YtbNq0CXCmCu/SpQvdunWjoKCAd999l4suugiAuLg4jhw5ckoV04UXXsi8efOYP38+qsrrr7/O3/72N9//4e2cr9coB2eW0eKjVRSXH6f4aBUl5VWUHK2q31e3XeJ+eOtaChAfHc5r3z+X1B5dGq2WNOZMWYLwZsqpy3ieqblz5zJr1iyWLl0KwDe/+U2uvvpq0tLSGDduHMOGDWvy83feeSc333wzY8aMYdy4caSnpwPO6nDjx49n5MiRp0wVfvvtt3P55ZfTp08fVq5cWb9/woQJzJs3r/4Yt956K+PHj7fqJA8tXd6yospFcfmJC7q3C73nvoqqGq/nCw8VErpEkNAlksQuEfRPjCGhSwSL/73Xa/myymoGJsX69o82pgG/TfcdaDbdd2B19O+2sRUGI8NCGNo7rj4pNJx+ok5EaAiJsRHui34EiXUX/9i67Qj3+86+uMgwr3cCTa10+O/50878DzWdXrCm+zamXXLV1Hq9KAMcd9XSPSaCwUmxzsXffcFP7BJZv53QJYLYRi74rdWZ5v0xbY8lCGPcKqtqeCVrPwtX7260TN/4aJ6/JT1gMdkANRNMHT5BqKo14vlYR6mWrFNWUc0L/9nLc5/tpfhoFeP7xTNjRC9eysimsvrEwjDB+uVuA9RMsHToBBEVFUVxcTGJiYmWJHxEVSkuLiYqqu0t+tJaB8qO8eynu3lpTTZHq2qYOjSJ700ZRHpqAiLC6JR4++VuOrUOnSBSUlLIycmhqKgo2KF0KFFRUaSkpAQ7jNO2q7Cchau/4vX1udQqXDWmD3dcOIgRyV1PKme/3E1n16ETRHh4OKmpqcEOw7QR67MP8eSqr3h/WwERoSHMTe/HbRcMbLOLyRsTbB06QRijqqzeeZAnPt7F57tL6BoVxl1TB/OdcwfQI7b9LoVpTCBYgjAdkqumlne2HODJj79iW/5heneN4v4rhzMnvR+xkfa/vTEtYf9STIdyrLqGV7JyeHr1brJLKhiY1IU/zB7DteP6tnhaa2OMwxKE6RDKKqv5++f7WPzvPRwsr2LsWfH84orhXDqil01eZ8xpsgRh2rWCw8d49tM9vLQmm/LjLqac7XRVPWdggnVtNuYMWYIw7dLuonIWrt7Na+tycdXWctWYZO6YMpCRyd2CHZoxHYYlCNOubNxfypOrvuK9rQeICA3hxklncdsFA+mXaF1VjfE1SxCmzfG2glpibCRPrvqKz74qpmtUGD+4aDDzzrOuqsb4kyUI06Z4W4fhp69sRBV6dY3kF1cMY256P+KiwoMcqTEdnyUI06Y8smLHKWssqDorqK3+2VQiw0KDFJkxnY8lCNMmlFVU897W/EbXYSirrLbkYEyAWYIwQXOsuoaPthfy5oZcPt5RRFVNLaEhQk3tqdOJJ8dHByFCYzo3SxAmoFw1tXz2VTFvbMjl/a0FlB930TMukm99rT8zxyXzVWE5v3h9i62gZkwbYAnC+J2qsn5/KW9tyOOfm/I4WF5FXFQYV47uw8xxyUwemEioe7TzmJR4RMTWYTCmDbAEYfxmZ8ER3tyQx5sbc9lfUklEWAgXD+/JNWP7MnVYUqNtCrYOgzFtgyUI41O5pZX8Y2Meb27IY3v+YUIEzhvcgx9NP5sZI3tZ91TTsax8GKbeF+wo/MYShDljJUereGdzPm9tyCNjbwkA4/vF88DVI7hyTDJJcTaYzXRQqxZYgjCmoYoqFx9sK+CtDXms+rIIV60yuGcs91x6NlePTaZ/Ypdgh2g6Mn/9cleF44eh8tDJj4oSqCxtsN/5McTSb0JcH+ia7DzqtuP6QGSs72MMIEsQpsWqa2r5ZGcRb27I4/2tBVRW19CnWxTfPT+VmeP6MrxPnM2gagKjuV/utbUeF/oS93Opl4v+oVMfWtP4ccO7gIRA1ZET+774Z+PlI7tB1z7upNHXY7sumSRDTCKEtM21SixBmHoN50C6d8ZQrhmbTOa+Q7y5IZd3NudzqKKa+JhwrpvQl5ljk5k0IMHWWzCBc7wc8jc42x8vaOSX/iE4Vgpa2/hxIuIgujtEx0NMAnTr637d8JHgsR0PYQ2qSx/oBg+UOdtVR+FwPhzJg8Pux5H8E9tFX0B5walxhYR7JI0+TtLo2udEAqlLKg3P7clPd1SieuqgpPYoLS1NMzMzgx1Gu9VwDiSAsBAhNjKU0koX0eGhXDKiFzPHJXPBkCRbnc34X43LuajmZkJuFnzxLlQUnVoushskpJ64kMckNHOxj4dQH3WW8EwQLf2bjhZ6TyB120fyobri1M/G9GiQQPq6E0sf+Pv1rYvDg4hkqWqat/fsDsIA3udActUqldW1PD5nHBcP70UXW8vZ+IsqlOU4iSA3E3LXQd4GqD7qvB8VD30nOo+UNHjpBvjvYggN8v+TU+a3rnxo2InqpcaoOndA9XcjdYkj78S+3CyoOHhmsbeA/Ys3AOQ1MgdSlauWmeNsTILxsWNlkLcectzJIDfTqX4BCI2A3qNh/E0nEkLCQGjYvhXs5AD+aSgXOXHX02tE4+U++i188uiJ1w+4F8uaMt9ncbWBb9i0BT27RlJw+Pgp+20OJHPGaqqhYIvzqzcny3k++CXgrt5OHAwDL4K+aU5C6D2q6fp2aP0v945o+n87D2h9VVcLWYIwlFZU4a0pyuZAMq2mCof2uquKspw7hPyNUOP+8RHTw7kjGD3bXWU0wfml3FodeOxBW+LXBCEilwGPA6HAM6q6oMH7/YFFQBJQAtykqjnu9/4AXAmEAB8AP9KO0qLehhyrruG2FzIprajmrqmDeH19ns2B1FAHHy3bap7fR0XJiSqiuqRQUey8FxYFfcZB+m1OIuibBvH9Tq0qMmfOT3dUfksQIhIK/AW4BMgB1orIW6q6zaPYo8ALqvq8iEwDHga+JSLnAucBY9zlPgWmAB/7K97OqKZW+fHSDWTuO8Sf507gyjF9uGfGsGCH1XbU1sKhPU6f+8l3OL90O+vFzXUcygvhcK7zfZR85SSDkt3uAgJJw2Do5e47gzToOdx3vYVM0/z0A8afdxDpwC5V3Q0gIkuBmYBnghgB/MS9vRJ4w72tQBQQAQgQDhT4MdZOR1X5zT+28t7WA/z66hFcOaZPsEMKrupjULgNDmyGA5vcz1tO9KL5Q6rTXz22F8T2dJ7jep38OtbjdXg7aLuprXXGDJQXNHgUntg+4n4+VnryZ/d+6iSCCd92npPHQ2RccP4O4zf+TBB9gf0er3OAyQ3KbASux6mGug6IE5FEVf2PiKwE8nESxJ9VdXvDE4jI7cDtAP369fP9X9CBPbHqK174zz7uuHAgN5+XGuxwAquixGk0zd90IiEU7TgxgjYiDqK6nUgOdWqrnT700d3dXTIz4ehB6htbPUV2a0Ei6dW6UbQtreqqqjj1Qt/w4n+kwOmPX+s69fPhMSfiSxoKqRc64xH2fnKizJF8ZwRxr1HO+6ZD8meC8HYv3vBf0j3An0VkHrAayAVcIjIYGA6kuMt9ICIXqurqkw6muhBYCM5AOR/G3qG9mpXDH97bwbXjkvn5ZR24SkkVyvY7ScAzGZR5/G6J6+N0qRx6hfPcZwzEDzj5ot1UD5Eal9Mfve7ie+RAg4tzodOds7wQqspP/byEQpekppNIbE/nsWoBDLuy6Qt/eaEzxcQp5wlxzhPbE2J7Q8+RHufqCXG9T2xHxDZdleanHjOm7fFngsgBzvJ4nQLkeRZQ1TxgFoCIxALXq2qZ+87gc1Utd7/3LnAOThIxZ2DVl0X8/NVNnDc4kT/MHttxpsmoqYaDO09UD+VvdJ7rq0YEegyBs9Jh0neh9xjnEZt0ZucNDXMurnG9my97vNz51V6fSAobXOAPODGXFzY+H9BTF5z8OrLriQt979HOxd8zycR53qnYmt6mdfyZINYCQ0QkFefOYA7wDc8CItIDKFHVWuA+nB5NANnAbSLyMM6dyBTgMT/G2ilszinjzr9ncXavOJ68aWLbny6jsSqV4+VQsNWdDDY5dweF2090pQyLgp4jYOS1zkWz91hnwFHEac4w66seIpGxziNhYNPlamudCebKC+DTx2DzslPLpN8BFz8AETG+ia01bAxCp+HXuZhE5AqcC3sosEhVHxKRB4FMVX1LRGbj9FxSnLuDH6jqcXcPqL8CF7rfe09Vf9rUuWwupqZlF1cw64l/ExkWyuvfP5eeXaOCHVLzHugG9+x0Vw9tOnF3UPwV9bWV0d2dO4E+Y07cFSQObhujbH3NqnaMHwRtLiZVfQd4p8G+X3lsLweWe/lcDXCHP2PrTIrLj/PtRWtw1Sovfze97SeHA5thzZPO9qNDTuyP7+/cEYy50X1nMNqZsKyzdj01xs864M8s46miysUtz2eSX3aMl247h0FJbXQBk9oa+HIFvPtzKMs+9f3zfgyX/CbwcbUlVrVjAswSRAfmqqnlrpfWszmnlKe+lcbE/qcxpYG/HT8CG16Cz59wBqV17QsX/8bpX/+HVKtS8WSjuU2AWYLooFSVX76+hX99UchD143ikhG9gh3SyQ7thYynYd0LTrfMlHSY/isYfrWNvjWmjbAE0UE99uFOXs7cz93TBvPNyf2DHY5DFbL/A5//Fb542+mbP+JaOOdOZwK3hqxKxZigsgTRAb20JpvHP9rJDWkp/OSSs4MdjjOPz9bXncSQv9HpeXTej2HSrc5Sj42xKhVjgsoSRAfz4bYC7n9jM1OHJvHQdaORYPbwKS+CrMWw9hmnT3+PoXDVY04vpGD03zfGtIoliA5kXfYh7lqyjtF9u/GXb04gPDRIA+EObIE1T8CmV5zBa4MvcaqRBk2zLqnGtCOWIDqIr4rK+e5za+ndNYpn500iJiLA/2lra2HnCqcaac9qZ8K38TfB5O9BUhuo5jLGtJoliA6g8PAxvrMog9AQ4flb0ukR28xyjb5U1011zZPO2gCe3VRjEgIXhzHG5yxBtHNHjlUzb/FaSo5WsfT2c+ifeJrzDbXWoX2QsfDkbqrT/tu6qRrTgViCaMeqXLXc+fd1fFlwhGe+k8aYlHj/nlAVsj93d1P9Z/PdVI0x7ZoliHaqtlb52fKNfLrrII9+fSwXDe3pv5O5qjy6qW5oeTdVY0y7Zgminfr9ii94Y0Me984YyuyJKc1/oDXqptk+ehAyF8Pap62bqjGdkCWIdmjxv/fw1KrdfOuc/nz/okG+P8GqBc7i9JuWWTdVYzoxSxDtzNub8nnwn9uYMbIXD1wz0vcD4T75f87z5uXWTdWYTq7ZkVQicpeItMFpQDufz3cX85OXNzCxX3cenzOeUF8uF7ryYWdBmo8edF67KiHzWdjyqu/OYYxpV1pyB9EbWCsi63CWBF2h/lyGzni148ARbnshk36JMTzznTSiwn28vvCgafDp/zq9kfb926bZNsY0fwehqvcDQ4BngXnAThH5HxHxQ+W38SavtJLvLMogJiKU529JJz4mwrcnKNkDS+dCtxS48e++PbYxpt1q0WQ97juGA+6HC+gOLBeRP/gxNgOUVVQzb3EGR4+7eO7mdPrGR/v2BJWH4KUbQGvhm684o59tmm1jDC2oYhKRu4HvAAeBZ4B7VbVaREKAncDP/Bti53Wsuobb/pbJnoNHef7mdIb36erbE7iqYNm3nTuIb78Jie6bQptm2xhDy9ogegCzVHWf505VrRWRq/wTlqmpVX66bAMZe0r409zxnDu4h29PoApv/9SZWO/aJ2HAeb49vjGm3WtJFdM7QEndCxGJE5HJAKq63V+BdWaqym//uY13Nh/g/iuHc83YZN+f5N+Pwfq/wYX3wri5vj++Mabda0mCeAIo93h91L3P+MnC1bt57rO93Hp+KrdeMND3J9j6Bnz4AIy6Hqb+0vfHN8Z0CC2pYhLPbq3uqiUbYOdDb6zP5ZEVO8grrSQ+JpxDFdVcPTaZX1wx3Pcny8mC1+9wZl+d+VcbGW2MaVRL7iB2i8jdIhLufvwI2O3vwDqLN9bnct9rm8ktrUSBQxXVhAhMGRNKouQAABmLSURBVNKDEF8OhAMozYYlcyC2F8xdAuFRvj2+MaZDaUmC+B5wLpAL5ACTgdv9GVRn8siKHVRW15y0r1bhfz/c6dsTHSuDF28A13GnO2sXHzd6G2M6nGarilS1EJgTgFg6pbzSylbtPy01LnjlZijeCTe9CklDfXdsY0yH1ZJxEFHAd4GRQH2dhKre4se4Oo3k+GhyvSSDZF8NiFOFd++Frz6Ca/4PBl7km+MaYzq8llQx/Q1nPqYZwCogBTjiz6A6k3tnDCU89OS2hujwUO6d4aNf+Z//FTIXOQv8TPi2b45pjOkUWpIgBqvqfwNHVfV54EpgtH/D6jyuHd+XAYkxhIYIAvSNj+bhWaO5drwPVmr74m1Y8UsYfg1M//WZH88Y06m0pLtqtfu5VERG4czHNMBvEXUy+4qPsrPwKP91ydn8cPoQ3x04bz28eiv0nQDXPQUhLZp2yxhj6rUkQSx0rwdxP/AWEAv8t1+j6kSWrt1PiMDX087y3UHLcuGlORCTCHOW2PKgxpjT0mSCcE/Id1hVDwGrAT8M6+28qmtqeSUzh2nDetG7m4/GJBw/Ai/dCFVH4bvvQ1wv3xzXGNPpNFnvoKq1wF0BiqXT+Wh7IQfLjzM33Ud3DzUuWP5dKNwGNzwHvUb45rjGmE6pJRXTH4jIPSJylogk1D38HlknsHRtNr27RjHl7CTfHPD9X8LOFXDFH2Dwxb45pjGm02pJG0TdeIcfeOxTrLrpjOQcqmDVl0X8cNoQwkJ90IC8ZiGseRLO+QFMuvXMj2eM6fRaMpI6NRCBdDbLMnMAuCEt5cwP9uUKeO/nMPQKuPS3Z348Y4yhZSOpvY6uUtUXfB9O5+CqqeWVzP1cOCSJlO5n2MPowGZYfgv0GgXXPwMhob4J0hjT6bWkimmSx3YUMB1YB1iCOE2rviwiv+wYv776DBuRD+c7PZYiu8I3XoaILr4J0BhjaFkV0w89X4tIN5zpN5olIpcBjwOhwDOquqDB+/2BRUASzqp1N6lqjohMBf7Xo+gwYI6qvtGS87Z1SzL20yM2kunDz6ALatVRZ+ruylK45T3o6odV54wxndrptI5WAM0O+RWRUOAvwOXACGCuiDT8yfwo8IKqjgEeBB4GUNWVqjpOVccB09znfP80Ym1zCg4fY+WOQr6elkL46TZO19bAa7fDgU3w9cXQZ4xvgzTGGFrWBvEPnF5L4CSUEcCyFhw7Hdilqrvdx1kKzAS2eZQZAfzEvb0S8HaHMBt4V1UrWnDONu+VzP3U1CpzJp3B2IcPfgVf/BMu+z2cPcN3wRljjIeWtEE86rHtAvapak4LPtcX2O/xum6xIU8bgetxqqGuA+JEJFFViz3KzAH+6O0EInI77sWL+vXr14KQgqu2Vlm6dj/nDkqkf+JpthdkLoL//BnSb4dzvufbAI0xxkNL6jiygTWqukpV/w0Ui8iAFnzO23qZ2uD1PcAUEVkPTMFZtc5VfwCRPjgzx67wdgJVXaiqaaqalpTko8FmfvTproPkHKpkbvppJrNdH8Hb98CQS2HGw74NzhhjGmhJgngFqPV4XePe15wcwLMeJQXI8yygqnmqOktVxwO/dO8r8yhyA/C6qlbTASxdm033mHAuHXkajdMF2+CVedBzOMxeBKEtufkzxpjT15IEEaaqVXUv3NsRLfjcWmCIiKSKSAROVdFbngVEpId7QkCA+3B6NHmaCyxpwbnavKIjx3l/awHXT0ghMqyVYxXKC53urOExTnfWyDj/BGmMMR5akiCKROSauhciMhM42NyHVNWFM9HfCmA7sExVt4rIgx7HuwjYISJfAr2AhzzOMwDnDmRVi/6SNu7VdTm4apU5rZ2Yr7oSlsyFioPwjaXQzQcjr40xpgVaUk/xPeBFEfmz+3UO0KK1K1X1HeCdBvt+5bG9HFjeyGf34jR0t3uqystr95M+IIHBPVvx67+2Fl6/A3Kz4Ma/Q/J4/wVpjDENtGSg3FfAOSISC4iq2nrUrfT57hL2HDzKD6cNbt0H//Vb2PYmXPo7GH6Vf4IzxphGNFvFJCL/IyLxqlquqkdEpLuI/C4QwXUUSzKy6RoVxhWj+7T8Q+v/Dp/+ESbOg6/ZkhzGmMBrSRvE5apaWvfCvbrcFf4LqWM5dLSK97Yc4LrxfYkKb2Hj9O5V8I8fwcCpcMWjIN56DBtjjH+1JEGEikhk3QsRiQYimyhvPLy2PpeqmlrmTm7B2IeVD0PRl7DsW5A4GG54HkLD/R+kMcZ40ZJG6r8DH4nIYvfrm4Hn/RdSx6GqLM3IZtxZ8Qzr3bX5D6xaAJuWQmgEfGMZRHXzf5DGGNOIljRS/0FENgEX44yOfg/o7+/AOoKsfYfYWVjO768f3Xxh13Hn+cgBmPc2dLev2BgTXC2dTvQAzmjq63HWg9jut4g6kCUZ+4mNDOOqMU1Mxb3yYXigG/yup/PadQyeme7sN8aYIGr0DkJEzsYZ/TwXKAZexunmOjVAsbVrZZXVvL05j1kTUugS2cSN2tT74KL58NevQdF2+HWpNUobY9qEpqqYvgA+Aa5W1V0AIvKTJsobD29tyOVYdS1zJ7WgcXrvJ05yAEsOxpg2o6kqputxqpZWisjTIjId7zO0mgZUlZcy9jMyuSujU1rQ0LzmKYhOgAv+y//BGWNMCzWaIFT1dVW9EWe5z49xFvbpJSJPiMilAYqvXdqUU8b2/MMtm9a7NBt2vAMTvwPTf9V8eWOMCZBmG6lV9aiqvqiqV+FM2b0BmO/3yNqxpWuziQ4PZea4FqwTnemewDbtFv8GZYwxrdSqRZFVtURVn1LVaf4KqL0rP+7izQ15XDWmD3FRzQxyq66ErOdh6BUQ3/ZXxDPGdC6tShCmef/YmEdFVQ1zWlK9tOVVqCyByXf4PzBjjGklSxA+tjQjm6G94pjQL77pgqpO43TScBhwQWCCM8aYVrAE4UNb88rYmFPGnPSzkOa6q+5fAwc2Qfpt1rXVGNMmWYLwoaUZ+4kIC+G68S1Y5yhjIUR2gzE3+j8wY4w5DZYgfKSyqoY3NuRyxajexMc0s2T34XxnIaDxN0FkbGACNMaYVrIE4SNvb87nyDFXy8Y+ZC2G2hpIv9X/gRljzGmyBOEjSzOyGZjUhfTUhKYLuqogczEMuRQSBgYmOGOMOQ2WIHzgy4IjZO47xJxJLWic3vYmHC2E9NsDE5wxxpwmSxA+sDRjP+GhwvUTUpovnPEUJAyCQTbW0BjTtlmCOEPHqmt4bX0Ol47sTWJsMyux5q6DnLXO3UOIffXGmLbNrlJnaMXWA5RWVLdsWu+MhRARC+O+4f/AjDHmDFmCOENLMrI5KyGacwclNl2wvMiZWmPsHIhqwfrUxhgTZJYgzsCeg0f5fHcJcyb1IySkmcbpdc9DTZU1Thtj2g1LEGdg6dpsQkOEr09spnG6xuVM6z3wIkgaGojQjDHmjFmCOE1VrlqWZ+YwfVhPenaNarrwF/+Ew7mQbrO2GmPaD0sQp+nD7QUUH61q2cjpjIXOeg9nz/B/YMYY4yOWIE7TkoxskrtFceHZSU0XPLAF9v0bJt0KIaGBCc4YY3zAEsRp2F9Swae7DnLDpLMIba5xOmMhhEXD+G8FJjhjjPERSxCn4eW1+xHghrSzmi5YUQKblsGYr0NMM3M0GWNMG2MJopVcNbW8krWfKWcnkRwf3XTh9X8HV6V1bTXGtEuWIFpp5Y4iCg4fb75xurYG1j4N/c6F3qMDE5wxxviQJYhWWpKRTc+4SKYN69l0wZ3vQ2k2TLa7B2NM+2QJohXyyyr5eEchX09LISy0ma9uzVMQlwzDrgpMcMYY42OWIFph2docahVuTGumeqnoS9i9EibdAqHhgQnOGGN8zBJEC9XUKssy93PBkB70S4xpunDGQgiNgAnzAhKbMcb4gyWIFlq9s4jc0krmNDet97HDsHEJjJwFsc0MojPGmDbMrwlCRC4TkR0isktE5nt5v7+IfCQim0TkYxFJ8Xivn4i8LyLbRWSbiAzwZ6zNWZqRTWKXCC4Z0avpghuXQFW5NU4bY9o9vyUIEQkF/gJcDowA5orIiAbFHgVeUNUxwIPAwx7vvQA8oqrDgXSg0F+xNqfwyDE+2l7I9RNTiAhr4iurrXWql/qmQd+JgQvQGGP8wJ93EOnALlXdrapVwFJgZoMyI4CP3Nsr6953J5IwVf0AQFXLVbXCj7E2aXlWDq5aZc6kZkZO7/4XFO+CyTZrqzGm/fNngugL7Pd4nePe52kjcL17+zogTkQSgbOBUhF5TUTWi8gj7juSk4jI7SKSKSKZRUVFfvgToLZWWZqxn8mpCQxMim268JqF0KUnjLjWL7EYY0wg+TNBeJvFThu8vgeYIiLrgSlALuACwoAL3O9PAgYC8045mOpCVU1T1bSkJP80CP9ndzHZJRXNj5wu2e0Mjps4D8Ii/BKLMcYEkj8TRA7gWSeTAuR5FlDVPFWdparjgV+695W5P7veXT3lAt4AJvgx1kYtycimW3Q4l43q3XTBtc8603mn3RKYwIwxxs/8mSDWAkNEJFVEIoA5wFueBUSkh4jUxXAfsMjjs91FpO62YBqwzY+xelVcfpwVWw8wa0JfosKbWMuh6iis/xsMvwa69glcgMYY40d+SxDuX/53ASuA7cAyVd0qIg+KyDXuYhcBO0TkS6AX8JD7szU41UsfichmnOqqp/0Va2NeW5dLdY02X7206WU4VmaN08aYDiXMnwdX1XeAdxrs+5XH9nJgeSOf/QAY48/4mqKqLFmbzYR+8ZzdK66pgk7jdO/RcNbkwAVojDF+ZiOpG7F27yF2Fx1t/u5h76dQtB3S7wBpZnU5Y4xpRyxBNGJJRjZxkWFcOaaZNoWMpyA6AUbPDkxgxhgTIJYgvCirqOadzfnMHJ9MTEQTtXCl++GLt2HCtyG8mdXljDGmnbEE4cXr63M47qptfmK+zGed50nf9X9QxhgTYJYgGlBVlq7dz5iUbozq263xgtWVkPU8DL0C4ptJJMYY0w5Zgmhg/f5SvjhwpPm7hy2vQWUJpNusrcaYjskSRANLM7KJiQjlmnHJjRdSdRqnk4ZD6oWBC84YYwLIEoSHI8eq+cfGfK4ek0xsZBON0/szIH8jpN9mXVuNMR2WJQgPb27Io7K6hrmTm6leyngKIrvBmBsDE5gxxgSBJQgPS9dmM6x3HGNTmmicPpwP296E8d+EyGam/zbGmHbMEoTbltwytuQeZm56P6SpaqOs56C2BibdGrDYjDEmGCxBuC3JyCYyLIRrxzVc08iDqwqyFsOQSyBxUOCCM8aYIOj0CeKN9bl87eGPeHFNNiEirNzRxNLX296E8gJn3iVjjOng/Dqba1v3xvpc7nttM5XVNQBUVtdw32ubAbh2vJc7iYynIGEQDJoWyDCNMSYoOvUdxCMrdtQnhzqV1TU8smLHqYVz10HOWqdra0in/tqMMZ1Ep77S5ZVWtnx/xtMQ3gXGfcPPURljTNvQqRNEcrz3GVhP2X/0IGx5FcbNhagmusAaY0wH0qkTxL0zhhLdYK3p6PBQ7p0x9OSCWc9BzXGYdFvggjPGmCDr1I3UdQ3Rj6zYQV5pJcnx0dw7Y+jJDdQ1LshcBKlToOewIEVqjDGB16kTBDhJwmuPpTo73obDuXDFI4ELyhhj2oBOXcXUImsWOus9nH1ZsCMxxpiAsgTRlANbYN+nzrQaIaHNlzfGmA7EEkRTMhZCWBSM/1awIzHGmICzBNGYihLYtAxGfx1iEoIdjTHGBJwliMZseBFclTDZ5l0yxnROliC8qa1xRk73Oxd6jw52NMYYExSWILzZ+T6U7nPmXTLGmE7KEoQ3a56CuGQYfnWwIzHGmKCxBNFQ0ZeweyWk3QKh4cGOxhhjgsYSRENrn4bQCJg4L9iRGGNMUFmC8HTsMGx4CUbOgtikYEdjjDFBZQnC08YlUFUO6bcHOxJjjAk6SxB1amudkdN9J0LKxGBHY4wxQWcJos7ulVC8C9JtYJwxxoAliBMyFkKXJBh5bbAjMcaYNsESBEDJHvjyPafnUlhksKMxxpg2wRIEwNpnnOe0W4IbhzHGtCGWIKqOwvq/Odtdk4MbizHGtCGdO0GsfBj+JxmOlTmvH+jmPFY+HNy4jDGmDfDrmtQichnwOBAKPKOqCxq83x9YBCQBJcBNqprjfq8G2Owumq2q1/g8wKn3OQ9wJ4cyn5/CGGPaK78lCBEJBf4CXALkAGtF5C1V3eZR7FHgBVV9XkSmAQ8Ddcu3VarqOH/FZ4wxpmn+rGJKB3ap6m5VrQKWAjMblBkBfOTeXunl/cCZMj9opzbGmLbInwmiL7Df43WOe5+njcD17u3rgDgRSXS/jhKRTBH5XES8Dk4QkdvdZTKLiorOLNq6qiZjjDGAfxOEeNmnDV7fA0wRkfXAFCAXcLnf66eqacA3gMdEZNApB1NdqKppqpqWlGST6xljjC/5s5E6BzjL43UKkOdZQFXzgFkAIhILXK+qZR7voaq7ReRjYDzwlR/jNcYY48GfdxBrgSEikioiEcAc4C3PAiLSQ0TqYrgPp0cTItJdRCLrygDnAZ6N28YYY/zMbwlCVV3AXcAKYDuwTFW3isiDIlLXZfUiYIeIfAn0Ah5y7x8OZIrIRpzG6wUNej8ZY4zxM1Ft2CzQPqWlpWlmZmawwzDGmHZFRLLc7b2nvtdREoSIFAH7zuAQPYCDPgqnPccAFkdDFsfJ2kIcbSEG6Bhx9FdVr718OkyCOFMiktlYFu1MMVgcFkd7iKMtxNAZ4ujcczEZY4xplCUIY4wxXlmCOGFhsAOgbcQAFkdDFsfJ2kIcbSEG6OBxWBuEMcYYr+wOwhhjjFeWIIwxxnjV6ROEiCwSkUIR2RLEGM4SkZUisl1EtorIj4IUR5SIZIjIRnccvwlGHO5YQkVkvYj8M1gxuOPYKyKbRWSDiARlJKaIxIvIchH5wv3/yNeCEMNQ93dQ9zgsIj8OdBzuWH7i/v9zi4gsEZGoIMXxI3cMWwP5XXi7ZolIgoh8ICI73c/dfXGuTp8ggOeAy4Icgwv4L1UdDpwD/EBERgQhjuPANFUdC4wDLhORc4IQB8CPcKZoaQumquq4IPZ3fxx4T1WHAWMJwveiqjvc38E4YCJQAbwe6DhEpC9wN5CmqqNwVqucE4Q4RgG34ax7Mxa4SkSGBOj0z3HqNWs+8JGqDsFZY8cnC9x0+gShqqtxljsNZgz5qrrOvX0E5wLQcO2MQMShqlrufhnufgS8F4OIpABXAs8E+txtjYh0BS4EngVQ1SpVLQ1uVEwHvlLVM5m54EyEAdEiEgbE0GCW6AAZDnyuqhXueedW4axp43eNXLNmAs+7t58HvK6h01qdPkG0NSIyAGdq8zVBOn+oiGwACoEPVDUYcTwG/AyoDcK5G1LgfRHJEpHbg3D+gUARsNhd5faMiHQJQhye5gBLgnFiVc3FWao4G8gHylT1/SCEsgW4UEQSRSQGuIKTlzcItF6qmg/OD06gpy8OagmiDXGvifEq8GNVPRyMGFS1xl2NkAKku2+lA0ZErgIKVTUrkOdtwnmqOgG4HKfq78IAnz8MmAA8oarjgaP4qPrgdLin7r8GeCVI5++O82s5FUgGuojITYGOQ1W3A78HPgDew1kd09Xkh9ohSxBthIiE4ySHF1X1tWDH467G+JjAt8+cB1wjIntx1jGfJiJ/D3AM9TwWrirEqXNPD3AIOUCOx53ccpyEESyXA+tUtSBI578Y2KOqRapaDbwGnBuMQFT1WVWdoKoX4lT57AxGHG4FItIHwP1c6IuDWoJoA0REcOqYt6vqH4MYR5KIxLu3o3H+MX4RyBhU9T5VTVHVAThVGf9S1YD/QgQQkS4iEle3DVyKU7UQMKp6ANgvIkPdu6YT3MWz5hKk6iW3bOAcEYlx/7uZTpA6M4hIT/dzP5yVMYP5vbwFfMe9/R3gTV8c1J9LjrYLIrIEZ+GiHiKSA/xaVZ8NcBjnAd8CNrvr/wF+oarvBDiOPsDzIhKK8+NhmaoGtZtpkPUCXneuQ4QBL6nqe0GI44fAi+7qnd3AzUGIAXdd+yXAHcE4P4CqrhGR5cA6nCqd9QRvuotXRSQRqAZ+oKqHAnFSb9csYAGwTES+i5NEv+6Tc9lUG8YYY7yxKiZjjDFeWYIwxhjjlSUIY4wxXlmCMMYY45UlCGOMMV5ZgjCmFUSkpsGspj4b1SwiA4I5q7AxDXX6cRDGtFKleyoSYzo8u4Mwxgfc60b83r2eRoaIDHbv7y8iH4nIJvdzP/f+XiLyunvtjY0iUjddRKiIPO1eY+B994h2Y4LCEoQxrRPdoIrpRo/3DqtqOvBnnBlpcW+/oKpjgBeBP7n3/wlY5V57YwKw1b1/CPAXVR0JlALX+/nvMaZRNpLamFYQkXJVjfWyfy/OYku73RMvHlDVRBE5CPRR1Wr3/nxV7SEiRUCKqh73OMYAnCnWh7hf/xwIV9Xf+f8vM+ZUdgdhjO9oI9uNlfHmuMd2DdZOaILIEoQxvnOjx/N/3NufcWJJzG8Cn7q3PwLuhPpFmroGKkhjWsp+nRjTOtEeM+6Cs1Z0XVfXSBFZg/PDa657393AIhG5F2dluLqZWH8ELHTPvlmDkyzy/R69Ma1gbRDG+IC7DSJNVQ8GOxZjfMWqmIwxxnhldxDGGGO8sjsIY4wxXlmCMMYY45UlCGOMMV5ZgjDGGOOVJQhjjDFe/X9vBOJs1JgvpgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_accuracies, val_accuracies = \\\n",
    "    UDA_pytorch_classifier_fit(simple_convnet,\n",
    "                               torch.optim.Adam(simple_convnet.parameters(),\n",
    "                                                lr=learning_rate),\n",
    "                               nn.CrossEntropyLoss(),  # includes softmax\n",
    "                               proper_train_dataset, val_dataset,\n",
    "                               num_epochs, batch_size)\n",
    "\n",
    "UDA_plot_train_val_accuracy_vs_epoch(train_accuracies, val_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========================================================\n",
      "         Kernel Shape     Output Shape  Params Mult-Adds\n",
      "Layer                                                   \n",
      "0_0     [1, 32, 3, 3]  [1, 32, 26, 26]   320.0  194.688k\n",
      "1_1                 -  [1, 32, 26, 26]       -         -\n",
      "2_2                 -  [1, 32, 13, 13]       -         -\n",
      "3_3    [32, 16, 3, 3]  [1, 16, 11, 11]  4.624k  557.568k\n",
      "4_4                 -  [1, 16, 11, 11]       -         -\n",
      "5_5                 -    [1, 16, 5, 5]       -         -\n",
      "6_6                 -         [1, 400]       -         -\n",
      "7_7         [400, 10]          [1, 10]   4.01k      4.0k\n",
      "---------------------------------------------------------\n",
      "                        Totals\n",
      "Total params            8.954k\n",
      "Trainable params        8.954k\n",
      "Non-trainable params       0.0\n",
      "Mult-Adds             756.256k\n",
      "=========================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Kernel Shape</th>\n",
       "      <th>Output Shape</th>\n",
       "      <th>Params</th>\n",
       "      <th>Mult-Adds</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Layer</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0_0</th>\n",
       "      <td>[1, 32, 3, 3]</td>\n",
       "      <td>[1, 32, 26, 26]</td>\n",
       "      <td>320.0</td>\n",
       "      <td>194688.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1_1</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 32, 26, 26]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2_2</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 32, 13, 13]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3_3</th>\n",
       "      <td>[32, 16, 3, 3]</td>\n",
       "      <td>[1, 16, 11, 11]</td>\n",
       "      <td>4624.0</td>\n",
       "      <td>557568.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4_4</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 16, 11, 11]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5_5</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 16, 5, 5]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6_6</th>\n",
       "      <td>-</td>\n",
       "      <td>[1, 400]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7_7</th>\n",
       "      <td>[400, 10]</td>\n",
       "      <td>[1, 10]</td>\n",
       "      <td>4010.0</td>\n",
       "      <td>4000.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Kernel Shape     Output Shape  Params  Mult-Adds\n",
       "Layer                                                    \n",
       "0_0     [1, 32, 3, 3]  [1, 32, 26, 26]   320.0   194688.0\n",
       "1_1                 -  [1, 32, 26, 26]     NaN        NaN\n",
       "2_2                 -  [1, 32, 13, 13]     NaN        NaN\n",
       "3_3    [32, 16, 3, 3]  [1, 16, 11, 11]  4624.0   557568.0\n",
       "4_4                 -  [1, 16, 11, 11]     NaN        NaN\n",
       "5_5                 -    [1, 16, 5, 5]     NaN        NaN\n",
       "6_6                 -         [1, 400]     NaN        NaN\n",
       "7_7         [400, 10]          [1, 10]  4010.0     4000.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deeper_convnet = nn.Sequential(nn.Conv2d(1, 32, 3),\n",
    "                               nn.ReLU(),\n",
    "                               nn.MaxPool2d(2),\n",
    "                               nn.Conv2d(32, 16, 3),\n",
    "                               nn.ReLU(),\n",
    "                               nn.MaxPool2d(2),\n",
    "                               nn.Flatten(),\n",
    "                               nn.Linear(in_features=400, out_features=10))\n",
    "summary(deeper_convnet, torch.zeros((1, 1, 28, 28)))  # (batch size, num channels, height, width)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9578\n",
      "  Validation accuracy: 0.9557\n",
      "Epoch 2 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9709\n",
      "  Validation accuracy: 0.9708\n",
      "Epoch 3 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9774\n",
      "  Validation accuracy: 0.9768\n",
      "Epoch 4 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9782\n",
      "  Validation accuracy: 0.9782\n",
      "Epoch 5 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9806\n",
      "  Validation accuracy: 0.9797\n",
      "Epoch 6 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9844\n",
      "  Validation accuracy: 0.9822\n",
      "Epoch 7 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9854\n",
      "  Validation accuracy: 0.9835\n",
      "Epoch 8 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9877\n",
      "  Validation accuracy: 0.9854\n",
      "Epoch 9 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9867\n",
      "  Validation accuracy: 0.9833\n",
      "Epoch 10 [==================================================] 48000/48000\n",
      "  Train accuracy: 0.9896\n",
      "  Validation accuracy: 0.9875\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdeVyVddr48c/FIqCiKLiCiuaKGxrZnlultmnaZsu0jk9NTdP01KQz/XqapkZnbNpmmpoWrWwxMzMr0xyz3CrFcMU1RWVREQUXQDlw/f64b/SIqKDncBCu9+t1Xpx7PddNdi6+u6gqxhhjTEUFBToAY4wxZxdLHMYYYyrFEocxxphKscRhjDGmUixxGGOMqZSQQAdQFWJiYjQ+Pj7QYRhjzFll2bJlu1W1Sdn9tSJxxMfHk5ycHOgwjDHmrCIiW8vbb1VVxhhjKsUShzHGmEqxxGGMMaZS/NrGISKDgZeBYOAtVR1X5ngbYALQBNgD3K6q6e6xvwFXu6f+RVU/dve3BSYDjYGfgTtU9XBlYysqKiI9PZ3CwsLTejZzvPDwcOLi4ggNDQ10KMYYP/Jb4hCRYOBV4AogHVgqIjNUNdXrtOeB91T1XREZAIwF7hCRq4HeQCIQBnwvIl+r6j7gb8CLqjpZRF4H7gVeq2x86enpREZGEh8fj4icyaMaQFXJyckhPT2dtm3bBjocY4wf+bOqqg+wSVU3uyWCycDQMuckAHPd9/O8jicA36uqR1UPAiuAweJ8ww8AprrnvQsMO53gCgsLiY6OtqThIyJCdHS0leCMqQamp2Rw8bhvaTv6Ky4e9y3TUzJ8en9/Jo5YYLvXdrq7z9sKYIT7/nogUkSi3f1DRKSuiMQA/YFWQDSQq6qek9wTABEZJSLJIpKcnZ1dboCWNHzLfp/GBN70lAzGTFtFRm4BCmTkFjBm2iqfJg9/Jo7yvkXKzuH+GNBXRFKAvkAG4FHVb4CZwGLgI+AHwFPBezo7Vd9Q1SRVTWrS5LjxK8YYUyONn72egqLiY/YVFBUzfvZ6n32GPxNHOk4poVQckOl9gqpmqupwVe0F/Mndl+f+fE5VE1X1CpyEsRHYDUSJSMiJ7nm2yMnJITExkcTERJo3b05sbOyR7cOHK9bWf/fdd7N+/cn/Mbz66qt88MEHvgjZGHMWyMwtqNT+0+HPXlVLgQ5uL6gM4BbgVu8T3GqoPapaAozB6WFV2rAepao5ItID6AF8o6oqIvOAG3DaTO4EPvfjMxwxPSWD8bPXk5lbQMuoCB4f1IlhvcqtJauQ6Oholi9fDsDTTz9N/fr1eeyxx445R1VRVYKCys/vEydOPOXnPPjgg6cdozHm7JGRW8C/vt1YfhUM0DIqwmef5bcSh9sO8RAwG1gLTFHVNSLyjIhc557WD1gvIhuAZsBz7v5QYIGIpAJv4HTTLW3XeAJ4VEQ24bR5vO2vZyhVFXWGpTZt2kS3bt24//776d27N1lZWYwaNYqkpCS6du3KM888c+TcSy65hOXLl+PxeIiKimL06NH07NmTCy+8kF27dgHw5JNP8tJLLx05f/To0fTp04dOnTqxePFiAA4ePMiIESPo2bMnI0eOJCkp6UhSM8ZUbzv3FfLU56vpP/47pi5L55L20YSHHPvVHhEazOODOvnsM/06jkNVZ+K0VXjve8rr/VSO9pDyPqcQp2dVeffcjNNjy2f+/MUaUjP3nfB4yrZcDheXHLOvoKiYP0xdyUdLtpV7TULLBvzftV1PK57U1FQmTpzI66+/DsC4ceNo3LgxHo+H/v37c8MNN5CQcOyvJy8vj759+zJu3DgeffRRJkyYwOjRo4+7t6qyZMkSZsyYwTPPPMOsWbP45z//SfPmzfn0009ZsWIFvXv3Pq24jTFVZ/eBQ7z+3S9M+nErxSXKjUlxPDSgA7FRET6vISmrVkxyeKbKJo1T7T9T55xzDuedd96R7Y8++oi3334bj8dDZmYmqampxyWOiIgIhgwZAsC5557LggULyr338OHDj5yTlpYGwMKFC3niiScA6NmzJ127nl7CM8b4396Dh3ljwWbeXZxGYVEx1/eK43cDO9A6uu6Rc4b1ivVpoijLEgecsmRw8bhvySinYSk2KoKP/+dCn8dTr169I+83btzIyy+/zJIlS4iKiuL2228vd6xEnTp1jrwPDg7G4/Ecdw5AWFjYceeonqhW1BhTXewrLOKtBVuYsHALBw97uKZHSx65vAPnNKlf5bHYXFUV8PigTkSEBh+zz9d1hieyb98+IiMjadCgAVlZWcyePdvnn3HJJZcwZcoUAFatWkVqauoprjDGVJWDhzy8Om8Tl/5tHq/M3cgl7WOY9bvL+OfIXgFJGmAljgopLfL5s87wRHr37k1CQgLdunWjXbt2XHzxxT7/jN/+9rf86le/okePHvTu3Ztu3brRsGFDn3+OMabiCg4XM+nHNF7/fjN7Dh5mYOem/P6KjnSLDfz/m1IbqimSkpK07EJOa9eupUuXLgGKqHrxeDx4PB7Cw8PZuHEjV155JRs3biQkpPJ/V9jv1ZgzU1hUzEdLtvHv734he/8hLu0Qw++v6Ejv1o2qPBYRWaaqSWX3W4nDcODAAQYOHIjH40FV+c9//nNaScOYQPJ3TyJ/O+wp4ZNl2/nXt5vIyivk/LaNefXW3vRp2zjQoR3Hvh0MUVFRLFu2LNBhGHPaSsdalU61UTrWCqj2ycNTXMK0lAxembuR9L0F9G4dxfM39uSic3wwCeu8sdB/jG8C9WKJwxhz1vvrzLXlzs/05y/W0L5pfdo3rU94mQ4ugVZcony5MpOX/ruRLbsP0j22IX8Z2o1+nZr4bsLQ78dZ4jDGmFJ5BUXMWJHJJ8nb2bX/ULnn7M0v4pp/LiRIID66Hh2bRdKxeSSdmkXSqXl94qPrERJctZ1LS0qUWWt28OKcDWzcdYDOzSP5zx3ncmVCM98lDM8hmOvOMrFjFTTv7pv7uixxGGPOGiUlyg+bc5iSvJ1Zq3dwyFNC5+aRNIwIIa/g+LFLTSPDePq6rqzfsZ8NO/ezfud+vkndQYnbJ6hOcBDtmtSjU/NIOjYrTSiRxEZFEBTk22UCVJW5a3fxjzkbWJu1j3Oa1ONft/biqm4tfPtZX/0vLH3r6Pbrlzg/+472WenDEocxptpL35vPp8sy+GTZdtL3FtAgPISbz2vFTUmt6NqyAZ8vzzymjQOcsVZ/vKoLV3VvwVXdWxzZX1hUzKZdB44kkg079pOctpfPlx+daLtenWA6uImktITSsXl9mtQPq3SpQFWZv3E3L8zZwIrtubSJrssLN/VkaGIswb5MGKqwbCKkfAB1o2Hoq/DRLfB0nu8+w2WJI0D69evHmDFjGDRo0JF9L730Ehs2bODf//53udfUr1+fAwcOkJmZycMPP8zUqcdN80W/fv14/vnnSUo6rgfdMZ8zatQo6tZ1pii46qqr+PDDD4mKijrDpzLGdwqLivkmdSefJG9n4abdqMIl7WN4fFAnBnVtfkybRWXGWoWHBtMttuFx4yH2FRaxcaebUNwSytx1O/k4+eh6dI3qhjolk9ISSvNIOjaNpGHd0CPnePfuiq5fh8iwELbk5BMbFcHfRnRneO84Qn1dPZa/B2b8FtZ9Ce36w/WvQ2Rz336GF0scleWjXgojR45k8uTJxySOyZMnM378+FNe27Jly3KTRkW99NJL3H777UcSx8yZM09xhTFVZ3VGHlOStzM9JYN9hR5ioyL43cAOjOgdR6vGdU943ZnOz9QgPJRz2zTi3DbHjpfYfeAQG9ySyXo3sXz2cwb7Dx2tGmveIJyOzSMJFmXhphyKitW99jC7DxzmhnPj+Ov13akT4of2lM3fw2f/Awd3w5XPwQW/gdKlGPoeP9GpL1jiqCwf9VK44YYbePLJJzl06BBhYWGkpaWRmZlJYmIiAwcOZO/evRQVFfHss88ydOixS7WnpaVxzTXXsHr1agoKCrj77rtJTU2lS5cuFBQcnVPrgQceYOnSpRQUFHDDDTfw5z//mVdeeYXMzEz69+9PTEwM8+bNIz4+nuTkZGJiYnjhhReYMGECAPfddx+PPPIIaWlpDBkyhEsuuYTFixcTGxvL559/TkSE7+b3N7Xb3oOH+Xx5BlOS00nN2kedkCAGd23OTUmtuOicaJ+3N1RGTP0wYuqHcdE5MUf2qSpZeYVHqrrW73RKKKszyp9l+4dfcnyfNDyHYd6zsOgViOkAt34MLXoee44felSBJQ7H16OdngcVNfHqU5/TvDsMGXfCw9HR0fTp04dZs2YxdOhQJk+ezM0330xERASfffYZDRo0YPfu3VxwwQVcd911J6xXfe2116hbty4rV65k5cqVx0yJ/txzz9G4cWOKi4sZOHAgK1eu5OGHH+aFF15g3rx5xMTEHHOvZcuWMXHiRH766SdUlfPPP5++ffvSqFEjNm7cyEcffcSbb77JTTfdxKeffsrtt99esd+XMeUoLlEWbdrNx8nbmbNmJ4eLS9wuqV25rmfsMdU/1Y2I0DIqgpZREfTv1PTI/rajvyp3ISVfrr4HwO5N8Om9kLUczr0bBv0V6py4NOZrljgqIncr5B2t52TrQudnw1YQ1ea0b1taXVWaOCZMmICq8sc//pH58+cTFBRERkYGO3fupHnz8usr58+fz8MPPwxAjx496NGjx5FjU6ZM4Y033sDj8ZCVlUVqauoxx8tauHAh119//ZHZeYcPH86CBQu47rrraNu2LYmJicCxU7IbU1nbcvKZumw7U5elk5lXSFTdUG49vzU3JbUioWWDQId3RlpGRZQ7k7bPVt9ThZT34es/QEgY3Pw+dLnWN/euBEsccNKSwXGebuizXgrDhg3j0Ucf5eeff6agoIDevXvzzjvvkJ2dzbJlywgNDSU+Pr7cadS9lVca2bJlC88//zxLly6lUaNG3HXXXae8z8nmLSudjh2cKdm9q8SMOZWCw8XMWpPFlKXp/LA5BxG4rEMT/nR1ApcnNCUspHoNzjtdjw/qVG7vLp/MpF2wF774HaR+DvGXwvA3oEHLM7/vabDEEUD169enX79+3HPPPYwcORJwVvJr2rQpoaGhzJs3j61bt570HpdddhkffPAB/fv3Z/Xq1axcuRJwpmOvV68eDRs2ZOfOnXz99df069cPgMjISPbv339cVdVll13GXXfdxejRo1FVPvvsMyZNmuT7Bze1gqqyMt1p6J6xPJP9hzy0blyXx67syPDecT5dA7u68NtM2mmLYNooOLADLn8aLnoYggKXbC1xVJaPeymMHDmS4cOHM3nyZABuu+02rr32WpKSkkhMTKRz584nvf6BBx7g7rvvpkePHiQmJtKnj7Oqbs+ePenVqxddu3Y9bjr2UaNGMWTIEFq0aMG8efOO7O/duzd33XXXkXvcd9999OrVy6qlzEmVnVzwgX7tKCwq4ZPkdNbv3E94aBBXdWvBjUmtOL9tY/83dPtpfqaK8unqe8VF8N04WPAPaNwW7v0GYs/1zb3PgE2rbnzKfq+1S9nJBb0ltoripqRWXNOzBQ3Cq7Ch24fVyQG1ZzN8+mvISIZet8Pgv0FY1S7cFJBp1UVkMPAyEAy8parjyhxvA0wAmgB7gNtVNd099nfgapxVCucAv1NVFZHvgBZAaSX7laq6y5/PYYw56sAhD+t37CM1az9jy5lcEJypPqY/6PtFx8ql6vQuSv0cNnzj7FvzGXQcDKFnYXWYKqz82Jk6JCgYbnwHul4f6KiO4bfEISLBwKvAFUA6sFREZqiq97qkzwPvqeq7IjIAGAvcISIXARcDpV2AFgJ9ge/c7dtU9dgihDHGp1SV9L0FpGbtY+2R13627ck/5bXZJ5h00IfBQcbPkDrdSRi5ZdoCP7nL+dm8uzMoLv6SgLYJVFhhHnz5KKyeCm0uhuv/A1GtAh3VcfxZ4ugDbFLVzQAiMhkYCngnjgTg9+77ecB0970C4UAdQIBQYKevA1RV381GaU7aK8tUb/mHPazbsZ91WfuPJIl1O/ZzwB0dLe7sst1iG3DjuXF0adGAzi0iufk/P5CRe3xvPb80fJeUQMayo8kibzsEhUC7fnDZ49D5aqjb2Kmq+tXnsHIKpM6A966DyJbQ/QbocTM07+b72Hxh208w7T7Iy4ABT8Ilj1bbZOfPxBELeA1+IB04v8w5K4ARONVZ1wORIhKtqj+IyDwgCydx/EtV13pdN1FEioFPgWe1nG8sERkFjAJo3br1ccGFh4eTk5NDdLQPFksxqCo5OTmEh4cHOhRzEqpKRm4Ba7P2sy5rH2t3OKWItJyDlP5fVD8shM7NI7m+VyxdWjSgSwtnTqa6dY7/unh8UGf/dT8FJ1mkL4E102HtDNiXAUGhcM4A6DcGOl8FEeUsqdqun/O6+h+w/msnifz4b1j8CjRNgB43QfcboWGcb+I8E8UemD8e5v8dolrDPbOh1XmBjuqk/Jk4yvs2LvsF/xjwLxG5C5gPZAAeEWkPdAFK/6vOEZHLVHU+TjVVhohE4iSOO4D3jvsg1TeAN8BpHC97PC4ujvT0dLKzs0/r4czxwsPDiYurBv8j1hKnWiq1sKiY9Tv2Hyk9pGbtY13WPvYVHp1jqU10Xbo0b8DQxJZ0adGAhBYNiGsUUeE/pvzS/bSkGLb96JQq1s6A/VkQXAfaXw4Dn3LaLiJOMiGnd8/H0AjoNtx5HcyBNdOcJPLfp+G/f3aqsLrfCAlDT35Pf9m7Fab9Grb/BD1ugavGQ3j1HwTpt15VInIh8LSqDnK3xwCo6tgTnF8fWKeqcSLyOBCuqn9xjz0FFKrq38tccxeQpKoPnSyW8npVGXM2K683U53gIK5IaIqIsDZrH1t2Hzyy7kTdOsF0bh7pliBKSxENqB9WTXrklxTD1kVusvgCDuyE4DDocAUkDIOOg3z7hbpnM6ya6jRC52xyPqvjIKck0uFKZ1S2v62aCl+6NfVXvwA9bvT/Z1bSiXpV+TNxhAAbgIE4JYmlwK2qusbrnBhgj6qWiMhzQLGqPiUiNwO/BgbjlFxmAS8BXwNRqrpbREKBj4D/qurrJ4vFEoepaS4e9225U1sAxDWKOJIgElpE0rl5A1o3rhvQiQLLVexxpu9ZM92ZDvxgNoREOMmi6zDnCzws0r8xqEJmilMKWT3ViSG8odOLqcfN0OqCozPN+sqh/TDzcVjxEbQ63xkB3ijet5/hI1XeHVdVPSLyEDAbpzvuBFVdIyLPAMmqOgPoB4wVEcWpqnrQvXwqMABYhVO9NUtVvxCResBsN2kEA/8F3vTXMxhTXZ0oaQiw8IkBVRtMZRQXwZb5Tsli3ZeQnwOhdZ2/9hOGOsmiTr2qi0cEYns7ryufhc3fwaopTiJZ9g40bH20Ub3pyQfjVkh6sjM5Ye42p43m0scguJqU+iqh1g4ANOZslL3/EE9/sYavVmaVezw2KoJFo6tZ4vAchi3fO72h1n3lzLlUp77TVpEw1Gm7qMKZXSvk0AFYP9NJIL98C1oMzXs4VVndboAGLU59D28lxbDwBWdUe4NYGPEmtL7AP7H7UJVXVVUnljjM2U5VmbosnWe/WkvB4WIu79KUb9fvorCo5Mg5EaHBjB3e3XfTXVSW91QfnkPwyzynZLH+K2d8QlgD6DTESRbnDDh7Bucd2AWrpzntIZk/AwLt+jqlkM7XnLrtJS/dmWdq6yLoNsJpzwhEQ/xpsMRhicOcpbbl5DPms5Us2pTDefGNGDu8B+2b1j9lr6oq93RDuOUjp2Sx/ms4tA/CGjpdZhOGwTn9q6bR2Z92b3Krsj6GvWlOm0ynIU4SaT8Qgr2mVpk3Fpp2gS8edkocVz0PPW9xqsfOEpY4LHGYs4ynuISJi9L4x5z1hAQF8cSQztzWp3X1aeQ+dMCZ6iM9GdKXOm0WAOFRzl/iXYdB274QUiewcfqDqvPMK6fA6k+hYA/UjYauw53qrKYJMNZN4rHnwoi3oHG7wMZ8GgIyV5Ux5vSsycxj9KerWJWRx+VdmvGXYV1p0TCAVTslxZC93plwLz3ZGcG9KxW05PhzC3OdgXUdrqj6OKuKCLTq47wGj4VNc51SSMokWPqm070XnMbvfqOPLYnUAFbiMKYaKSwq5uW5G3lj/mYa1Q3lz9d146ruzat+doP9O9wE4SaKzOVweL9zLLyh81d0bBLEJTnv68XUnFlpz8Scp2HRi8fv7zs6oFO9ny4rcRhTzf24OYcx01axZfdBbkqK449XdSGqbhVU8xzOP1rllJEM6ctgX7pzLCgEmnWDnjcfTRSNz/H92Iaa4oqnnRfU6ERqicOYAMsrKGLc12v5aMl2Wjeuywf3nc/F7WNOfeHpKCmBnI3HliZ2rnG6m4IzV1KrPhD3GydRtOhR8d5PPl7kzFRfljiMCaBZq3fw1Oer2X3gEP9zWTseubwjEXUqOSPqyVa8O5Dt1S6RDBkpcMj9KzisgTPw7ZLfH61yqt/09B/mLKyK8asanEgtcRgTADv3FfJ/n69h1podJLRowNt3nkf3uIand7Pvxzlf2kUFkLXy2ESRu805R4KhWVfoPuJolVN0B6ty8qcanEgtcRhThVSVyUu389eZaznsKeGJwZ2579K2hAafxhe4qjOqGeA/fWHnaihxZ75t2MopQfQZ5VY59ax+o7PNWcsShzFVZMvug4yZtpIfN+/hgnaNGTu8B21jTmNepqJCZ4W7DV8f3Ze13PnZdbjTPTSyuU9iNqY8ljiM8bOi4hLeXLCZl/67kbCQIMYN787N57WqfBfbA9mQ/DYseRPyd0Oz7nDhgzD9/hrbe8dUT5Y4jPGjlem5PPHpKtZm7WNIt+b8+bquNG1QyVUSd62FH151RikXH4IOg5yE0fYyZyDa9Pv9E7wxJ2CJwxg/yD/s4cU5G3h74RZi6ofx+u3nMrhbJaqPStsvfngVfpnrzInU6zY4/wFo0vHYc2tw7x1TPVniMMbHFm7czZjPVrJ9TwEj+7Rm9JDONIyo4JQTRYXOJHo//Buy10L9ZjDgSTj3HqgXXf41Nbj3jqmeLHEY4yO5+Yd59qu1TF2WTtuYekwedQEXtDvBl31ZB7Jh6VvOq7T9YtjrzlrZZ/uMsqbGscRhzBlSVb5cmcWfv1jD3vwiftPvHB4e2IHw0AoM5DtV+4Ux1ZAlDmMqyXsdjGYNwoiuX4c1mfvpHtuQ9+45n4SWp1jYR9Vpt/jhVacd42TtF8ZUQ5Y4jKmE6SkZjJm2ioIiZ26nHfsOsWPfIYYltuT5G3sScrKBfKfTfmFMNWSJw5hK+NusdUeShrelaXtPnDSs/cLUMH5NHCIyGHgZCAbeUtVxZY63ASYATYA9wO2qmu4e+ztwNRAEzAF+p6oqIucC7wARwMzS/f58DlO7qSrLt+cy6cetZOUVlntOZm7B8Tut/cLUUH5LHCISDLwKXAGkA0tFZIaqpnqd9jzwnqq+KyIDgLHAHSJyEXAx0MM9byHQF/gOeA0YBfyIkzgGA15zLxjjGwWHi/liRSaTftzKqow86tUJpl6dYA4ePr7E0TLKnXrc2i9MLeDPEkcfYJOqbgYQkcnAUMA7cSQAv3ffzwOmu+8VCAfqAAKEAjtFpAXQQFV/cO/5HjAMSxzGh7bsPsgHP27lk2Xp5BUU0bFZff4ytCvX947jv6k7j7RxPBIylZc8NxARGswTl8fDz+9Z+4WpFfyZOGKB7V7b6cD5Zc5ZAYzAqc66HogUkWhV/UFE5gFZOInjX6q6VkSS3Pt43zO2vA8XkVE4JRNat27tg8cxNVlxifLtul1M+nEr8zdkExIkDOrWnDsuaMP5bRsfmVdqWC/nn9v42et5pHAac+pdyz/il9L52wet/cLUGv5MHOVV4pZti3gM+JeI3AXMBzIAj4i0B7oAce55c0TkMqCciuTj7unsVH0DeAOcNccrHb2pFXYfOMTHS7fz4U/byHC71/7+8o7c0qcVzU4wp9Swro0Y1gCYBF957of11n5hahd/Jo50oJXXdhyQ6X2CqmYCwwFEpD4wQlXz3NLCj6p6wD32NXABMImjyaTcexpzKqrKz9v2MumHrcxctYPDxSVcdE40T17dhcsTmh2/NobnsLMo0pb5kPI+5HkVpIsPOT9b9oJ2favuIYwJIH8mjqVABxFpi1OSuAW41fsEEYkB9qhqCTAGp4cVwDbg1yIyFqfk0hd4SVWzRGS/iFwA/AT8CvinH5/B1CD5hz18vjyTST9sJTVrH5FhIdx6fmtuv6A17ZtGHj2xpNhZ32LLfOe17UcoygfEWRCp6/XQti98MMKmMze1kt8Sh6p6ROQhYDZOd9wJqrpGRJ4BklV1BtAPGCsiilNV9aB7+VRgALAKpypqlqp+4R57gKPdcb/GGsbNKfySfYD3f9zK1GXp7C/00Ll5JM9d341hibHUCwtxekLtTHUTxfeQtujoutxNukCvO5wqqPiLIaJRYB/GmGpAasMQiKSkJE1OTg50GKYKeYpL+O/aXbz/41YWbtpNaLAwpFsL7riwDUmto5C9W46WKLbMdxq2ARq1dZJE28sg/lKIbHbiD5k31mamNTWaiCxT1aSy+23kuKlRsvcfYvKSbXy4ZBtZeYW0bBjOY1d2ZGTnEKJ3/Qgpb8C0+bDP7ZwX2QLaX+4mi0shqhI98CxpmFrKEoc566kqS9P2MunHrcxanUVRsXJVuxBe67WLHkUrCFo9H+b/4pwc0dhNEo867RTR51gvKGMqyRKHOWuUzkp744FJfFL/Dh4e0J6iEuX9H7eSsWMnfcM3MCl2K72KVxKWudbpb1cn0mmbOO9eJ2E07QpBJ5mI0BhzSpY4zFnBe1baR8Kn8XrutXwx/UMuClrDy2Hr6BC+iSBKYE84tL4AEm90ShQtEiHY/pkb40v2f5SpdopLlKy8Arbm5JOWc5CtOfl8sngdvUo2cH7IWgBWht1HHSnGQzDBsX2Qttc7JYq482zEtjF+ZonDBERRcQnpewtIyznINq8EkZZzkPQ9BYQVH+DcoPWcH7SOIcHreCJ4E8EhR3sA1hFnosH/eK7mwXsmBeoxjKmVLHGYCvFe9a5lVASPD+p0ZN6mEyksKmb7nnzScvLZmnPwSJE6gIUAACAASURBVHLYmpNPRm4BxSVHE0HLOgcZErmF20LW07XhaprmbySIEjQoFGLPZVJmD+YWtGdZSQdWh99HfOGHAMRGRRwZ/GOMqRqWOMwplV31LiO3gDHTVgFweUIztnolBO8EsWNfId7DhBqEh9A2ph6JraK4NaEOiSWptMtfTuPsZEJy1sFBICTcqW6KHw5tLkJik6BOXRqkZLBk2ioKSo5OaR4RGszjgzpV5a/CGIMlDlMB42evP27Vu4KiYh6dspySMuNHY+qH0Sa6LheeE018dD3aRNelTXQ92obk0HDXUkj7ArYuhvVu99g69d3G7JugzcXOnE/ltFF4z0r70oHhxFaw1GOM8T1LHOaUyl3dDihR+MPgTsckiPqlU3jk/AJbF8HmxTBv0dGJAcOjoM1FkHS3kyia96hwr6dhvWLdRDGAR3z0bMaYyrPEYU4pqm4oe/OLAI4sXgRO+8Jv+rWHkhLIXgcrFjnJYutiOLDTubheEydBXPSwkzCaJtg4CmPOcpY4zEkt3rSbfQVFBIlTwngkZBqveIbTK3Q7T7fPhY/egm2LoWCvc0GDWGf8RJuLIP4SiG5vI7ONqWEscZgTSs3cx6hJyzinaX3uurA1O/77KnhgZfgo6pMPq3EmBex0tTM6u81FENXGEoUxNZwlDlOu7XvyuWviEiLDQ/i08/dEzvrHkWP1yXfeXPAgDP5rgCI0xgSKJQ5znD0HD3PnxCUUFhUz9f4LiUyZ4xy45FFY+IItXmRMLWetlOYYBYeLuffdpWTsLeDtO5PouOp5+Ok1p3Qx8KlAh2eMqQYscZgjPMUlPPThz6zYnssrI3txXtobsOhlSLoXBj3ntF30HR3oMI0xAXbKxCEiD4mIrZdZw6kqf/psNXPX7eKZod0YtOcD+H4c9Lodrnr+aIO3LV5kTK1XkRJHc2CpiEwRkcEi1mWmJnpxzgY+Tt7OwwPac7t+CXOfge43wbWv2LgLY8wxTvmNoKpPAh2At4G7gI0i8lcROcfPsZkq8v6PW3nl203cnNSK30fNh9l/hIShMOw1CAoOdHjGmGqmQn9KqqoCO9yXB2gETBWRv5/sOreEsl5ENonIcZXjItJGROaKyEoR+U5E4tz9/UVkuderUESGucfeEZEtXscSK/nMxsus1Tt46vPVDOzclL+2XY7MfAw6XQUj3rYFkIwx5TrlN4OIPAzcCewG3gIeV9UiEQkCNgJ/OMF1wcCrwBVAOk511wxVTfU67XngPVV9V0QGAGOBO1R1HpDo3qcxsAn4xuu6x1V1auUe1ZS1NG0PD09OoWerKF7rsYngGQ9D+8vhxncgODTQ4RljqqmK/EkZAwxX1a3eO1W1RESuOcl1fYBNqroZQEQmA0MB78SRAPzefT8PmF7OfW4AvlbV/ArEaipow8793PvOUuIaRfDeBZnU+eI3zhQhN79vK+gZY06qIlVVM4E9pRsiEiki5wOo6tqTXBcLbPfaTnf3eVsBjHDfXw9Eikh0mXNuAT4qs+85t3rrRREp91tOREaJSLKIJGdnZ58kzNonK6+AOycsISw0mI8v20Pkl/dDq/Ph1o8hNCLQ4RljqrmKJI7XgANe2wfdfadSXu+rMqs38BjQV0RSgL5ABk4binMDkRZAd2C21zVjgM7AeUBj4InyPlxV31DVJFVNatKkSQXCrR3y8ou4c8ISDhR6+HTgQZp8PQpa9IRbp0CdeoEOzxhzFqhIVZW4jePAkSqqilyXDrTy2o4DMr1PUNVMYDiAiNQHRqiq93wWNwGfqWqR1zVZ7ttDIjIRJ/mYCigsKubX7yWTtjufz4YcpvV/R0HTLnD7pxDeINDhGWPOEhUpcWwWkYdFJNR9/Q7YXIHrlgIdRKStiNTBqXKa4X2CiMS4jezglCQmlLnHSMpUU7mlENzxJMNw5mg1p1BcojwyeTlL0vYwcUARXb+/Hxq3gzumQ4SN7zTGVFxFEsf9wEU41UjpwPnAqFNdpKoe4CGcaqa1wBRVXSMiz4jIde5p/YD1IrIBaAY8V3q9iMTjlFi+L3PrD0RkFbAKp+H+2Qo8Q62mqjw9Yw2z1uzgn5d6uPinB5x1M371OdQr26RkjDEnJ161UDVWUlKSJicnBzqMgHl13ibGz17PU+ce5p5Nv4O6jeHumdCgZaBDM8ZUYyKyTFWTyu6vyDiOcOBeoCsQXrpfVe/xaYTGL6Ykb2f87PX8pkshd2/+XwhvCHd+YUnDGHPaKlJVNQlnvqpBONVGccB+fwZlfOPbdTsZM20VN8fn8/jOPyAhEXDn5xDV6tQXG2PMCVQkcbRX1f8HHFTVd4GrcbrImmosZdtefvPBzwxseoCxB/6ESBDcOcNpEDfGmDNQkcRR2hU2V0S6AQ2BeL9FZM7Y5uwD3PPOUnrUy+O14qcJKvE4SSOmQ6BDM8bUABUZj/GGux7HkzjdaesD/8+vUZnTtmtfIb+asITm7OGDOs8RfPgA3PmlM17DGGN84KSJwx1jsU9V9wLzAavnqMb2FxZx58SlBB3cyfSovxFauNdp02jRI9ChGWNqkJNWValqCc5YDFPNHfIUc//7y8jZmcHXUc8TVrALbp8KsecGOjRjTA1TkTaOOSLymIi0EpHGpS+/R2YqrKREeeyTlazetJVvYl6gXn66M2Fh6wsCHZoxpgaqSBtH6XiNB732KVZtVW08N3Mt363YxNwmLxF1cAuMnAxtLw10WMaYGuqUiUNV21ZFIOb0vDl/M5MXpjIr+iWaHNwAN38A7QcGOixjTA1WkZHjvypvv6q+5/twTGVMT8ngHzOXM6PRK8Tlr0VunAidBgc6LGNMDVeRqqrzvN6HAwOBnwFLHAG0YGM2f/xkKVMa/pMOBSuREW9BwtBAh2WMqQUqUlX1W+9tEWmIMw2JCZDVGXn8dtJPTKz7T3oc+hmG/hu63xDosIwxtURFelWVlQ/YEOQA2ZpzkHsnLOblkFc435MM17wIvW4LdFjGmFqkIm0cX3B0ydcgIAGY4s+gTPl2HzjE3W//wNPF/6QvP8HgcZBkkxQbY6pWRdo4nvd67wG2qmq6n+IxZUxPyWD87PVk5hYQGqyMDfoPQ4IXweV/hgseCHR4xphaqCKJYxuQpaqFACISISLxqprm18gM01MyGDNtFQVFxTwSMpWm5DIieD5rOz9El0seCXR4xphaqiKJ4xOcpWNLFbv7ziv/dOMr42evp6CoGFAeCZkGwKue6/hwywAWBTY0Y0wtVpHEEaKqh0s3VPWwiNTxY0zGlZlbAMD/hnwCwFueIYz33IzkFQYyLGNMLVeRXlXZInJd6YaIDAV2+y8kU+rJep+TFn4rvw2ZDsB9IV+TFn4bT9b7PMCRGWNqs4okjvuBP4rINhHZBjwB/E9Fbi4ig0VkvYhsEpHR5RxvIyJzRWSliHwnInHu/v4istzrVSgiw9xjbUXkJxHZKCIf1+TST9gVf2Js0cgj2/GFH9Kl+GOir/m/AEZljKntTpk4VPUXVb0ApxtuV1W9SFU3neo6EQkGXgWGuNeOFJGEMqc9D7ynqj2AZ4Cx7mfOU9VEVU0EBuCMHfnGveZvwIuq2gHYC9xbgec8K+UePMRNwd+xQjoBEBsVwdjh3RnWKzbAkRljarNTJg4R+auIRKnqAVXdLyKNROTZCty7D7BJVTe7bSSTgbJzYiQAc93388o5DnAD8LWq5ouI4CSSqe6xd4FhFYjlrHPIU0zK4m84JyiLntf+FvqOZtHoAZY0jDEBV5GqqiGqmlu64a4GeFUFrosFtnttp7v7vK0ARrjvrwciRSS6zDm3AB+576OBXFX1nOSeNcKXK7IYdOgbikPqQtfrof+YQIdkjDFAxRJHsIiElW6ISAQQdpLzj5xazj4ts/0Y0FdEUoC+QAbOIMPSz2oBdAdmV+KepdeOEpFkEUnOzs6uQLjVh6rywYI1XBvyE0HdhkNY/UCHZIwxR1SkO+77wFwRmehu341TRXQq6UArr+04INP7BFXNBIYDiEh9YISq5nmdchPwmaoWudu7gSgRCXFLHcfd0+vebwBvACQlJZWbXKqrHzbn0D57DhGhhdC73FntjTEmYCrSOP534FmgC06bxCygTQXuvRTo4PaCqoNT5TTD+wQRiRGR0hjGABPK3GMkR6upUFXFaQspnQr2TqDG9U19e8EWbgv9npLoDtCqT6DDMcaYY1R0dtwdQAlOe8RAYO2pLnBLBA/hVDOtBaao6hoRecZrXEg/YL2IbACaAc+VXi8i8Tgllu/L3PoJ4FER2YTT5vF2BZ/hrLA5+wBp61PoyQaCet8BUl7tnDHGBM4Jq6pEpCNOKWEkkAN8DIiq9q/ozVV1JjCzzL6nvN5P5WgPqbLXplFOw7eqbsbpsVUjTVi0hZEh36NBIUjPkae+wBhjqtjJ2jjWAQuAa0vHbYjI76skqloqN/8w05el8UOdRUiHwVC/aaBDMsaY45ysqmoEThXVPBF5U0QGUn6vJuMjH/y0jYuKlxFZvBd63R7ocIwxplwnTByq+pmq3gx0Br4Dfg80E5HXROTKKoqv1jjsKeHdxWk80GAx1G8O7a8IdEjGGFOuivSqOqiqH6jqNTjdX5cDx807Zc7MV6syYf8OEg8thcSREFyRntLGGFP1KrXmuKruUdX/qOoAfwVUG6kqby3YwqiGPyFaAolWTWWMqb4qlTiMf/y0ZQ9rMvO4Kfh7aH0RxLQPdEjGGHNCljiqgbcWbGFgxCYa5G+F3ncEOhxjjDkpq0gPsC27DzJ33U6+iFsCeZGQUN4EwcYYU31YiSPAJi7aQlRQIV33fgvdhkOdeoEOyRhjTsoSRwDl5RfxSXI6f2ydingKbEJDY8xZwRJHAH24ZBsFRcVcXTwXmnSG2HMDHZIxxpySJY4AKSp2Bvzd1GY/dXelQC+b0NAYc3awxBEgM1dlsWNfIQ9F/QBBIdDzlkCHZIwxFWKJIwBKB/x1jKlDq+1fQKchUC8m0GEZY0yFWOIIgKVpe1mVkcef2m9F8nOglzWKG2POHpY4AuCtBZuJqhvKJfu/hsiW0H5goEMyxpgKs8RRxbbmHGTO2p3cnxhO8OZvnQkNg4IDHZYxxlSYJY4qNnFRGiFBwu0Ri0BLbN0NY8xZxxJHFcorKGJK8nau69Gc+qmTIf5SaNwu0GEZY0ylWOKoQpOXbCP/cDEPnbML9qZZacMYc1bya+IQkcEisl5ENonIcYs/iUgbEZkrIitF5DsRifM61lpEvhGRtSKSKiLx7v53RGSLiCx3X4n+fAZfKSou4Z3FaVzYLpq226ZBWAPocl2gwzLGmErzW+IQkWDgVWAIkACMFJGEMqc9D7ynqj2AZ4CxXsfeA8arahegD7DL69jjqprovpb76xl86evVO8jKK2RUnxhI/Ry63wB16gY6LGOMqTR/ljj6AJtUdbOqHgYmA2XnDE8A5rrv55UedxNMiKrOAVDVA6qa78dY/UpVeXvBZtrG1KPv4e/BU+hMMWKMMWchfyaOWGC713a6u8/bCmCE+/56IFJEooGOQK6ITBORFBEZ75ZgSj3nVm+9KCJh/noAX1m2dS8r0vO45+J4gpZPgqZdoWWvQIdljDGnxZ+Jo7wZ+7TM9mNAXxFJAfoCGYAHZ4GpS93j5wHtgLvca8YAnd39jYEnyv1wkVEikiwiydnZ2Wf2JGforQVbaBgRyg2t8iAzxVnlzyY0NMacpfyZONKBVl7bcUCm9wmqmqmqw1W1F/And1+ee22KW83lAaYDvd3jWeo4BEzEqRI7jqq+oapJqprUpEkTXz9bhW3Lyeeb1B3cdn5rIlZ9BMF1oMfNAYvHGGPOlD8Tx1Kgg4i0FZE6wC3ADO8TRCRGREpjGANM8Lq2kYiUfuMPAFLda1q4PwUYBqz24zOcsYmLtxAkwq/OawErJ0Onq6Bu40CHZYwxp81vicMtKTwEzAbWAlNUdY2IPCMipf1Q+wHrRWQD0Ax4zr22GKeaaq6IrMKp9nrTveYDd98qIAZ41l/PcKb2FRYxZel2ru3ZkuY7voWCvU41lTHGnMVC/HlzVZ0JzCyz7ymv91OBqSe4dg7Qo5z9A3wcpt98vGQ7Bw8Xc+8lbeHbZ6BBHLTrH+iwjDHmjNjIcT/xuAP+zm/bmG719sEv30LirTahoTHmrGeJw09mrdlBRm6BU9pY8RGg0Ou2QIdljDFnzBKHn7y1YAvx0XUZ2LkJpEyCtn2hUXygwzLGmDNmicMPlm3dy/LtudxzSVuCty6A3G02UtwYU2NY4vCDtxdupkF4CCN6x0HK+xDeELpcE+iwjDHGJyxx+Nj2PfnMWr2DW89vQ72S/ZA6A7rfBKERgQ7NGGN8whKHj72zOI0gEe68qA2smgrFh2zdDWNMjWKJw4f2Fxbx8dLtXN2jBS0aRjiN4s27Q8uzYskQY4ypEEscPvTx0u0cOORxuuBmrYSsFdDrV4EOyxhjfMoSh494ikuYuCiNPvGN6REX5ZQ2gsOcBZuMMaYGscThI9+k7nQG/F3aFooKYeUUpyeVTWhojKlhLHH4yFsLNtO6cV0u79IM1n0Jhbk2dsMYUyNZ4vCBn7ft5edtudxzcTzBQeJUUzVs7YwWN8aYGsYShw+8vXALkeEh3JjUCvZuhc3fO/NSBdmv1xhT89g32xlK35vP16uyuLVPa+qFhcDyD50DibcGNjBjjPETSxxn6N3FaYgId14UDyXFsPwDaNcPoloHODJjjPEPSxxn4MAhD5OXbOeq7i1oGRUBW76HvO22yp8xpkazxHEGpizdzv7SAX8AP0+CiEbQ2SY0NMbUXJY4TlNxiTJx8RaS2jQisVUU5O9xuuF2vwlCwgIdnjHG+I0ljtM0J3UH2/cUcN+lbmlj1SdQfNiqqYwxNZ4ljtP01oIttGocwRUJzUHVqaZqkehMamiMMTWYXxOHiAwWkfUisklERpdzvI2IzBWRlSLynYjEeR1rLSLfiMhaEUkVkXh3f1sR+UlENorIxyJSx5/PUJ7l23NJ3rqXuy9q6wz4y1oOO1fZ9OnGmFrBb4lDRIKBV4EhQAIwUkQSypz2PPCeqvYAngHGeh17Dxivql2APsAud//fgBdVtQOwF7jXX89wIm8v3EJkWAg3ndfK2ZHyPoSEQ/cbqzoUY4ypcv4scfQBNqnqZlU9DEwGhpY5JwGY676fV3rcTTAhqjoHQFUPqGq+iAgwAJjqXvMuMMyPz3CcjNwCZq7K4pY+ragfFgJFBbDyE+hyHUREVWUoxhgTEP5MHLHAdq/tdHeftxXACPf99UCkiEQDHYFcEZkmIikiMt4twUQDuarqOck9ARCRUSKSLCLJ2dnZPnokeG9xGoAz4A9g7ZdwKM+qqYwxtYY/E4eUs0/LbD8G9BWRFKAvkAF4gBDgUvf4eUA74K4K3tPZqfqGqiapalKTJk1O6wHKOnjIw4dLtjG4W3PiGtV1dqa8B1FtIP5Sn3yGMcZUd/5MHOlAK6/tOCDT+wRVzVTV4araC/iTuy/PvTbFrebyANOB3sBuIEpEQk50T3/6JHk7+ws93Fc64G/PFtgy35k+3SY0NMbUEv78tlsKdHB7QdUBbgFmeJ8gIjEiUhrDGGCC17WNRKS0qDAASFVVxWkLKV1W707gcz8+wxHFJcqERWn0bh1Fr9aNnJ3LPwQEEkdWRQjGGFMt+C1xuCWFh4DZwFpgiqquEZFnROQ697R+wHoR2QA0A55zry3GqaaaKyKrcKqo3nSveQJ4VEQ24bR5vO2vZ/D237U72bYnn/subefsKJ3QsP1AaBh38ouNMaYGCTn1KadPVWcCM8vse8rr/VSO9pAqe+0coEc5+zfj9NiqUm8v2EJsVARXJjRzdvwyD/ZlwKC/VnUoxhgTUFYxXwEr03NZkraHuy+OJyTY/ZWlTIKIxtBpSGCDM8aYKmaJowLeXriF+mEh3Fw64O9gDqz7CnreYhMaGmNqHUscp5CVV8BXK7O4+bxWRIaHOjtXfgwlRU5vKmOMqWX82sZxNpueksH42evJyC0AoGVUuHNA1ZlipGVvaFZ2BhVjjKn5LHGUY3pKBmOmraKgqPjIvudnbyC6XhjDmu6AXWvgmhcDGKExxgSOVVWVY/zs9cckDYCComLGz17vTJ8eEgHdRpzgamOMqdkscZQj062eKmtPbi6s/hQShkJ4wyqOyhhjqgdLHOVoGRVR7v6Rkcvh0D5b5c8YU6tZ4ijH44M6EREafMy+iNBgftNgMTRuB20uDlBkxhgTeJY4yjGsVyxjh3cnNioCAWKjInjlygbE5CyFxNtAypuk1xhjagfrVXUCw3rFMqyX11Ifc58BCYLEWwMXlDHGVANW4qiIYo8zE277K6BBy0BHY4wxAWWJoyJ++Rb2Z9kqf8YYgyWOikl5D+rGQMfBgY7EGGMCzhLHqRzIhvVfuxMa1gl0NMYYE3CWOE5l5cdQ4rEJDY0xxmWJ42RUYeELEHceNO0c6GiMMaZasMRxMunJkJ9jpQ1jjPFiieNkUiY5P7teH9g4jDGmGrEBgOWZNxa+H3d0e5y78l/f0dB/TGBiMsaYasKviUNEBgMvA8HAW6o6rszxNsAEoAmwB7hdVdPdY8XAKvfUbap6nbv/HaAvkOceu0tVl/s08P5jjiaIpxvC03knP98YY2oRvyUOEQkGXgWuANKBpSIyQ1VTvU57HnhPVd8VkQHAWKC0QaFAVRNPcPvHVXWqv2I3xhhzYv5s4+gDbFLVzap6GJgMDC1zTgIw130/r5zjgdd3dKAjMMaYasWfiSMW2O61ne7u87YCKF1K73ogUkSi3e1wEUkWkR9FZFiZ654TkZUi8qKIhJX34SIyyr0+OTs7+/Sfwto0jDHmGP5MHOXNPa5lth8D+opICk67RQbgcY+1VtUk4FbgJRE5x90/BugMnAc0Bp4o78NV9Q1VTVLVpCZNmpzZkxhjjDnCn4kjHWjltR0HZHqfoKqZqjpcVXsBf3L35ZUec39uBr4DernbWeo4BEzEqRIzxhhTRfyZOJYCHUSkrYjUAW4BZnifICIxIlIawxicHlaISKPSKigRiQEuBlLd7RbuTwGGAav9+AzGGGPK8FuvKlX1iMhDwGyc7rgTVHWNiDwDJKvqDKAfMFZEFJgPPOhe3gX4j4iU4CS3cV69sT4QkSY4VWHLgfv99QzGGGOOJ6plmx1qnqSkJE1OTg50GMYYc1YRkWVuW/Ox+2tD4hCRbGDraV4eA+z2YTiny+KoXjGAxVGWxXGs6hDHmcbQRlWP611UKxLHmRCR5PIyrsVRu2OwOCyOsyEOf8VgkxwaY4ypFEscxhhjKsUSx6m9EegAXBbHUdUhBrA4yrI4jlUd4vBLDNbGYYwxplKsxGGMMaZSLHEYY4ypFEscJyAiE0Rkl4gEbEoTEWklIvNEZK2IrBGR3wUojnARWSIiK9w4/hyIOLziCRaRFBH5MoAxpInIKhFZLiIBG10qIlEiMlVE1rn/Ti4MQAyd3N9D6WufiDwSgDh+7/77XC0iH4lIeFXH4MbxOzeGNVX5eyjvO0tEGovIHBHZ6P5s5IvPssRxYu8AgwMcgwf4X1XtAlwAPCgiCQGI4xAwQFV7AonAYBG5IABxlPodsDaAn1+qv6omBriv/svALFXtDPQkAL8XVV3v/h4SgXOBfOCzqoxBRGKBh4EkVe2GM83RLVUZgxtHN+DXOJOv9gSuEZEOVfTx73D8d9ZoYK6qdsBZ+8gnCwxZ4jgBVZ2Ps5xtIGPIUtWf3ff7cb4Uyq5pUhVxqKoecDdD3VdAelWISBxwNfBWID6/OhGRBsBlwNsAqnpYVXMDGxUDgV9U9XRnajgTIUCEiIQAdSkzG3cV6QL8qKr5quoBvsdZa8jvTvCdNRR4133/Ls7EsGfMEsdZQkTicaaW/ylAnx8sIsuBXcAcVQ1IHMBLwB+AkgB9fikFvhGRZSIyKkAxtAOygYlu1d1bIlIvQLGUugX4qKo/VFUzcJai3gZkAXmq+k1Vx4EzW/dlIhItInWBqzh2eYmq1kxVs8D5QxRo6oubWuI4C4hIfeBT4BFV3ReIGFS12K2KiAP6uEXyKiUi1wC7VHVZVX92OS5W1d7AEJwqxMsCEEMI0Bt4zV3T5iA+qoo4He7yCdcBnwTgsxvh/HXdFmgJ1BOR26s6DlVdC/wNmAPMwlnl1HPSi85CljiqOREJxUkaH6jqtEDH41aFfEdg2n8uBq4TkTScNewHiMj7AYjDe6GxXTj1+YFYUCwdSPcq/U3FSSSBMgT4WVV3BuCzLwe2qGq2qhYB04CLAhAHqvq2qvZW1ctwqo42BiIO106vNYxa4NQYnDFLHNWYu1jV28BaVX0hgHE0EZEo930Ezv+k66o6DlUdo6pxqhqPUyXyrapW+V+VIlJPRCJL3wNXEoAFxVR1B7Bd/n97988aRRSFYfx5iX+IiE2CISAxhcFCsBARwTL4BRQJQSSIhYiolQg2NhbaBm0UBQUVghJsRIQUgiixUFRsg4gQwYAigoQQjsWc1SUm4MDMjsr7a+bu3WXmzsLumbkzc460NbuGyYJnDRmlgWmq9B7YLWld/m6GaegGCkkbczkA7KO57wSK4nlj2R4D7lex0toKOf3rJN2hKDTVK+kDcC4irnV4GHuAQ8CbvL4AcDYiHnR4HP3ADUldFAcbExHR2K2wf4E+YLL4f2IVcDsiHjY0lhMUxc3WADPA4SYGkfP5e4GjTWw/IqYl3QVeUEwNvaS5lB/3JPUAC8DxiPjciY0u958FXAAmJB2hCK4HKtmWU46YmVkZnqoyM7NSHDjMzKwUBw4zMyvFgcPMzEpx4DAzs1IcOMwqIGlxSYbYyp7gljTYZJZms6X8HIdZNb5nShaz/57POMxqlHU7LmY9k+eStmT/ZklTkl7nciD7+yRNZu2TV5JaaTO6JF3NGg+P8gl+s0Y4cJhVo3vJVNVI23tfI2IX1yonkwAAAQtJREFUcIkiuy/ZvhkR24FbwHj2jwOPs/bJDuBt9g8BlyNiG/AF2F/z/pityE+Om1VA0reIWL9M/zuKIlgzmbDyY0T0SJoD+iNiIftnI6JX0idgU0TMt61jkCKV/VC+PgOsjojz9e+Z2e98xmFWv1ihvdJnljPf1l7E1yetQQ4cZvUbaVs+y/ZTfpU2PQg8yfYUcAx+Fs/a0KlBmv0pH7WYVaO7LYMxFHXAW7fkrpU0TXGgNpp9J4Hrkk5TVPFrZbU9BVzJbKaLFEFktvbRm5XgaxxmNcprHDsjYq7psZhVxVNVZmZWis84zMysFJ9xmJlZKQ4cZmZWigOHmZmV4sBhZmalOHCYmVkpPwAByIRfWrmnxwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_accuracies, val_accuracies = \\\n",
    "    UDA_pytorch_classifier_fit(deeper_convnet,\n",
    "                               torch.optim.Adam(deeper_convnet.parameters(),\n",
    "                                                lr=learning_rate),\n",
    "                               nn.CrossEntropyLoss(),  # includes softmax\n",
    "                               proper_train_dataset, val_dataset,\n",
    "                               num_epochs, batch_size)\n",
    "\n",
    "UDA_plot_train_val_accuracy_vs_epoch(train_accuracies, val_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Finally evaluate on test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_dataset = torchvision.datasets.MNIST(root='data/',\n",
    "                                          train=False,\n",
    "                                          transform=transforms.ToTensor(),\n",
    "                                          download=True)\n",
    "test_images = torch.tensor([image.numpy() for image, label in test_dataset])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.9243\n"
     ]
    }
   ],
   "source": [
    "predicted_test_labels = UDA_pytorch_classifier_predict(simple_model, test_images)\n",
    "print('Test accuracy:', UDA_compute_accuracy(predicted_test_labels, test_dataset.targets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.9783\n"
     ]
    }
   ],
   "source": [
    "predicted_test_labels = UDA_pytorch_classifier_predict(deeper_model, test_images)\n",
    "print('Test accuracy:', UDA_compute_accuracy(predicted_test_labels, test_dataset.targets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.9804\n"
     ]
    }
   ],
   "source": [
    "predicted_test_labels = UDA_pytorch_classifier_predict(simple_convnet, test_images)\n",
    "print('Test accuracy:', UDA_compute_accuracy(predicted_test_labels, test_dataset.targets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.9861\n"
     ]
    }
   ],
   "source": [
    "predicted_test_labels = UDA_pytorch_classifier_predict(deeper_convnet, test_images)\n",
    "print('Test accuracy:', UDA_compute_accuracy(predicted_test_labels, test_dataset.targets))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

## UDA_pytorch_utils.py
"""
Helper code for Carnegie Mellon University's Unstructured Data Analytics course
Author: George H. Chen (georgechen [at symbol] cmu.edu)

I wrote this code for my class to make teaching how to use PyTorch as simple as
using Keras. Note that this code only has been tested using categorical cross
entropy loss.
"""
import matplotlib.pyplot as plt
import numpy as np
import sys
import torch
import torch.nn as nn
from matplotlib.ticker import MaxNLocator
from torchnlp.encoders.text import stack_and_pad_tensors
from torchnlp.samplers import BucketBatchSampler
from torchnlp.utils import collate_tensors


def UDA_pytorch_classifier_fit(model, optimizer, loss,
                               proper_train_dataset, val_dataset,
                               num_epochs, batch_size, device=None,
                               sequence=False):
    """
    Trains a neural net classifier `model` using an `optimizer` such as Adam or
    stochastic gradient descent. We specifically minimize the given `loss`
    using the data given by `proper_train_dataset` using the number of epochs
    given by `num_epochs` and a batch size given by `batch_size`.

    Accuracies on the (proper) training data (`proper_train_dataset`) and
    validation data (`val_dataset`) are computed at the end of each epoch;
    `val_dataset` can be set to None if you don't want to use a validation set.
    The function outputs the training and validation accuracies.

    You can manually set which device (CPU or GPU) to use with the optional
    `device` argument (e.g., setting `device=torch.device('cpu')` or
    `device=torch.device('cuda')`). By default, the code tries to use a GPU if
    it is available.

    Lastly, the boolean argument `sequence` says whether we are looking at time
    series data (set this True for working with recurrent neural nets).
    """
    if device is None:
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model = model.to(device)

    if loss._get_name() != 'CrossEntropyLoss':
        raise Exception('Unsupported loss: ' + loss._get_name())

    if not sequence:
        # PyTorch uses DataLoader to load data in batches
        proper_train_loader = \
            torch.utils.data.DataLoader(dataset=proper_train_dataset,
                                        batch_size=batch_size,
                                        shuffle=True)
        if val_dataset is not None:
            val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
                                                     batch_size=batch_size,
                                                     shuffle=False)
    else:
        proper_train_loader = \
                UDA_get_batches_sequence(proper_train_dataset,
                                         batch_size,
                                         shuffle=True,
                                         device=device)
        if val_dataset is not None:
            val_loader = \
                    UDA_get_batches_sequence(val_dataset,
                                             batch_size,
                                             shuffle=False,
                                             device=device)

    proper_train_size = len(proper_train_dataset)
    val_size = len(val_dataset)

    train_accuracies = np.zeros(num_epochs)
    val_accuracies = np.zeros(num_epochs)

    for epoch_idx in range(num_epochs):
        # go through training data
        num_training_examples_so_far = 0
        for batch_idx, (batch_features, batch_labels) \
                in enumerate(proper_train_loader):
            # make sure the data are stored on the right device
            batch_features = batch_features.to(device)
            batch_labels = batch_labels.to(device)

            # make predictions for current batch and compute loss
            batch_outputs = model(batch_features)
            batch_loss = loss(batch_outputs, batch_labels)

            # update model parameters
            optimizer.zero_grad()  # reset which direction optimizer is going
            batch_loss.backward()  # compute new direction optimizer should go
            optimizer.step()  # move the optimizer

            # draw fancy progress bar
            num_training_examples_so_far += batch_features.shape[0]
            sys.stdout.write('\r')
            sys.stdout.write("Epoch %d [%-50s] %d/%d"
                             % (epoch_idx + 1,
                                '=' * int(num_training_examples_so_far
                                          / proper_train_size * 50),
                                num_training_examples_so_far,
                                proper_train_size))
            sys.stdout.flush()

        # draw fancy progress bar at 100%
        sys.stdout.write('\r')
        sys.stdout.write("Epoch %d [%-50s] %d/%d"
                         % (epoch_idx + 1,
                            '=' * 50,
                            num_training_examples_so_far, proper_train_size))
        sys.stdout.flush()
        sys.stdout.write('\n')
        sys.stdout.flush()

        # compute proper training and validation set raw accuracies
        train_accuracy = \
                UDA_pytorch_classifier_evaluate(model,
                                                proper_train_dataset,
                                                device=device,
                                                batch_size=batch_size,
                                                sequence=sequence)
        print('  Train accuracy: %.4f' % train_accuracy, flush=True)
        train_accuracies[epoch_idx] = train_accuracy

        if val_dataset is not None:
            val_accuracy = \
                    UDA_pytorch_classifier_evaluate(model,
                                                    val_dataset,
                                                    device=device,
                                                    batch_size=batch_size,
                                                    sequence=sequence)
            print('  Validation accuracy: %.4f' % val_accuracy, flush=True)
            val_accuracies[epoch_idx] = val_accuracy

    return train_accuracies, val_accuracies


def UDA_pytorch_model_transform(model, inputs, device=None, batch_size=128,
                                sequence=False):
    """
    Given a neural net `model`, evaluate the model given `inputs`, which should
    *not* be already batched. This helper function automatically batches the
    data, feeds each batch through the neural net, and then unbatches the
    outputs. The outputs are stored as a PyTorch tensor.

    You can manually set which device (CPU or GPU) to use with the optional
    `device` argument (e.g., setting `device=torch.device('cpu')` or
    `device=torch.device('cuda')`). By default, the code tries to use a GPU if
    it is available.

    You can also manually set `batch_size`; this is less critical than in
    training since we are, at this point, just evaluating the model without
    updating its parameters.

    Lastly, the boolean argument `sequence` says whether we are looking at time
    series data (set this True for working with recurrent neural nets).
    """
    if device is None:
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model = model.to(device)

    # batch the inputs
    if not sequence:
        feature_loader = torch.utils.data.DataLoader(dataset=inputs,
                                                     batch_size=batch_size,
                                                     shuffle=False)
    else:
        feature_loader = \
                UDA_get_batches_from_encoded_text(inputs,
                                                  None,
                                                  batch_size,
                                                  shuffle=False,
                                                  device=device)

    outputs = []
    with torch.no_grad():
        idx = 0
        for batch_features in feature_loader:
            batch_features = batch_features.to(device)
            batch_outputs = model(batch_features)
            outputs.append(batch_outputs)

    return torch.cat(outputs, 0)


def UDA_pytorch_classifier_predict(model, inputs, device=None, batch_size=128,
                                   sequence=False):
    """
    Given a neural net classifier `model`, predict labels for the given
    `inputs`, which should *not* be already batched. This helper function
    automatically batches the data, feeds each batch through the neural net,
    and then computes predicted labels by looking at the argmax. The output
    predicted labels are stored as a PyTorch tensor.

    You can manually set which device (CPU or GPU) to use with the optional
    `device` argument (e.g., setting `device=torch.device('cpu')` or
    `device=torch.device('cuda')`). By default, the code tries to use a GPU if
    it is available.

    You can also manually set `batch_size`; this is less critical than in
    training since we are, at this point, just evaluating the model without
    updating its parameters.

    Lastly, the boolean argument `sequence` says whether we are looking at time
    series data (set this True for working with recurrent neural nets).
    """
    outputs = UDA_pytorch_model_transform(model,
                                          inputs,
                                          device=device,
                                          batch_size=batch_size,
                                          sequence=sequence)

    with torch.no_grad():
        return outputs.argmax(axis=1).view(-1)


def UDA_pytorch_classifier_evaluate(model, dataset, device=None,
                                    batch_size=128, sequence=False):
    """
    Evaluate the raw accuracy of a neural net classifier `model` for a
    `dataset`, which should be a list of pairs of the format (input, label).

    You can manually set which device (CPU or GPU) to use with the optional
    `device` argument (e.g., setting `device=torch.device('cpu')` or
    `device=torch.device('cuda')`). By default, the code tries to use a GPU if
    it is available.

    You can also manually set `batch_size`; this is less critical than in
    training since we are, at this point, just evaluating the model without
    updating its parameters.

    Lastly, the boolean argument `sequence` says whether we are looking at time
    series data (set this True for working with recurrent neural nets).
    """
    if device is None:
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    model = model.to(device)

    if not sequence:
        loader = torch.utils.data.DataLoader(dataset=dataset,
                                             batch_size=batch_size,
                                             shuffle=False)
    else:
        loader = UDA_get_batches_sequence(dataset,
                                          batch_size,
                                          shuffle=False,
                                          device=device)

    with torch.no_grad():
        num_correct = 0.
        for batch_features, batch_labels in loader:
            batch_features = batch_features.to(device)
            batch_outputs = model(batch_features)
            batch_predicted_labels = batch_outputs.argmax(axis=1)
            if type(batch_labels) == np.ndarray:
                batch_predicted_labels = \
                        batch_predicted_labels.view(-1).cpu().numpy()
                num_correct += (batch_predicted_labels == batch_labels).sum()
            else:
                num_correct += \
                        (batch_predicted_labels.view(-1)
                         == batch_labels.to(device).view(-1)).sum().item()

    return num_correct / len(dataset)


def UDA_plot_train_val_accuracy_vs_epoch(train_accuracies, val_accuracies):
    """
    Helper function for plotting (proper) training and validation accuracies
    across epochs; `train_accuracies` and `val_accuracies` should be the same
    length, which should equal the number of epochs.
    """
    ax = plt.figure().gca()
    num_epochs = len(train_accuracies)
    plt.plot(np.arange(1, num_epochs + 1), train_accuracies, '-o',
             label='Training')
    plt.plot(np.arange(1, num_epochs + 1), val_accuracies, '-+',
             label='Validation')
    plt.legend()
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    ax.xaxis.set_major_locator(MaxNLocator(integer=True))


def UDA_compute_accuracy(labels1, labels2):
    """
    Computes the raw accuracy of two label sequences `labels1` and `labels2`
    agreeing. This helper function coerces both label sequences to be on the
    CPU, flattened, and stored as 1D NumPy arrays before computing the average
    agreement.
    """
    if type(labels1) == torch.Tensor:
        labels1 = labels1.detach().view(-1).cpu().numpy()
    elif type(labels1) != np.ndarray:
        labels1 = np.array(labels1).flatten()
    else:
        labels1 = labels1.flatten()

    if type(labels2) == torch.Tensor:
        labels2 = labels2.detach().view(-1).cpu().numpy()
    elif type(labels2) != np.ndarray:
        labels2 = np.array(labels2).flatten()
    else:
        labels2 = labels2.flatten()

    return np.mean(labels1 == labels2)


class UDA_LSTMforSequential(nn.Module):
    """
    This helper class allows for an LSTM to be used with nn.Sequential().
    """
    def __init__(self, input_size, hidden_size, return_sequences=False):
        super().__init__()
        self.return_sequences = return_sequences
        self.model = nn.LSTM(input_size=input_size,
                             hidden_size=hidden_size,
                             batch_first=True)  # axis 0 indexes data in batch

    def forward(self, x):
        # x should be of shape (batch size, sequence length, feature dimension)
        outputs, _ = self.model(x)
        if self.return_sequences:
            return outputs
        else:
            return outputs[:, -1, :]  # take last time step's output


def UDA_get_batches_sequence(dataset, batch_size, shuffle=True, device=None):
    """
    Helper function that does the same thing as
    `UDA_get_batches_from_encoded_text()` except that the input dataset is a
    list of pairs of the format (encoded text, label). This function
    basically converts the input format to be what is expected by
    `UDA_get_batches_from_encoded_text()` and then runs that function. See
    the documentation for that function to understand what the arguments are.
    """
    text_encoded = []
    labels = []
    for text, label in dataset:
        text_encoded.append(text)
        labels.append(label)
    return UDA_get_batches_from_encoded_text(text_encoded, labels,
                                             batch_size, shuffle, device)


def UDA_get_batches_from_encoded_text(text_encoded, labels, batch_size,
                                      shuffle=True, device=None):
    """
    Batches sequence data, where sequences within the same batch could have
    unequal lengths, so padding is needed to get their lengths to be the same
    for feeding to the neural net. The input text `text_encoded` should already
    be encoded so that each text sequence consists of word indices to represent
    indices into a vocabulary. The i-th element of `text_encoded` should have a
    label given by the i-th entry in `labels` (which will be converted to a
    PyTorch tensor). The batch size is specified by `batch_size`.

    If `shuffle` is set to True, a bucket sampling strategy is used that reduces
    how much padding is needed in different batches while injecting some
    randomness.

    You can manually set which device (CPU or GPU) to use with the optional
    `device` argument (e.g., setting `device=torch.device('cpu')` or
    `device=torch.device('cuda')`). By default, the code tries to use a GPU if
    it is available.
    """
    if device is None:
        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    if shuffle:
        # use bucket sampling strategy to reduce the amount of padding needed
        sampler = torch.utils.data.sampler.SequentialSampler(text_encoded)
        loader = BucketBatchSampler(
            sampler, batch_size=batch_size, drop_last=False,
            sort_key=lambda i: text_encoded[i].shape[0])
    else:
        indices = list(range(len(text_encoded)))
        loader = torch.utils.data.DataLoader(dataset=indices,
                                             batch_size=batch_size,
                                             shuffle=False)

    if labels is None:
        batches = [collate_tensors([text_encoded[i] for i in batch],
                                   stack_tensors=stack_and_pad_tensors
                                  ).tensor.to(device)
                   for batch in loader]
    else:
        batches = [(collate_tensors([text_encoded[i] for i in batch],
                                    stack_tensors=stack_and_pad_tensors
                                   ).tensor.to(device),
                    torch.tensor([labels[i] for i in batch],
                                 dtype=torch.long).to(device).view(-1))
                   for batch in loader]
    return batches
	"""
	Helper code for Carnegie Mellon University's Unstructured Data Analytics course
	Author: George H. Chen (georgechen [at symbol] cmu.edu)

	I wrote this code for my class to make teaching how to use PyTorch as simple as
	using Keras. Note that this code only has been tested using categorical cross
	entropy loss.
	"""
	import matplotlib.pyplot as plt
	import numpy as np
	import sys
	import torch
	import torch.nn as nn
	from matplotlib.ticker import MaxNLocator
	from torchnlp.encoders.text import stack_and_pad_tensors
	from torchnlp.samplers import BucketBatchSampler
	from torchnlp.utils import collate_tensors


	def UDA_pytorch_classifier_fit(model, optimizer, loss,
	proper_train_dataset, val_dataset,
	num_epochs, batch_size, device=None,
	sequence=False):
	"""
	Trains a neural net classifier `model` using an `optimizer` such as Adam or
	stochastic gradient descent. We specifically minimize the given `loss`
	using the data given by `proper_train_dataset` using the number of epochs
	given by `num_epochs` and a batch size given by `batch_size`.

	Accuracies on the (proper) training data (`proper_train_dataset`) and
	validation data (`val_dataset`) are computed at the end of each epoch;
	`val_dataset` can be set to None if you don't want to use a validation set.
	The function outputs the training and validation accuracies.

	You can manually set which device (CPU or GPU) to use with the optional
	`device` argument (e.g., setting `device=torch.device('cpu')` or
	`device=torch.device('cuda')`). By default, the code tries to use a GPU if
	it is available.

	Lastly, the boolean argument `sequence` says whether we are looking at time
	series data (set this True for working with recurrent neural nets).
	"""
	if device is None:
	device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
	model = model.to(device)

	if loss._get_name() != 'CrossEntropyLoss':
	raise Exception('Unsupported loss: ' + loss._get_name())

	if not sequence:
	# PyTorch uses DataLoader to load data in batches
	proper_train_loader = \
	torch.utils.data.DataLoader(dataset=proper_train_dataset,
	batch_size=batch_size,
	shuffle=True)
	if val_dataset is not None:
	val_loader = torch.utils.data.DataLoader(dataset=val_dataset,
	batch_size=batch_size,
	shuffle=False)
	else:
	proper_train_loader = \
	UDA_get_batches_sequence(proper_train_dataset,
	batch_size,
	shuffle=True,
	device=device)
	if val_dataset is not None:
	val_loader = \
	UDA_get_batches_sequence(val_dataset,
	batch_size,
	shuffle=False,
	device=device)

	proper_train_size = len(proper_train_dataset)
	val_size = len(val_dataset)

	train_accuracies = np.zeros(num_epochs)
	val_accuracies = np.zeros(num_epochs)

	for epoch_idx in range(num_epochs):
	# go through training data
	num_training_examples_so_far = 0
	for batch_idx, (batch_features, batch_labels) \
	in enumerate(proper_train_loader):
	# make sure the data are stored on the right device
	batch_features = batch_features.to(device)
	batch_labels = batch_labels.to(device)

	# make predictions for current batch and compute loss
	batch_outputs = model(batch_features)
	batch_loss = loss(batch_outputs, batch_labels)

	# update model parameters
	optimizer.zero_grad() # reset which direction optimizer is going
	batch_loss.backward() # compute new direction optimizer should go
	optimizer.step() # move the optimizer

	# draw fancy progress bar
	num_training_examples_so_far += batch_features.shape[0]
	sys.stdout.write('\r')
	sys.stdout.write("Epoch %d [%-50s] %d/%d"
	% (epoch_idx + 1,
	'=' * int(num_training_examples_so_far
	/ proper_train_size * 50),
	num_training_examples_so_far,
	proper_train_size))
	sys.stdout.flush()

	# draw fancy progress bar at 100%
	sys.stdout.write('\r')
	sys.stdout.write("Epoch %d [%-50s] %d/%d"
	% (epoch_idx + 1,
	'=' * 50,
	num_training_examples_so_far, proper_train_size))
	sys.stdout.flush()
	sys.stdout.write('\n')
	sys.stdout.flush()

	# compute proper training and validation set raw accuracies
	train_accuracy = \
	UDA_pytorch_classifier_evaluate(model,
	proper_train_dataset,
	device=device,
	batch_size=batch_size,
	sequence=sequence)
	print(' Train accuracy: %.4f' % train_accuracy, flush=True)
	train_accuracies[epoch_idx] = train_accuracy

	if val_dataset is not None:
	val_accuracy = \
	UDA_pytorch_classifier_evaluate(model,
	val_dataset,
	device=device,
	batch_size=batch_size,
	sequence=sequence)
	print(' Validation accuracy: %.4f' % val_accuracy, flush=True)
	val_accuracies[epoch_idx] = val_accuracy

	return train_accuracies, val_accuracies


	def UDA_pytorch_model_transform(model, inputs, device=None, batch_size=128,
	sequence=False):
	"""
	Given a neural net `model`, evaluate the model given `inputs`, which should
	not be already batched. This helper function automatically batches the
	data, feeds each batch through the neural net, and then unbatches the
	outputs. The outputs are stored as a PyTorch tensor.

	You can manually set which device (CPU or GPU) to use with the optional
	`device` argument (e.g., setting `device=torch.device('cpu')` or
	`device=torch.device('cuda')`). By default, the code tries to use a GPU if
	it is available.

	You can also manually set `batch_size`; this is less critical than in
	training since we are, at this point, just evaluating the model without
	updating its parameters.

	Lastly, the boolean argument `sequence` says whether we are looking at time
	series data (set this True for working with recurrent neural nets).
	"""
	if device is None:
	device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
	model = model.to(device)

	# batch the inputs
	if not sequence:
	feature_loader = torch.utils.data.DataLoader(dataset=inputs,
	batch_size=batch_size,
	shuffle=False)
	else:
	feature_loader = \
	UDA_get_batches_from_encoded_text(inputs,
	None,
	batch_size,
	shuffle=False,
	device=device)

	outputs = []
	with torch.no_grad():
	idx = 0
	for batch_features in feature_loader:
	batch_features = batch_features.to(device)
	batch_outputs = model(batch_features)
	outputs.append(batch_outputs)

	return torch.cat(outputs, 0)


	def UDA_pytorch_classifier_predict(model, inputs, device=None, batch_size=128,
	sequence=False):
	"""
	Given a neural net classifier `model`, predict labels for the given
	`inputs`, which should not be already batched. This helper function
	automatically batches the data, feeds each batch through the neural net,
	and then computes predicted labels by looking at the argmax. The output
	predicted labels are stored as a PyTorch tensor.

	You can manually set which device (CPU or GPU) to use with the optional
	`device` argument (e.g., setting `device=torch.device('cpu')` or
	`device=torch.device('cuda')`). By default, the code tries to use a GPU if
	it is available.

	You can also manually set `batch_size`; this is less critical than in
	training since we are, at this point, just evaluating the model without
	updating its parameters.

	Lastly, the boolean argument `sequence` says whether we are looking at time
	series data (set this True for working with recurrent neural nets).
	"""
	outputs = UDA_pytorch_model_transform(model,
	inputs,
	device=device,
	batch_size=batch_size,
	sequence=sequence)

	with torch.no_grad():
	return outputs.argmax(axis=1).view(-1)


	def UDA_pytorch_classifier_evaluate(model, dataset, device=None,
	batch_size=128, sequence=False):
	"""
	Evaluate the raw accuracy of a neural net classifier `model` for a
	`dataset`, which should be a list of pairs of the format (input, label).

	You can manually set which device (CPU or GPU) to use with the optional
	`device` argument (e.g., setting `device=torch.device('cpu')` or
	`device=torch.device('cuda')`). By default, the code tries to use a GPU if
	it is available.

	You can also manually set `batch_size`; this is less critical than in
	training since we are, at this point, just evaluating the model without
	updating its parameters.

	Lastly, the boolean argument `sequence` says whether we are looking at time
	series data (set this True for working with recurrent neural nets).
	"""
	if device is None:
	device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
	model = model.to(device)

	if not sequence:
	loader = torch.utils.data.DataLoader(dataset=dataset,
	batch_size=batch_size,
	shuffle=False)
	else:
	loader = UDA_get_batches_sequence(dataset,
	batch_size,
	shuffle=False,
	device=device)

	with torch.no_grad():
	num_correct = 0.
	for batch_features, batch_labels in loader:
	batch_features = batch_features.to(device)
	batch_outputs = model(batch_features)
	batch_predicted_labels = batch_outputs.argmax(axis=1)
	if type(batch_labels) == np.ndarray:
	batch_predicted_labels = \
	batch_predicted_labels.view(-1).cpu().numpy()
	num_correct += (batch_predicted_labels == batch_labels).sum()
	else:
	num_correct += \
	(batch_predicted_labels.view(-1)
	== batch_labels.to(device).view(-1)).sum().item()

	return num_correct / len(dataset)


	def UDA_plot_train_val_accuracy_vs_epoch(train_accuracies, val_accuracies):
	"""
	Helper function for plotting (proper) training and validation accuracies
	across epochs; `train_accuracies` and `val_accuracies` should be the same
	length, which should equal the number of epochs.
	"""
	ax = plt.figure().gca()
	num_epochs = len(train_accuracies)
	plt.plot(np.arange(1, num_epochs + 1), train_accuracies, '-o',
	label='Training')
	plt.plot(np.arange(1, num_epochs + 1), val_accuracies, '-+',
	label='Validation')
	plt.legend()
	plt.xlabel('Epoch')
	plt.ylabel('Accuracy')
	ax.xaxis.set_major_locator(MaxNLocator(integer=True))


	def UDA_compute_accuracy(labels1, labels2):
	"""
	Computes the raw accuracy of two label sequences `labels1` and `labels2`
	agreeing. This helper function coerces both label sequences to be on the
	CPU, flattened, and stored as 1D NumPy arrays before computing the average
	agreement.
	"""
	if type(labels1) == torch.Tensor:
	labels1 = labels1.detach().view(-1).cpu().numpy()
	elif type(labels1) != np.ndarray:
	labels1 = np.array(labels1).flatten()
	else:
	labels1 = labels1.flatten()

	if type(labels2) == torch.Tensor:
	labels2 = labels2.detach().view(-1).cpu().numpy()
	elif type(labels2) != np.ndarray:
	labels2 = np.array(labels2).flatten()
	else:
	labels2 = labels2.flatten()

	return np.mean(labels1 == labels2)


	class UDA_LSTMforSequential(nn.Module):
	"""
	This helper class allows for an LSTM to be used with nn.Sequential().
	"""
	def __init__(self, input_size, hidden_size, return_sequences=False):
	super().__init__()
	self.return_sequences = return_sequences
	self.model = nn.LSTM(input_size=input_size,
	hidden_size=hidden_size,
	batch_first=True) # axis 0 indexes data in batch

	def forward(self, x):
	# x should be of shape (batch size, sequence length, feature dimension)
	outputs, _ = self.model(x)
	if self.return_sequences:
	return outputs
	else:
	return outputs[:, -1, :] # take last time step's output


	def UDA_get_batches_sequence(dataset, batch_size, shuffle=True, device=None):
	"""
	Helper function that does the same thing as
	`UDA_get_batches_from_encoded_text()` except that the input dataset is a
	list of pairs of the format (encoded text, label). This function
	basically converts the input format to be what is expected by
	`UDA_get_batches_from_encoded_text()` and then runs that function. See
	the documentation for that function to understand what the arguments are.
	"""
	text_encoded = []
	labels = []
	for text, label in dataset:
	text_encoded.append(text)
	labels.append(label)
	return UDA_get_batches_from_encoded_text(text_encoded, labels,
	batch_size, shuffle, device)


	def UDA_get_batches_from_encoded_text(text_encoded, labels, batch_size,
	shuffle=True, device=None):
	"""
	Batches sequence data, where sequences within the same batch could have
	unequal lengths, so padding is needed to get their lengths to be the same
	for feeding to the neural net. The input text `text_encoded` should already
	be encoded so that each text sequence consists of word indices to represent
	indices into a vocabulary. The i-th element of `text_encoded` should have a
	label given by the i-th entry in `labels` (which will be converted to a
	PyTorch tensor). The batch size is specified by `batch_size`.

	If `shuffle` is set to True, a bucket sampling strategy is used that reduces
	how much padding is needed in different batches while injecting some
	randomness.

	You can manually set which device (CPU or GPU) to use with the optional
	`device` argument (e.g., setting `device=torch.device('cpu')` or
	`device=torch.device('cuda')`). By default, the code tries to use a GPU if
	it is available.
	"""
	if device is None:
	device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

	if shuffle:
	# use bucket sampling strategy to reduce the amount of padding needed
	sampler = torch.utils.data.sampler.SequentialSampler(text_encoded)
	loader = BucketBatchSampler(
	sampler, batch_size=batch_size, drop_last=False,
	sort_key=lambda i: text_encoded[i].shape[0])
	else:
	indices = list(range(len(text_encoded)))
	loader = torch.utils.data.DataLoader(dataset=indices,
	batch_size=batch_size,
	shuffle=False)

	if labels is None:
	batches = [collate_tensors([text_encoded[i] for i in batch],
	stack_tensors=stack_and_pad_tensors
	).tensor.to(device)
	for batch in loader]
	else:
	batches = [(collate_tensors([text_encoded[i] for i in batch],
	stack_tensors=stack_and_pad_tensors
	).tensor.to(device),
	torch.tensor([labels[i] for i in batch],
	dtype=torch.long).to(device).view(-1))
	for batch in loader]
	return batches