Infinite MNIST (PyTorch 0.4).

Infinite MNIST (quantized).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "from IPython.display import clear_output\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn.utils import clip_grad_norm_\n",
    "from utils.show_array import * # https://github.com/kylemcdonald/python-utils\n",
    "import random\n",
    "import time\n",
    "\n",
    "cpu = torch.device('cpu')\n",
    "gpu = torch.device('cuda' if torch.cuda.is_available() else cpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>img{image-rendering: pixelated}</style>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "IPython.core.display.display_html(IPython.core.display.HTML(\"<style>img{image-rendering: pixelated}</style>\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "seq_length = 128\n",
    "batch_size = 32\n",
    "hidden_size = 1024\n",
    "n_layers = 2\n",
    "quantization = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(54210, 28, 28)\n",
      "(854232, 28)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAAIECAAAAAAvfBGXAAAG3klEQVR4nLVa0ZbrIAiEnP7/L3MfEuMMDMZ0e33YthJREYbBrNmiuZmZBf+kX1GeTkL+craDBriTmsO4uVlEJ6R2jMdxEYHCsv5AoWvpPXIuMu4/x1xkXZDD81P59aN0GNgJlgJ6vWzFyzfaRJSeRbt3rAZMI4h9woKqGLZWdZMRpIPh6OJgnTjNwrrL9tA7695hsDLMPVi55q7pT03CsluGn94ohP5wnubX0C4Ew8yihqBbmGdfTUYI+oCwX/mQo6tEEppwJNxK+ZZOhY34yYtAKY309iTFsSxBajVwc6QyX69zqI1m3COkJvBuRkZz2PQMnqeGuJ0FvRUCjANO06SI8cVKMztECZV7G3lUCQ18qAWMojaN3QCUAowPqKmxL25TXb0lVvqwT+17IbVs4BdqYyX8ekGv1CJZ8tO6LIzyNAvpmTaXUSJcpqvH1oLMsZJu2LYZKgFjgSZTzZE7SiZA/oUmOqhPYcI0O3G4xBjD3GVGEnAyGWNaKSvEtSfAgCF+JyZOV+JbbZH4rWovwuFpHmxin0VYiS8JIaK6bM9qSG0uSxRLzdTuYu1TAVYAo5Ol2nzO3ucOcDtmZoLmvDSqAMjvXWBfh9RchbxsfKZeZBwryQPoaOlpznJHA3tDLeTkJD0skbZqBDeLhWvCuiZcF2ac5zxVFECXddk8mhq8bUGcDiw5WFtAzYkZeEK7wI3xCqyPu7+peWNEwrUVis+7lIpkfobxawJKAO6AJbDsyU0EnqQS85papsjzxEOeSk7SsorUI9PTwt4V4/H5r2O0mTOfOfO+1PPS+1q/TVLGoWCpvKuhME8HcR/ZZ8Fb7AM7DyPHLKSM7SBpVid8JFnVhdUA4iayuuiQcdIsz8YqYS9YTeCz4jyLlVCI0/B5erNcmSLHk7nqEEVbSo1YYtJMKAXDMzw4L2iRHVQCVDcR+ccXrQMMc6vYR+3TSobaLo1/dHcRvtwb+G0duXGb1aBhKU7RsRWkCjeZRCrTyWwjpM3yLkveGRehS6mIMkVxREzLuFQgFUg9zIr5Cs3abC0fUu3vVYc0fGLuemCfIpY3aF1LqDkuic4Ptm2iTUVtgPvn4rSCMdHfs/n9RxgIQSqbR/uQW7jFyyDvg1dLUbj2oVDCGwujCmN2kmp0kxJcbD5nd2oqlbRahVJv7qTYDqU8qFthxXkredpy2HV09zpjdXz59BbwhidThKGWJt96CWFtFIKZ/OJdTSl4AIybQukcGU3SomeWq+2Z1DNZ2pr1x2AcvXCu9z/dLI0JalacbchChX3lfHJBXjBl1Kbpk6dMT0kSMWCnIRFnx4E/IqHwEoe+b7kqoFDcYDVoM1071PVO1BSaiVOn9JkwIfpYKdIMqaFWK1e1vHEpbcU1qp6OtwDwttinwRlp88K2Yuz02waMJ/so3qv0Xd18cZ5md9F391eSD3v65prClbA82q+zudztdK1dYddPVg8eq0mOpuAws+k8cHQM48l/cghCES7rfORSBW+5Nql9Y2V4y0NLD1XxkMYO43+Em+zxLoQdYxNRW/MnsMm0z7CkP9e8geMJQtAu4NUQnxVxUublJaTKNdpTgf3WQPpZJhM+hN0dl+2hprTtkC0D+/zJvpyEDLby5WgpFp8xPszSGwsiLmaqnpBvvUgIkEKEQNS8aWQn5dQxT6Q5MskTKB66OUF/vUb0+VPM+VM/ISXqPHP7RrhiqTgyVMW2M2cUAAIhIlspvTS9lnOuq6vZci1omPeuBc30pejkLBLBExLGnz8p6aikMNU2qMfe10NjPZjGD1vi8pA/9RJSgmizlNkDpG5eQDZEtG1vrgwEgm2oXQtLjkn5kztljAzBklOz96Wtfpr+/faQrnqhnvNHCNYx3askWZ2nV6g5u4XqnDpMx8qIEiiXqt9qj987l9W/QqnVbgsXeUVm+3dzLlsuDzqueRs985BIKjp4A3ozR+b3vc5Hxiw1EppkDot3Nc07Ibs4cRJCrPTl+0rpVP3QXl8Gvv3f5k2h2MrPWc2GkF6O5nccpWi7O40jtoPdU5ivR4RrroRdkzy+a3+Nz9B7Awstk+tlX/ClESsjKCejwpHnRUKADnHfh9cNF82aQIDLGni7oLCEBgLjoZi5saPsPKBT3uQ3lwRzbQuWt1FIOIAirgXnPz88HSwcdtyP1CXcR9JYaOgpRsASUxohbp8SMO7XjNq2iWjJU9lKtGi5sv/EwbwKozytVpsa8QQ3jpUUgpwFl/fxcwARS7TtuuqYei9VWL57lvLXtCBMkSb/LzN7AnWKqqM9yxfFttQxTP1ZKSvEvqjbvve31qNuWVu0zcFvE4CvhLN9xhQtjw/DvaobF9H+AeP1YK9e50PbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 516,
       "width": 28
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = np.load('mnist-ordered.npy')\n",
    "print(data.shape)\n",
    "data = np.vstack([np.fliplr(x.T) for x in data])\n",
    "data = data[~np.all(data == 0, axis=1)] # remove empty rows\n",
    "data = data.astype(np.float32) / 256. # must be 256 for quantization to work\n",
    "data *= quantization\n",
    "data = np.floor(data)\n",
    "levels_per_quantization = 255 / (quantization - 1)\n",
    "print(data.shape)\n",
    "show_array(levels_per_quantization * data[:4*(seq_length+1),:28])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def time_since(since):\n",
    "    s = time.time() - since\n",
    "    m = np.floor(s / 60)\n",
    "    s -= m * 60\n",
    "    return '%dm %ds' % (m, s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([206, 129, 32, 28])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAACBCAAAAAD2noXVAAAB0klEQVR4nJ1XQZbFIAgD39z/ysyirSYErP0sZvqNUEQI1GwjbmZmwT/pV8juBPLDJYMU3MnMMBY3i+hAkvFsRycCQfE/EPQanZrLyZh/xnJSHXLYv4zfP2TBIE7gCth1OYrLEx0iZGUj88SVwgpCcU5wSGE4mtqmIJQJhtqSYB2c3sK25XiYnXp2UK4CM5Wr1DwN/WWpiOxR4Fc2FqC/3Kf5rdqVYJhZaAm6hXnO1RSEoH9Q9rscckyVSKAViYRHkad0KxzEv+wEoqTp7U0W17IlqZ3ioWYVvt7mYzYavVdKTeTdaEZz2bQH77OmuBOHvoJA48DT9FLkeInS6g4hpTKPkbWkNHBTSxhiNukeEIoQ4wtr1twXM1T3qtRKX/ZJfgdJcoA/mI0d+LNDn8zisORXdBkM2c0g7Wl7GTXCbbt6lZZkxg49iG2jWhLGhk2WmZEXpBPg/IUhGrRWccIKO81waWIMcy87UkEna2JMnrJB9D0RBqj4bEzcroonlUjzbSUfyuHtPSjFOQXUwZdAqKiu27MZMps/S6opNY9299S+DOAXwLPIaB0+5+xzB7p93swDmrNr9AVAee8F93VMzV8hH4Xv1AXjWkkZQFdLu7nL/QOXqZGhe8PmywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 129,
       "width": 28
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAACBCAAAAAD2noXVAAAB8klEQVR4nJ1X3brDIAgLfn3/V+ZcrNUQwnY2L7bWCPIT0AaQAAL3KC/r9Zx5v2/gBhE8EwICAeQWNgvPZrknF69U2Q1GdLRYlwAQuWdXW0nSBax6FBRU1r7URgHTrbrVJrqPD5iPtI7YSnO7wim7XwJICf/aJp6/vWABQMSZYLMv9g/iUQj42jqaEjwZT5sVSE5b1JJml4ITE5tskxQa/ziUfUOETAC/ZN/IW0FXAbKiPQiEanm88nIbdnnGPuDxPFGIWfJ+Nqi1IugESpvqo1PYCSxerU6RF92GdXRqsFrZZ0928lqTzxYlBnmbms8YzDWMPyuvYZ4kj97gJUt3YpQCX9tDVIPenA6CSfGO9fXLmBoGAr33lXGNyKPWeMGS3oBrhj4M4m2XPLw1Fm/2uR2VJmBiu5ZqaBK1wxCoMUoCwwd37zmdn9DqUsnSlPsVx9S0rUvXpJKvHkALX42wrU87xvuQG7+D03YAuMpnZppL3yfss7VHLsvVpcZWrk1NbRL9ywEgT7ulqsGxf0yAuElpeDyHAhnIL4t8Ll6PMvieQ+nA3Quzg3kmi2qmSSuuGr6odCpn9ngfcl3qH64c1Bx0ZEB3pSpWV3Tbluwu7RhvPmuqsIBv2htnpoHpTCODpqNjGKUE9fK7CGkfPNSM+4fSH6eZorGBdFb2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 129,
       "width": 28
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAACBCAAAAAD2noXVAAABzElEQVR4nK1XwZYDIQgD3/7/L2cPHccQotPtlkt1goASwUZMARARgTgICB0G3oNJY7PyKMvrceXfQezBFe/nPo8yHUDmLBMDn/uliSxqc9YPPjWnmEbrb3UpWkPQGQkWmB4dPAFSVNibO4KPJKfNnNbJ9H1CjunXVujMDOPTxXubbefN4ELtXUmOWsGOlmRnBFy0NiqhycPJwx2Vk4yodFQQO3+j6XTQcDlkK4ou3pqIxnToCt/NIfd50Lh5T/Pt/v7ToqA9tZRptGZZOrCp7uP0K7eEOFPhXZ6cFMfJybDX65JJHkpdLePCH72CCy2KKfrJBWU0LTJcThGytDDh5U+4ohZ3Nf5LdbMyPg3Yu9Aa2+sTUS/vJbJPhNjPuvLafdLEJZlYTfezVxzpvDWEUZWwzQrtt1+kr3UywyH+7MvlqdQ0efvKtoX7/lm5LGAttrcVPj7o6LnGIy6GyAOmNlcZ9OaqZqmklAdBq5qH5lrR2jpWRjYps++Ech92Psn+4Mm91tUhVXRx/EPg8qnyCXh6pfJKwKOPPtEKEIFc2aCgf15bn+d/V0tKytZzciW7tC/3nMS9kJggNf41LU3HNYVldlP1Kvv2pbEn5hdqCpifOoHMsgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 129,
       "width": 28
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAABwAAACBCAAAAAD2noXVAAABzklEQVR4nLVXSY7EMAiEVv7/5ZpDbNbCHbUyHEYjyiwGirhVukBERUTkQ7ElShFlILJSKxB1VwsZfF0jIjHbmprQq1CwmxZLnMDfYopWvzsLqlSWyAaUY+p/kXUm16B/Lugp2T0ZFIrAYr5TW56V6XHqp+a0P6Ymrt0S+wzjymYJXNnnlk/8s75YTi+Vzx3PoGX2e8yjeMGbJnHFiq4FRHExrTf1trklYrQ7SmxZmAEVQdkmGTVwDcAwmgL0uQ1c6WZDQoPrL+JHHvaT+Hxj+shV3h7qJ+Ci4J0LCgvzSo1KyYyd1u4Norgho3kCJ6mJ/Ss/we8WKnT8uK76hlnaXNmkhIQbOR1UBNu6u705uU45qGERxLT2vqUs+dRbIJxLG2ZH3buj3RxB2QofPx6kpFMxDf9O8TvB8JLQAvo61dLY0GzYkZ6CtWSo0PbTihB2MC8CbKbIGtcVkde2PLRoVx59aGPl2v3LG0w7iHaaZVskvRNUMlcKBfNX0CzZYLhBeljG2g4FKU8ec4UAakXzvyWh+Ilck1BM6iQkJfnVMfbyAUdK6GavImthTM7aw765e8LVEHJMyPs3zBBOoEygnkCXa4cY3/GQeNe247n8AWnKlKEI6jf4AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 129,
       "width": 28
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# drop last chunk of data\n",
    "batch_length = batch_size * (seq_length + 1)\n",
    "data_length = (len(data) // batch_length) * batch_length\n",
    "data = data[:data_length]\n",
    "\n",
    "def build_batches(data, seq_length):\n",
    "    cols = data.shape[1]\n",
    "    \n",
    "    # reshape into:\n",
    "    # batch_size x batch_count x (seq_length + 1) x columns\n",
    "    batches = data.reshape(batch_size, -1, seq_length + 1, cols)\n",
    "    \n",
    "    # transpose into:\n",
    "    # batch_count x (seq_length + 1) x batch_size x columns\n",
    "    batches = batches.transpose((1,2,0,3))\n",
    "    \n",
    "    return batches\n",
    "\n",
    "batches = build_batches(data, seq_length)\n",
    "batches = torch.LongTensor(batches)\n",
    "print(batches.shape) # correct shape is (518,33,64,28)\n",
    "\n",
    "# these pieces from different batches in the same position should fit together\n",
    "for batch_number in range(4):\n",
    "    batch = batches[batch_number].numpy().transpose((1,0,2))\n",
    "    show_array(batch[0] * levels_per_quantization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, n_layers, embedding_classes, embedding_dimensions=None):\n",
    "        super(RNN, self).__init__()\n",
    "        self.input_size = input_size\n",
    "        self.hidden_size = hidden_size\n",
    "        self.n_layers = n_layers\n",
    "        self.embedding_classes = embedding_classes\n",
    "        if embedding_dimensions is None:\n",
    "            embedding_dimensions = embedding_classes\n",
    "            print('using', embedding_dimensions, 'embedding dimensions for', embedding_classes, 'classes')\n",
    "        self.embedding_dimensions = embedding_dimensions\n",
    "\n",
    "        self.embedding = nn.Embedding(embedding_classes, embedding_dimensions)\n",
    "        self.rnn = nn.GRU(input_size * embedding_dimensions, hidden_size, n_layers, dropout=0.2)\n",
    "        self.decoder = nn.Linear(hidden_size, input_size * embedding_classes)\n",
    "        \n",
    "    def forward(self, x, hidden):\n",
    "        # input to forward(): seq_len, batch, input_size\n",
    "        seq_length, batch_size, input_size = x.shape\n",
    "        x = x.view(seq_length * batch_size, input_size)\n",
    "        \n",
    "        # nn.Embedding\n",
    "        # input: batch, inputs (indices)\n",
    "        # output: batch, inputs, embedding_dim\n",
    "        x = self.embedding(x)\n",
    "        x = x.view(seq_length, batch_size, input_size * self.embedding_dimensions)\n",
    "        \n",
    "        # nn.GRU\n",
    "        # input: seq_len, batch, input_size\n",
    "        # output: seq_len, batch, hidden_size * num_directions\n",
    "        x, hidden = self.rnn(x, hidden)\n",
    "        x = x.view(-1, self.hidden_size)\n",
    "        \n",
    "        # nn.Linear\n",
    "        # input: batch, input_features\n",
    "        # output: batch, output_features\n",
    "        x = self.decoder(x)\n",
    "        x = x.view(-1, self.embedding_classes)\n",
    "        \n",
    "        return x, hidden\n",
    "    \n",
    "    def create_hidden(self, batch_size):\n",
    "        # should this also be stored in the class rather than being passed around?\n",
    "        hidden = torch.Tensor(self.n_layers, batch_size, self.hidden_size)\n",
    "        # trying small random instead of zeros?\n",
    "        hidden.uniform_(-0.08, +0.08)\n",
    "        return hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using 2 embedding dimensions for 2 classes\n"
     ]
    }
   ],
   "source": [
    "cols = data.shape[1] # seq_length + 1\n",
    "model = RNN(cols, hidden_size, n_layers, quantization)\n",
    "model = model.to(gpu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhIAAAAcCAAAAAARwwk6AAAMFUlEQVR4nMVb0ZLcOA4DUvn/X8Y9EABpT89kt+q20tlMptuyRJEgCMq9FP7yixAoAhAogIB/JyCAEKH5BxDnzw7gXNuPAHH+ZiaIAnju9cIE+kHWHYOQeWKCQFGAl+V8Ess9Arb/9auNkaf2LLk/bxmzcHbsbcXYXo6Nyq27Qkd4BW+rXsps4+79ERf+/i+i/O9eY1rMdVTt7ToKmpEU5uME1nua3+KtRQcSaySQhOMlT8o62osVV9RZiEaLZgABBiuei1kECxpQCxrHMUDTTFFgTERkn8BQVWK17jKomk00pkCJFMBZZn5qrDf0te6hXbdrAxD+FiQ2ooi/GMT7HetWcQINANQggxpnU2xeJQ81sWA/iPtKHkx+xQIEUOWbLgdA2PWeNiT5Zqren3A3vjAAe0l5x5kvtoUcjTs512eWWTggXnixWLbt69xlCtwkE7o8u8mZ7tf/I77/+kXeN4BECaTBK9gVDjilCYeDPY6dtBQExd9cDPizUoSWVCdTZqIsgUlnTei9yASOAMmGJlY3wI4fTTvBqTRrAjdx/ZtXKEwAKOQkBDQho4lbbhta0ELQpjfZbbEtz9LFXtf2yjOTQBISdCDBR6D+yxfjiyyc3BAIyaw/LK2BwzgNqTEAydaUWp6UCPvqbt1Ot6qoU0WJLMmA8MzFFD1fE99pmfTj0kANfeYxWvuxaw7VzA8Y77vMAHJgap/JNNocny2Rl17qiogtZmCqRgLAZOMlT4IL/AXtCd3P4rO7/XDpy0yH+5+AOCqKVZiplqnjzaBTSVdlrQITHnO08kQpBI1blUL1aLFXzC90I3HOAlf5vW5LQKsa+6O2I2qukxe9tVSRqEXT0cbHvJqghozYxdeokmZ1NqpBq6ixWuIDQ/DrG326+Ar/50m+Z6AjkQPMLX8dBJOgRvBF1Hkz1NTjIGD/5HZXGgt8E0A4eSDCi9aoz0NpIjFl5bY0k+JzsRr5OqfmIhrJ1o75nmzTYrFTiqg2CTxkDmUl6eLsSvTIhlWfJhKFBBV+i/Rtx+FQfBfdLwF9f/CFFV6jHrTw5BVDfCmnOVKInBW4ZuoAfy7FA3H4CoxxhuvpU7AqBdiO9O1tGCvpG5fuhzrdqdhVIk9Qfxe+i8xlkLnRZrYKtR6sEG6pg2GitBWpU2E9rEaOith0SxvVzmstq1QtMb14YJkeQeYr6B+ubHQ+QuIrIlSzvCW0JIjrg5IitqBu8ViqRYrRVvtsLntkh4dYfC052xu2VKTAxcDCuPUH5fHc0rcNx1J6zW4KBODh9OYD2zAd0r/zNLItRTGzbmEM1ZqT5VJLZ/6Vl82qb0L34g/5hU+vhMbMdUd94Ij4jHGIglhsPpAOnRSoj7AKK/jO6CulMkzKKYqyXSoxjc5UI4SqCEu6K+xWgmhDM7iQ13Z9SIlLAQnO4wS5+tiU9gdKc+O6pBbIbKgyYKjBenyTALTB0/pETwKXIjFOGcNpLe/bHc0pHGz0nqG6v/6kF75/6fz9+Ao3zrFEVaN3b27EsrrjqSrpuv4pJlpylTxKFvFRww8DmK2TkEVTxgaqOJREN63FgnfNYKgMmFmD/mAgRT2KgIkzKyHQ/Rpi3YBLi7cXvrsFt5Bk1hyfRmwrJmYg9QubAm8oqP9+xwave78fct8Y1ltts7WtAVONDewcMfg84foF2Z0SS8yEgCYPRrDtWiNFIjOBoRpLAbs/swBWHZZmYx5DIO4JobochC4bsIo40J7teB85GE3ewtogJGZzzVVetmzqNEpzWVUaF+ZIJQo1xzERGnGm8d9I41fY6msc9zN9iO2J6xaKDyD4Bi+8o8a6CAVvF7wSe49caPXG8RsdTfMvDIDTRrAzG1CTxduOh6kiSUuoiVO90UMf5ihk6CNAgY9LglnZBsdKo+wQFgtKmKGgkUw0ZMejSeaInXuQod6hlkCqB2BzgluHAWxS0oxkP/P3D2H7+ZWIfr35KRfu44jnvTwuCfeHY9Fuflk6omvVZVK6KpTxbXSU9TwqDJGKlLBRrda6VWQ52fccRjwUj7MVtZy4vGVDWzFy1ancKIcJMglSSiuaWw7XGdUVIQ7sjML2uQFXRUgiM3BXGysJ1O8Puf8lqodYXh98uO/LK0z/HBWV3uNkntCbpTc63IBFCqLiDCnbyde4NXpx1cE5sXCeqvFvQ3D6+nQCZKgwV9pELiZs29bvVQWt2N7/Ym3Gr7YIuVtaVduoE8RCRhdsYHqK4HW7YuWCHmedbcW9EQrib7yC/ymy38T/8xB9Ga9S8Fq5HVZ+NzxUcBTaq7bq/kQtaZUk4XoFRRFazzfTakkfYyQTe3xuobCgxU2Vo/y1OZ48R9giG0M6nfkk5N5NpbN8PPqIM7aaqi3Gld0ht+1xNFtmp6k3Untg7eWnNxaey0InsEH4wclJjw8VILE+Vz9WCuVKyV3x7V2hGbtOy4yh2x3q+4sZxyLU3OMBzx8eSjkt9FLDNuOWsx51KtPXcvTmTFzbwxynUDzeBEM2Ke1L5yxHXaPOCKzvS3m9XDbcUnWpacmklNkVHk9C3+ViPgzZnTd9dfg9o/ieVLb+EC9tlDroAUwK+7GMyDkzCMR9HrAaNA/0stBMampMg5sGwelhJzw8JZKPVHQ/0lkrzHQ2KveiOZx0+iVrXX2Kfqu+c4YSuUvMI8nOM751wbLL2aAwG1S6ITy+LaGzkURwnv6aIU/5cy26kLgpqPuhIzZ9zT95XvoC1vsWvhUnA1Ef4KSkTsNQxYA8z96SrxVN0we8ZeA5hULZ0jNqwTddZcSLAfioFkYkZRETtmjvvL7fI6TSwcH9breFaYxJiWghysPRGDNiOY9hW1zslCCHgYhLcRi28xxxV/CYMEncr9BcjrgBuyy/zf9O+afXzaX13B1Rk3viNolp96mSESkKQKr5ONzypDUGHWFHno/CsTYuTj1pn9ya0mx+KP0uQQ+UmOA3bNhnY6b1rVWsE/Pr7YDQYLuQqQtzj+duIPKG1/AWjlYqrQmGCA0kyAQVxshRVQjoHWY+LsGUwb36x9fhRuArY2QE6dNBgdxvSskUPwWXjAajDyoOtPZLKqnQW4sj7V4qnc5fOzdFeZXR1vE8FIr/h8V6dNRv8ZTVsgxymIRDDJaD6JkRnNAt0E7zVu2JWnlpuHQKR9DtCeTx4feKb+PhsmS+QhIlShE90Aaue1+v55Ua/vNNG+4DKD7e+uKm7PYOiCKC/dkczTxKDo67GEY47SxyenC5hftv+We9r8WSGx4VKSGZkQWXvrlfbMocJnSX6/DRIUMih5Fn6qv3LGIOr2Rb60wLgT02Ic4EOypbCAXvvp7a+XjopR5uUL8E/eb5J0S8+eZn1KzcbicGNk41tnvMFtLlRVlrHYowcm0IN1dYF0Csz8r0unPG57G1++qd25FcL98auy3LumQDfbMmjQLij35aXxkCuJppxVgBUdGY2nXizweQWuP867nxS0R/iCJ+CvYjGH9GBNaRyfg6HEg410n37TFHC5ybjE6/aNFGLUV8bc3MJ4D4MtV2bP3sLl6s3la1WmDrCNHtRdfc0L1k0QNHxq0NTp1AbmlSxMaTYvmJF7I24Vxcv4noHwP501Wdffxppoy9qG7+PnGxUjGkeVKoTq87WpMCh8isUu6JSxgCIaAF6t3KcsiCI5qjuMHqiP4IkLK9NXCt1EmM5z6w2KpEarqvSbUeG94jXu86OFx83Dqnl6/XH/HwxzH6p9OMLkmYtPvSeS5Ukqzru+3U5C1lW1KUyoLHM4b1iDYfNhkTSD9Lhk4qMToDc549d+a4s9dPnd7wrCL3Bo+ASoU3M9xEnjZgxqhbjSCgGMXFi3bqrn5LGuq+EVFbCksv+jtf2t9Xg7HBBpyt8n8heTuuozeV8qUDFvrwvOljAMy513G7nbcPSpXZZlqP9f+ck1Ty3HmYRtXQedgp29R2qDdZAs9ZGGsK0IMWPuZgU8ODYi1LL3NEpvhp8KuzItpPu8x07hZP9CHt9Ht//f/2upIxyQuYzy6bt/vYSpGDTbeAiTwaFafY3nHkAPCuf7S7g8dU3yMENlHdF5uVloeiHFBrLwHsHP5SiKh+3eaQuQF56k6/EqmXhm2B7R5tD05TW+V+tO9MfXoYV6H/AUo6c+lB9xQKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 1060
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 1.4ms per call\n",
    "def sample(logits, temperature=1.0):\n",
    "    if temperature == 0:\n",
    "        return torch.argmax(logits, dim=-1)\n",
    "    noise = torch.FloatTensor(logits.shape).to(logits.device)\n",
    "    noise.uniform_(1e-5, 1-1e-5)\n",
    "    return torch.argmax(logits / temperature - torch.log(-torch.log(noise)), dim=-1)\n",
    "\n",
    "# # 0.4ms per call\n",
    "# # but dangerous: often causes inf\n",
    "# def sample(logits, temperature):\n",
    "#     probs = torch.exp(logits / temperature)\n",
    "#     return torch.multinomial(probs, 1)\n",
    "\n",
    "def evaluate(prime_rows, predict_len=100, temperature=1.0):\n",
    "    with torch.no_grad():\n",
    "        hidden = model.create_hidden(1).to(gpu)\n",
    "        prime_rows = prime_rows.to(gpu)\n",
    "\n",
    "        for prime_row in prime_rows[:-2]:\n",
    "            input_tensor = prime_row.view(1, 1, -1)\n",
    "            _, hidden = model(input_tensor, hidden)\n",
    "\n",
    "        input_tensor = prime_rows[-1]\n",
    "        predicted = []\n",
    "        \n",
    "        for p in range(predict_len):\n",
    "            out, hidden = model(input_tensor.view(1, 1, -1), hidden)\n",
    "#             input_tensor = sample_temp(out, temperature)\n",
    "            input_tensor = sample(out, temperature)\n",
    "            predicted.append(input_tensor)\n",
    "            \n",
    "        return np.asarray(torch.stack(predicted).squeeze().to(cpu))\n",
    "\n",
    "# seed = batches[0].transpose(0,1)[0]\n",
    "# %timeit evaluate(seed, predict_len=1).shape\n",
    "# %timeit evaluate(seed, predict_len=100).shape\n",
    "\n",
    "def show_prediction(length=256, temperature=1.0):\n",
    "    seed = batches[0].transpose(0,1)[0]\n",
    "    out = evaluate(seed, length, temperature)\n",
    "    combined = np.flipud(np.vstack([seed, np.ones(cols), out]).T) * levels_per_quantization\n",
    "    combined = 255 - combined\n",
    "    show_array(combined, zoom=2)\n",
    "\n",
    "show_prediction(length=400, temperature=0.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAACsElEQVR4nO1a2Y7DIAz0VPn/X/Y+EMAnoVVVQNqRumnAYMA3XTAtBpavYDFeqxfwj1UiABYx3g/XEq7/5y8gRACiH7ll0JdYxUv+3Ua+gi4CkAuNeNjKrcwRkZ+pH8zXjihY8oFoIgh8A/wLR53DU4B5zqLSD+ZGzP00L1dFACIiRnaablu2wWm9oWLVFxIbmpAf+6aE9TkoGVHPT6Ryd89RBCS3akT1xr5TCYhJQETM3PggsCUE35LV7Y0eCxjlU2GPymyLO5lHDSKMOwSo82JNGoMFaQvf6Cov2gzr03AR1e2k9lxs4KPpWXxC9Lmt9Hva1KXGTQa9l+HCwVFGQC8yOaI4CW5P5mxXE/mljerSt6TSRSQBbVAh2/OMgF5li247yl/0ZNJStbCbOpc8vusoHGSq7L5kHHQ4OcsIiiP6aM151qhPk+HLi5pOxjqr8189modGcKIE6MriaajXYVkQ0Cl4Z12cTAn/95lCnrlipTNliL9T3PfHRU927l8jqArA0DOaKqN2c6qxtgZM+Ke6PopaW0Jf0926aGIsWiOse27WMXQBLLNJamlqWK6BdG5s3wYY0W0rACOC8PJGFbrWrbfjicpWj+7PkIyo0pfO57mOEFltlNrtDSkCiL0Yne2PUEwWD7VzlMm6/GsaSkfOk0B2WW3uIloXp+qbQ1YacT6j3BD7/pnpgxByCC4qRi98uS2ltJsvtpBWCh7P2qk9/+y0YrGR5kxHkOVoFxQ0UiGXUyLpcnAZZjymFeIAjyXANV7rJfh5jzEIcU0nWqVjHv8mEndC2VM8cyfWNvJQXjn5fB5ENgF8PvkccMfXNDK1zYnUXJy9hgPE5GyaT8T1ifLwWEzcb+DmZs8S0Lkh7w3cD6t/en3/VudcdU+w+l+5mE5W4K9gtRXssILF+ANL0+twLHa2FwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAAC3ElEQVR4nO1a25LbIAyVMvn/Xz59wMa6HBmcbgvbrmZ2nIAQQkc3vFHIYtLlGiym12oFfmgVBKqLNt6P3kt2/bG/IQOBivyltKzy21upcBnDQxD0FxejCwKVVBp17jiMKUu6jPMFWCvbRCbCizEsbgg6BEQ3zV/AJm+PoOE5TR5f7/Vd6FPjnQt36sJOCNoRtTpTMmAcSF4fuODmKLMb7SsVRhzCZLHtkHZC4ICAnkm9CwrsCeFP+8DHSwTC3seGDQFYTjXKaMJtIiS3QsDUAqg3bDRV0BsXW6bTZtCjBBiMg6RivRAETEggOcEMbYlAg6AdqozpoXPVBPNHqct20EAPN3cIhLTWMMihM6PTTvSS0CPaZNOfQKX3RH8Zq/qRVLTP5fV6zNNeqw9/Vot3Q0Be7UysuzNO2cciV8/OZXKp67urwqxRhTd13h5ueBQEmyLQEtFHatX9nTcnNF8vzlJapHNnTNvVIicsvqjUdz8E5F3WQ+bX9FpA+BxBWe/Yiq2eGT3JxrXc9qUlDeb3rMQi0qKA+vHNV0auVAZ+qGkr4ZrKItkzBwBTI8TRfdXfE4Hwmq53geEudQ5e3uzz841Bz+lrqeaEYr+kfh9WnRCemj5kovfDjchBwGqiv+jGtN69kNqlkoUsyLEEh7+3nKnJozxZKLbNa7oLgdikwj4oTJECRzx67DfVefkDk6h7DitVJWWT13Q+BqxSNmsguO/MWe1No3++jQEeI93erldipfrpvfkZ7l9Nb2lnMrk8XqVCgyL0vlpT8FCyxGCPXq8LgvppgwDvVUtBXsGFgdBfUNyqkXrK6RIXAMD4vCUHcsJAIfSRQVcXafOazozai/F9puST6uJpYBxy2aJS1TBkWasN+Tm9h46ZZ4YZvVF9fWZcUyL77una8H0RYD4+0/Pc89iEPhI1b7+pXuwb0uofUj2phat1/UO0+qdckHnf/jcRWA6BxH8V/X/0C4D+BnsmeLb6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAADCUlEQVR4nO1a25brIAiFWfn/X2Ye6oXLRm2Tjuk55SWtEkBA2Nqy0Gbi7RZspp/dBnxpVwiYNym+Hx1btH79r0iFgIn+qCwzzVQxTVmgGP6rJVxIPQRMoTXOFlSSGTFFST3C81gzkgGZ/oFm3kIAagPHL4Imh25g91yhwIs2BQPlSEkdEzB/i/jVEDwWyZk3sVP0d5/1jkvMHGRuo6XAcBXahEFvj/dBt4AFreIGQXiEwBjaPurKQURmCWKX80SOpxHQHpeiUX9nZxwFn8YKpxlYgr/5DoWsg1Ih8mlmzHO2SiEolZsP5cGai4XB83ZIGWQ1SFL4RuGfOFieSp430UFUF5Tk5iI8wSQUY5up6UmvPatCJiZppQwZWUGPLm7IvqsicGI3/ZADd7rYtGeW7Sv4MnT14mluc3Y3Wdhk3CpBSlSQUN7jLgnCGSFHghHR0qOeDnfS4pLkGffEqTkP0RZ0nutJK1g3x0pXNINTbf142QqPY8yMGhSOBlbcPwKRBcP4/irZzhsmInoloLaMpu45B6wObELSNJNEDXyGJOCOBmeEyy7x6Kt8FguTIcbVmrBRSdCW+0Cq9ppGckTBUfaCKmOm4xduecIkKgKp01w/tjNpZVscNFpOwKlpRiySvabrPdC2uzrYU9Lm7NihJAaAc3VkdgSpPYKJxacqV3wK1XRx+eTS8BJ124ZtaEYmBFlj7g9f9ZoVs2sCMyNp+XTBF29QwH4Owqb633ICu0aoDoFejsd/7bEU8EnN9KUZI09A7IXFY0VakvYfgxNKLqu1vbpqeIyy0pAchIwKsAkLp6le/fafcV+ngzrQU3cyjdhMUd0L6UkhkkPf4JXueAWekjPfY7pZnGQNoNtugn5BQSMrA6ZcBgMuADJs2+qNCZvbTfYsN9dxK+LYUF21j7cfqAC7eeOZsTdsuk8C0ECtYUp2zIfQMU2aODOr6IXy4zPicrLTXkEYAn9uBNAN3wrmGfPoXF0536z778bA5mXa/ZPFk3X7G4J3GPDRReQK2v5vOv9T0f9Hv6IGH3A3k/4bAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAAEB0lEQVR4nO1a7Y7cMAiEKu//yvRHYpuBwXZ0q/VVKlK7G4JhzHevpyaHSY8jOEx/TgP4T6dCoHrI8Cfos+DPhOCfDoCoyCfxX6D5S21ZZWlqLULgKuGsTb2ltfvfWR3DkJzLV6JgmJAmTcNl6Dw6jrVUPJHIYasBzqDPb72E1kO0EwR1VUBiq/nB2MvpTqPhc4tUxERnirV9bNmmgSFvwtta5xJYoZwJX/7BtDqVcEUGv+WQMnhXYXte3R+GMYC7D8XrMO2wV/XOUYAq65yOMcOX6AlpIaA30nBh8xYNL/Iix1MEWHjtsaHxsjdDHZwkAnot8UfDjRfM8LKL4Su2iZE8jxSBn5CYH8em6NjoqpAmC+X26HxGgIsxaeCJbEACtC6zrBeMEuf0CxHd6hKWY6LYmovRySiAl8nwoTQa8xoPdcMEu6/I3B9KMQKYta0zdh/dmeVchtfMxgtMgm4ZotV3rjbWi8UYBPg0LBeAEcipcWzeb1f7pX+v4949q8KmZMmZs05t06n9mMgSLq0mNUB3XJqR1mOgEMPSwHDcVUukWOQ8G+tO2VyyQ5tI9wtpyMAKqUX1Ve/oAgMbadX8J1uD5KxoB03Z8AEDvaM93Isj36BulL3xxarFv5hEITwDlGncH3xeKe/xJNLVjqDhAZVZwU8HUyE8f+cZjiEOEZCrHNYsr/muUIFsMFKr6AuNae9yHpSFCIiTqSgGplhmvHpF7XBI+9N2fhb9MKZZ++zSFzXC9riF/f4tr4GuQWp7baGlFjNFrZ1/dJdjQdkT2Zm61SaSO0djN1Vh+WtZHLd+1q7VRgZW/fQCvuYy6SDQKtqLDo338bvYY0SDCQKjr3Bt1hXqE3nxSUvvbS8zJfo9CMT9E2znXhGDBycgBLMm2yZjaJxsItRFM6BoPkGWwZZB07mYsga3neIwlO0EeGyjk1HB+HcZ+HNpPPgQtOOAe1zx/pgvCojT6UjPOxEo96kpDW0Yizjc6DUYs9z2gNksQrxIc0hLSPHD6hD3/io24x0POZk0Z4cAybS5LmCkXhyrE5OZF0fpXUoFyGV+2lgH7o9L7g4fitcjQA/dtcBGT0GhjxdHtoas4zuoOP/hRMw/J8IjELnAg/SlKykZ/YRgHBiUhUMYPJZ3yhKKYJ+P5/P1E0chF5LpuHHFBkALbQM1toG+utGiJ6rZ7CFBVcRv4jsT7cXCwjpLBlCgtQzDT/TnrIz7XZ5eM0NTQNBjt7A72aIv8AtAbDXvk+uBWyUaKihShukqEpY1C6MSmz/vL1rQTwnLxxHL3syjPXEno6aLom2n0VYCv4H2fXpzBZIrp3+R6mVebv/v1lfpRQzIBU7/KpfJq8S23xgBkR8V5+kq+A0IDtNfNnCzlOKQM24AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAAEHklEQVR4nO1a227lMAg0lf//l9mHGBuYwXG0R81ZaXlJamM83Elb0fYyyesIXqaftwH8p7dcIPLSxR+gD2PvH5V2Sv+w/T8P3rlAWvulsizt4KpLU84GAuQZ9ALAkZS/8sA4HLFr2I0Q7iAxgSDXJC0PR1/zdjwVpcLTDoFe0zIhnjjwwV347CKHa9Vhlxxgot3mdqaR9Dyh5VoZwoPikq5loqvomOuyBEjLL3toG6apbTAUAvUm635XpbImaJkXIOoTl4Y9ZEbfauRQMQNd7A6A1uiyPiuQ1Bk+n6rJQfXcwcbeveJRzMMSgPUkjrrPgNdqP4hx6oF8hXHo2NTWtK0AufjEw4s+Xu/xLsXUuYPuQ6LwgLOxGmpRA+7jUSf8BWwNpdow8gL6iEwHFbDtLr1Ya7EpmOzc2FwemKuXACcSEnKD2sdrCk06johIOpifDIPJHMBHwPhD0Auu3VLoo24XSRv6Fq9xoRLSixXeaSi1SMp1SH3C19dmD5ToiN/8YsQuCh5o7IjLz55hCF6im357MF+G0073afgYWDOLl2x+gZkeIuRBT60qy7msDKmaiFKOubjpPAGpwxHkGnfq4sJVcx0JUlxyT3bt7GSgdXfQ/Rg2joVk3Km8SLSgVJNO32zuiU0uc8frJTjr2aAgacVOzDfbVTYwFt5lIuEosBDuyuR38qLwoWv9qdA50qJp1j1ws6zCRla9Js1lx+C1sDTTO/PtkGxKZACkkdufUgmWM5vUrTEWIYOt5oQCTqfiMAPvaLEo8DtVxLY1arIfcYMZaPHhKlQx7H2AiaCer/iyoGRh7ytq1cxWCU+/phMMheY7Jw2eBgbNFOdgMTdp5AF4oL4+qz11FdmISaBCqcToWhvuhPt8CTcWORlcUDXm9cgljZWR/Sw4mKU+IW0qCxEPBVKd7w+S1Wj8HiC6IiDi9ioaoLjABfcKAe6LWi/WU8yux1GPukneXCSCTNukqsA3wNg6K06IyCdzRGRz8cp7qX2cSgeBoxBl46n+TzbeEGnMFhmfo6qnAe94Qp8gF0TuuWl6EQZ6OY2+8hKcr9IPIkJmOeQlk0I+dOV0BL4m8t7a/KJj+Sw5RC7D4FBXUhwTToa/SniIMZSDaVPOgXcemMIUl9jha5uXQ5mHPYPTobtTm/aVhbLAppQ00ehPyu81QUvly9fYkcy2G1ZRIdL6gUrww57WEWi7BODz2WE1vuvNF2jRtUJAcMmLObdYzk1MmktVNWLl41AoDhs5U1bgHS4OAeJYxsKPMZSGJq1lvW1b3m1yORzsNmUmXYujO93ekAUT0HomYyu6agQrEidHbLkkxk9mnj3P/ACjiBDfkfrHjL9L3hLMKvfWfPsfqe56g2f9QgecWPjbXXD2nfHF9PB/N5iE1/V/H8HL9AefE99Zr0VwbAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAAEGUlEQVR4nO1a244bMQiF1fz/L9OH2OYAx5fZZDVbtUhVMjYGzOGW2arJw6SPW/AwfT1twH96CgLVhxT/Proe0fqD/ld5qrCpfE81QPBdEfdJZa1KZcMgxNUqYtq/vGPdRu9M9kFY8Xu5RLK/i6imlTFpkeSeiT4id9Kp2Kg4sMzkyx7yT9DWZmq0CGQBAVHrg7HN5Uyj6fOAVIwop6qrHhExDSZpWTkxoYlyg9I34DM5jRoTqZZcgUNnhpZb54US9YkruYQxGz5pcbm6Q2ZahzIN2W3t7ucgDBVqQk8i4PVrcE4EqNzCGgR+RsPNg1csRdrEpB3NEGiqayXvK9od4rdZYODfx+cxCIqOo6YGyWgUcrUnw91uGYDs7dg0mp59kczn+sZiV9haAGCcJNWYToGn2fqu3BCDVO5pPttpoHicliSYhImfmukJ0YzJoJcr47EpCeJ7ZPCP0oin5k+F1WgsHGgLAQMmti7frEUpF7L/mFH9WC4ZIccVEfnKAmgKm61Gse34GB8a/Op7JiLmKZAcWMRb3ckup0PaeTdQRQSyc7mUYkJzGkYgQ8C+Xuus3/haTKJgat+ZFZfZrRW4ckghrFGyMo9sNHWh99oVk7qWsArSlua8CF+r00uajbklcEj0tXJjWOK0B4zCExFbl606hw8Lp/c8bhpUGxVk6gWjl6V2+2vWT2lczzN/sVyH8pcxr/ZfrjuStfq2XSDgywCIpsL0Ufc4QTkcZTz4JPrj1ZSK6Bkyptjr5aIW6fKRUZhWEr/p8Nv41RUmtSYgTBsmKmrjlQOoIb0BlomtFeUbad/9RQU6OKPWlCg1fBgGg7XxNd2IwBRFdQyPFy4OTSZL/K3UzJ1EzJgCXod0iJAW9YXq0JL1J3avA5wjHGagBkH48gVdpL7Qo67vDgGWICAFGCexfobZd5KAGBikgZcy03+ppCrGVWGx0Mw2M6TajizZXYXRAAOtBdFbXbse7zEIAdSCUPUMPyhMmUrY1WeOQD1W5x/6Az48Q21eVdlzkIiqlwSIz1Sc8nY1vqM8eVmNjFg18q/8k9EBa8z8+tArKndcG9cICQoneuXkPj5qBEeRlvUSISmVGF0iXnFJ59AcjOFy5wjkWYKwtO8Y42ys6RtRbpRvYek+VQDOISFsKWRcpom0LJjmLBcbZoSNYQkAi3g6j493WA6JEXFGKm31XvTObT4T4km9rm4eM2yYqA01zyrl1G6UGAK20A6mWN5Jj+djZ21d79HUkRMdMGsTDpgxmNxxuM6T+0DaNDSN0XkLARBfF40zHL37OXtBlJJ2c3zjqQIBGx3o7HaSy2se/LPXTtRxDL8Z7J+oUFngHoEFgzfUz9p1m7YX+b20BXXJ4JtPQ/D56PxbaHj+8f9Nl/9U9M/QuPYfLljGhhMZchAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAAEY0lEQVR4nO1aSW7kMAxkBf7/l2sOlsRdcjKDdg5DIN1uiyJL3G0ElJcJryN4mb7eBvCf3nIB8JLi5/QpiNdn1AT6/fb/IEbjAoh8qCxDPqbqGzohluWJNU48USfGL/UuxTZDj8Bt2alIm8ZSkqR4PfKyHRdgGu2cl0djlDwwoBzLAxc88YBluVFChM4Hl+evcPfy5uUOCML3E8L82ghecjmuIwrnFjbqA7ryeJuN3N2JIrA+PdTLrhOdNROiJD9GfeDy0AprLa5lDajU7sBDM5vjGrodhZEzMdTLeM4nigFdKFyShkGTZbKeeyLS5m+DWyvHLc/u7ox4po0HBBM6afQhOdTpy8cUDBGRGsUlJON9AA894FXRxlRL2o7pi1TCFUCaQMo044wYLcBZOYDP9U5PP8vMfRQwcWXvFDomx4xP+luwN4M4C8+p12sDcZUZzWuPrCoyXyqSnV+f9MaGaP5KmkE/g9ZGv4g7HmkNrvzTfKbqMUUfiiuPK8QSE0O1DO+BGxUGoPmhTaCkS7xTbU7PK2767YP50u2Gj8XsXQZWo6CoOH2jdbL88BQGszmooOBxMtKwNLD5xr+6gDnJ1kJfo3DG+y6Xw9xmuOrZxTL2/R2eYxVmDfUQiTQbnXW3xVBEoGlRZANH7pHciosaAW04adpIudaXkq925USjR53cJyye9O87SIelCEZcYyGPE4VPPG17PdHBdBEscQnTJ3tptwMajm8MJyJytT6v4prxxhPlRDWyzmeUWUimSXi33Qmg2+xvMRek6qECnO3yBtoUiDvRiHBe24yTfVxWHhpI0HaVK+k4RzK9MHc2iD6a03igBkWE6mrE0Xyq4uEBaHx3xLbQ6PoAtrwVKEVjpbSayUpxiK/pNPt9dVuVNHaphWc7NY3IdkpQWvJmd71k/8Jy7oP57RD6dDiPiGvFoQhN3KdYMRHoLXh8CYPQuaD0HexXTO+lKpqzJo2gTQOPdkprrBn1jo11zkd+GyRttbAhXjzaLg/4hTz8b8IG1EpIAV0WTA/EY9J+1SkWaJ+l2QLZPCZtfJlFEhfyFdrSxdgr6twewow/bkrVqtdEYpLsAnh4lsoFti+rA9i1FHPwSe+3pXxdZwvYRC+Tu9y1pprbi5jOZGCowqqEGucv/7N7c5i7kMZ6NGrs45dokBSCBD5EZy74VrAlEwDdFh+h5yRTCsMSWVj3CUjLmePECM1d0aIxB9Fqs77jM+qgy/zcdie/O3e9hoIDWLVt64FVZru3ttyoZgWnbBjeGGWHqLRjFcNdJ7lZbJcM794MG2WMyk5lRJi0tYUvCji6djHR/TTNpxe78WMNlx0n6qkiK86yegyeMzY7KwR5njw33E1FNwL2cL2sam4sXL8R1bsgrm1qVSMALj5bviaYOqVrqczcYzne86wHMDmLKipTc8wDrkejmux91e14yP8UQtj1gz3/kqre0PO+AvZnhn1Ob/8r17fGn3fC5fQS8G/p7Sz4DQhepj+5PAJ+nK7YFAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAAEaklEQVR4nO1a7bLdNgiEjN//lbc/ZEnLArJzpxO3nWomsY+FYPnGThz28fLPEXy8fn0N4P/1lQvcPxL8z1vXJ1L/0/Z3s9+qreSC3z368+X2LOqIxi1w4Hbif0qJbj2ZMSl/hS1pjU/q3MFcEWVOG9oLX5/TxKMI1x370BOvbBasc4Wtgjr+QLV5nGlcrm+Wmxk88qXguW+9keswM7gjyFRaN9x/is0fLIqwB3PAWDVfLhhIOq2yAfVBinqhCqXCEjHYxuMOObqHkHl+oq2SIEOsAC4RWW3RDwl0J2bT0QYZHDe3TXQJtYc4p7CzaBMkyW9X5YFwfhJ4KDWYVIUx1u9tqUCVe4/vA2UZHQfIGJqUkY0ZxnbtgVWd5pVcYti9AB4Nq6oKgEpSEgnHDvGabSoWiwBOEXCrABY90iADcCj0lLM2fVqG9tzmzemDcIQ9iHCpMJWCMLJgZr7C4NuflUtYVqYTM3B6ci5HtrjM1QPwpqklt6+seiZnNJr8rdFi0Zi6RQBD0EWIFulicl8PDebFfKldlXNyN/mFantgRr7HSJMCk+t/7rzpN0hywtt5QIpI6UIU9z4x3plEIGFmVy8y+aJI+XXti0sdak6uoevNyLFZhnoUMSDZMbXNhRDhySFyZmzoOJYf9y06LixpmH0PW9IoRD8qMqujVTvBKrnizgFnx8OMkzma2bKa6yBUvhBUDyQ/EB8iRQ7R1R4Qmxct5smS8Oz8q+unZTBBHxTnCrHVRAgb7R9sk+Wc8ffO5InITxnnFo9xksysix7g5GAPiYy2CrTN4OiKwG68F7S9p/lZLR7PlB4ep7LlgWKuolLp8yVLSjxnGDs4o8S28f0+dIfWnKzoxYLLXqNvmmBvaIW901sl3XKEjGz3GNsQQguds6gopL920kx3v49yUlfzNOSQbe7hxQVEvlteE4N7o8hxeLJCyUH9EI/uUUubNkOUrApfSp+nsSoOmCufaHSwogxr2kTm1DIy72kSpq0nOC7dooWpWXr0EvghZ6jtVhBc6G+NrkgCIg0P1+XVGCAUuW1lJihux4ce/VaRQvl+MuK/94Gnm4oGFbiFi5o6x7nETBYbTCjwfgVadm6Q7IbhXrwNeKIJfaIQELjkMJcD4oGl2KxATQ1xrpltEOHgAUbRN4w0uOhpxLOjHcvrgsxy0gRCqXvlgQipa3XVbaZ19cAdeVwekEmFb0rSOBUcstxLL1La2n4RI5arlyYAbvsDxVF03HHm91CVRClEfy4aMo9nM2sdYiY++WamrdKy4rems0JzBGj4zSkn95o9uukBzw1VXHz+kNsU3ZBPT0r1ZWlHJ3ST4PbjSyG/LkPn8qNcdSQ6cOKfGbu5mV11YNLKOyh4dWifiMqg1GPZyjjsRTbFxstnLVcpqkXEHwRHlyE1Z8tE9TrTrIH9DGlyelHMXpvoaf2d/6D5ELuL5kTix4+gf2Q9NkCzbs7/N6w3Be7r/8qF9dcT0Tfrzcx3WG+Qfx9g3yP4eP0FLFYHlbKmq9AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAAETUlEQVR4nO1a23LdMAiEjP//l+mDbIldFtnNJHE6U82k58hCsNyJUw97efnrCF5eH28D+L/ecoH7S4J/33rHBf+WA74Z7AGSfqgsuz0Ute8TbpmPX1/x8VNE0cnzb7LKlOVJBWPkd9LP8FBEXjgtD6OvezPf2NLNLLww3YC6XzngA9h+htnu5mJ8pEc7POcm1OGdqI7/zRLa5/Q5PUDS3SzMARRG2eU0DK+xQ4znvacemJcDdvk0cH+a1COTMC7QF5iAiFBRj1QoHaQk5aV0x/DGsL+cMj6Xi0rO1fgWkNBw0SaBD++LkIzNzqo9za4sWE8cmAKLyBeDIkbgbFYT3Ols/Bt+eQfIA0gnqLjCKgifiG9lVCIK8Q3xciMy8kfkvMgKFs+udhyOhmVTERblz3T3TGIPQlPYOqd/tqJDsY9BjTW6RDetCwjkDN3zCfFB1VEKF1t5mFm4L4zLCB4zvMzsHEoHSXBRha+fa0qRfuRqFErf0QMqFM6gxIPg7WWDUlNTm6vpWYDnEpMO0XZXG4kpMQakWD6ZWn4YKSayMYYCcj2YL7mrnyicz2LJS5+Tmj3A1PcrTxSl6g9n3v7CMsIxQjRzRU0g4wwZID+algMPRe7iE2IKhtVKecXHLS5vrszuoHCL1xpZUiUa4qYBJyYlCCoY0ExxzIPPkXn91aoJnU5yino1yLjrcqqQYBwHFgwQNyyuvQeU0dK44vMJp8oaqwSLJsoadwbP8EfXT2Vcc/1U94TMOrKeo0x4xi+GjGl7j84DpRXoOB4jk4xgejj4hVkeq8ajZj6ZfbdoKeCUAnyYAuzbrVoplgp9zDkmN9US8mUyAgLHLfAfBKtGaap0EH5JJ88sBXxxLfFXwBLylbKNaZeXbRQDiO0z3zDMroeqoqScDbig6PyM/nwTyjtumjLtEl0gUV1O7Ndqi6mHrzTddJsKdGgaBY8XCx9GVxWY9cHGnfj6cBa86lRAkXY6tDDqPOAGodCsVRGhs8zAiULr5itiBNyCZnL0pSObxuPSbpwceD6xcWWdH/uJgbSqgOd+5wHd4FY0rtLreFISZweMIrR0rIWjFEEnx1W4VyuBE12bmpfVCHUdhZWUu11ydth6YH2nnJvUaUKKjOouQJzAp610e3qoPZDhF2GWMy/7EYx4WGo5IpedfReZw3MPUFtVJNiTnOsQ7Kg5OZ4shiqh+y0KHPUCyhX1HRu5U5MaUSs35kpW/cywi4BijCKWRdEeQ+j87UC+dNYdsOSSd75YyuyKCbM1pmW6kgOKll8Qlupx2bs2VI6Owj5D1bLRIPve0RAJk7EHKuKbeLspUrjdeMAYiqTFF/a1cqVuwoF533Cbik4M1LDU8ZK5xKgwN2WYSG7PgNCNnth7XET3yIvqL15PZp49zXr5e8/qIdmzSWwn5PO3P4vkIeHb/5HqRzzwpes5koe2fdsFzaz8m9dXm+x1F/wCBC+vP68QDHJEx9+aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAAcCAAAAABJiufSAAAEzElEQVR4nO1aSXLjMAxEp/z/L/ccSGInpTjxaA7DqsS2RGLvBuQElIcXHrfg4fX1tAH/11MpAD4l+ENyP7dej2j9XJx+QGsQEXmAFV0K8NcMgPxElbcTSQ7mlXz9UtAvZ+CW/rksBZBSQ1eCZjF3m6okc/g614cc4WQY7HUDBzDu9EfvOHszsmvbEZTTi5d93ijdWOBuXujZyj8fQdSVdMy7bGwE94HCVuAxuqal91XvM27cRgbDcFAsBQOG2J3piibJ7G7broj4FmyrtEd8kAoWHE5VZ7Y21gX7Xey9yACTCi8EzVVGuMYKHs4NrOKVZYTaS+edBbH8vlnjxf3sD0W4diqMugzkz+wMMdoi2CIg2ZSaBPUEZ68JNlV97iq7O9OggRNaLyCiBzlUSZsltDNCTWaMYxVbzqsNcHmIWLe2xSyYq1mEYx5QbBtJZthUbXTvg42rLkJ9jKShCFWr10UqCoaMlkr07XuzAt1Pu9Q9Ix5aECCkLHYcFUyRvtZDzCn65JHAc4bJxkSnQ+3kqgRNt5pF79gSQVc56yIEwpdYFpejwSUZ8D3ZdzXetBiHxkZDn+VKApAtNiPPidz9W5SQh4/lnoS+OgyGVYKvTy6cMeuH0Gor7sbX0FKiGETsBjmttGp3PtvI9zXjJUDl0oho0Yx1bH3jtWHZTe8SsIDFpS7gwtvYD0qagQVpjXwK3cxXZfHIUmbs7AVvkUyIYL4TKqDiXIeckwLtgOuQjUkG+TSB7iTSub5y0BcWAryy6doXTFgzTOmIZ+oMNipiEchr1093vBAvNOfKIjJpKDUSeXgLbXceb0A1mahV1xmV0RKMCvEKSSzyXa8ubdssCwME/IhgBhqDv3ZGHz52y7ZU+l6j3LTHNacD/hwNwr1uO8AgaO0R3l8/Nun+lQOrWIoggkpJsALGVUrGTaJWCEgFvFWanolf0xnx1g4VAxCNughoNBMrTR5TGQSJnuFfg7v6blXb4i8WYaVFmkPoSseRDkIuCed06y5NlbLPkEn/IiIpBQ2tOTyN35UbvZf+RLP8nNPm68BvjLfnY4zVvGvdq2faQNtYKQS4QjWGmPJ1hysMa8Q6lI7fsTTn2AD6WDYUsUFB7hP+or60acor7ciBTSiL+aB5fFqjEHMGZv3hNEd7IRqS+sAWMxBhanTu2qs4RxnDpPvB8gi8+7K68oAl/V7Bt3tMdR+hGcQ812gI/JNzGS1cebGMTpISvW5OQYykXz3zPcSuBXO0o1dLm6O2XuJqwSfXJESa3/Wn7fKG9UfgHdxUr9WRRdseo8PU5Im4gRNUyahKiM+EN9PVdhASntNCGtyh8ZVniB3WdBBNQk1N5iFJFsQoNiGrxZQ+R4eSOjbjQEX62BdrhvbB9jc939VsbDB9BmJ0a2424SiA15PW2SjiiShxj747s+qGUEJsesm9OtSkt1DTJipr5ugUHAcDfyJ3p6RM4RZUqHt6I887aRZwA5VnnTpPXjfca0b31XUWVltzPoLdhht/Hdw96V506yYjG6wcS7CKjdnjbvOdmee8x3+nfmOyyTRXae/WeHMh+jt/zn1Lwbv7n/5HqhsB/kDsrtc9pd+L3sbXp/+V64YP9Wvcz6+bGbg1Eupi7+7TKPgXLHh4/QEByluWyXFPdgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 56,
       "width": 772
      }
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Speed: 136.27/s (2054 in 8m 5s)\n",
      "Epoch: 10/10\n",
      "Batch: 200/206\n",
      "Loss: 0.16\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAusAAAH0CAYAAACEkWPuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAWJQAAFiUBSVIk8AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzs3XeYVNX9x/HP2b4ssPTeQZogVYrYQCQosfuLRiWKLUaNxphiYoyxGzXEaNRoLMSWWKOJWFAEpASURRCll4Wlw1K21zm/P2Z2mNmd2b07O9yZhffreXzu7i1nzi4qn/nO955rrLUCAAAAEH8SYj0BAAAAAKER1gEAAIA4RVgHAAAA4hRhHQAAAIhThHUAAAAgThHWAQAAgDhFWAcAAADiFGEdAAAAiFOEdQAAACBOEdYBAACAOEVYBwAAAOIUYR0AAACIU4R1AAAAIE4R1gEAAIA4RVgHAAAA4hRhHQAAAIhTSbGegJuMMZslNZeUHeOpAAAA4OjWQ1KetbZnQwY5psK6pObp6emtBgwY0CrWEwEAAMDRa/Xq1SouLm7wOMdaWM8eMGBAq6ysrFjPAwAAAEexESNGaNmyZdkNHYeedQAAACBOEdYBAACAOEVYBwAAAOIUYR0AAACIU4R1AAAAIE4R1gEAAIA4RVgHAAAA4lTUwroxposx5kVjzA5jTKkxJtsY87gxpqXD6083xlgH/3SN1pwBAACAeBaVhyIZY3pLWiSpnaT3Ja2RNErSrZImG2PGWWtz6xgmW9I9YY4NlnShpO+stTnRmDMAAAAQ76L1BNOn5Q3qt1hrn6zaaYyZLuk2SQ9IuqG2Aay12ZL+EOqYMeafvi+fi8JcAQAAgEahwW0wxphekibJWxl/qtrhuyUVSppqjMmIcPzWki6QVCzplchnCgAAADQu0ehZn+DbzrLWegIPWGvzJS2U1ETSmAjHv0pSqqS3rLUHIp0kAAAA0NhEow2mn2+7Lszx9fJW3vtKmh3B+Nf6ts86vcAYkxXmUP8IXh8AAACIiWhU1jN920Nhjlftb1HfgY0xp8kbsL+z1i6KYG4AAABAoxWtG0xrY3xbG8G11/u2jqvqkmStHRFyIt6K+/AI5gEAAAC4LhphvapynhnmePNq5zlijGkl6SIdBTeWejxWHmtlJSUao4QEU+c1AAAAQDTC+lrftm+Y48f5tuF62sO5Ut4bS/9hrT0YycTixflPL9Q327zvVd67aZyGdq13RxAAAACOQdHoWZ/j204yxgSNZ4xpJmmcvNXxxfUc9zrfttGvrR5YR7c2km4gAAAAHIsaHNattRslzZLUQ9JN1Q7fIylD0svW2sKqncaY/saYsCuzGGNOkTRA0rdHxY2l5nBcJ6oDAADAqWjdYHqjpEWSnjDGnCFptaTRksbL2/5yZ7XzV/u24Zq3q24sbfRVdal6ZT1m0wAAAEAjE402mKrq+khJM+QN6bdL6i3pCUljrbW5TscyxrSUdLGOghtLq5igtySkdQAAADgTtaUbrbU5kqY5PDfscii+p5SmR2te8YDKOgAAACIRlco6amfoWQcAAEAECOsuoLIOAACASBDWXRDYs87SjQAAAHCKsO4CI9pgAAAAUH+EdTcEVdZjNw0AAAA0LoR1FwT1rFNbBwAAgEOEdReY4LQOAAAAOEJYdwE96wAAAIgEYd0Fhp51AAAARICw7oKgsE5tHQAAAA4R1l0Q1AZDVgcAAIBDhHUXBFfWAQAAAGcI6y4wAWndQ2kdAAAADhHWXRC4ciOldQAAADhFWHcBN5gCAAAgEoR1FwQ9E4msDgAAAIcI6y4I7FknrAMAAMApwroLgirrMZsFAAAAGhvCuguCn2BKXAcAAIAzhHVXBLTBxHAWAAAAaFwI6y4IrqzHbh4AAABoXAjrLghaZ53aOgAAABwirLuAyjoAAAAiQVh3gaFnHQAAABEgrLuAyjoAAAAiQVh3QVBYp7YOAAAAhwjrLghqgyGrAwAAwCHCuhuCKusAAACAM4R1FySYwMo6cR0AAADOENZdELjOOlkdAAAAThHWXcANpgAAAIgEYd0FVNYBAAAQCcK6C4xhNRgAAADUH2HdBUGV9ZjNAgAAAI0NYd0NQU8wJa4DAADAGcK6C4IeihTDeQAAAKBxIay7wNAHAwAAgAgQ1l0QnNVJ6wAAAHCGsO6CoHXWyeoAAABwiLDuAnrWAQAAEAnCuguorAMAACAShHUXBIV1ausAAABwiLDuCp5gCgAAgPojrLsguLIOAAAAOENYd0Hg0o2U1gEAAOAUYd0FVNYBAAAQCcK6CxIMPesAAACoP8K6CwLbYDykdQAAADhEWHeBobIOAACACBDWXUZWBwAAgFOEdRcEP8GUuA4AAABnCOsuMMGLNwIAAACOENZdEFxZj908AAAA0LgQ1l0QWFe3dK0DAADAIcK6C6isAwAAIBKEdRcELd0Yw3kAAACgcSGsuyCoDYa0DgAAAIcI624IbIOhtg4AAACHohbWjTFdjDEvGmN2GGNKjTHZxpjHjTEtIxhrsDHmZWNMjm+sPcaYecaYH0Vrvm4KXLqRyjoAAACcSorGIMaY3pIWSWon6X1JaySNknSrpMnGmHHW2lyHY10l6XlJRZI+kJQtqYWkQZLOlvRyNObsJsMy6wAAAIhAVMK6pKflDeq3WGufrNppjJku6TZJD0i6oa5BjDFj5A3q30qabK3dVe14cpTm66rgnnVK6wAAAHCmwW0wxphekibJWwF/qtrhuyUVSppqjMlwMNwjkhIlXVE9qEuStba8YbONDZZuBAAAQCSiUVmf4NvOstZ6Ag9Ya/ONMQvlDfNjJM0ON4gxpoukUyQtlfSdMWa8pBHyrna4XNKc6uM3FkE96zGcBwAAABqXaIT1fr7tujDH18sb1vuqlrAu6cSA8z+XdHq14yuNMRdaazdEOM+YobIOAACASEQjrGf6tofCHK/a36KOcdr5tj+QtE/ShfKG+7byttNMlTTTGDPYWltW20DGmKwwh/rXMYcjIvChSB7SOgAAABxyY531qqRaV0pNDNhea639t7U2z1q7UdKV8rbH9JV00ZGZ5pETdINpzGYBAACAxiYalfWqynlmmOPNq50XzgHftlTSh4EHrLXWGPO+pJHyLgn5z9oGstaOCLXfV3EfXsc8os7wCFMAAABEIBqV9bW+bd8wx4/zbcP1tFcfJz/MjaRVYT69HnOLC9xgCgAAgEhEI6zP8W0nGWOCxjPGNJM0TlKxpMV1jPONvL3qbYwx7UMcH+TbZkc+1djgBlMAAABEosFh3ddTPktSD0k3VTt8j6QMSS9bawurdhpj+htjgm72tNZWSHrW9+0jgcHfGDNY0lWSKiS93dA5uy24Z520DgAAAGei9QTTGyUtkvSEMeYMSasljZY0Xt72lzurnb/atzXV9j8o6QxJP5I02BgzV97VYC6SlCbpdpZuBAAAwLEiKqvB+KrrIyXNkDek3y6pt6QnJI211uY6HKdI3rB+j6Qm8lbqz5X3jcDZ1trp0Ziv2wKXbiSrAwAAwKloVdZlrc2RNM3hudUr6oHHiiT9wffPUYfKOgAAAJxyY531Y15QGwy1dQAAADhEWHdB4NKNZHUAAAA4RVh3gSGrAwAAIAKEdRcEP8CUuA4AAABnCOsuYOlGAAAARIKw7oLAnnWyOgAAAJwirLuAyjoAAAAiQVh3GUs3AgAAwCnCugsSAp9gSlYHAACAQ4R1FwS3wZDWAQAA4Axh3QVBSzfGbBYAAABobAjrLjC0wQAAACAChHUXBD/BlLQOAAAAZwjrLgh+gmnMpgEAAIBGhrDuBsNDkQAAAFB/hHUXUFkHAABAJAjrLgjsWae2DgAAAKcI6y4wYjUYAAAA1B9h3QXBD0WK3TwAAADQuBDWXRD8UCTSOgAAAJwhrLuAyjoAAAAiQVh3QVDPegznAQAAgMaFsO4GKusAAACIAGHdBfSsAwAAIBKEdReYoKb12M0DAAAAjQth3QXBlXUAAADAGcK6CxICfssemtYBAADgEGHdBTzBFAAAAJEgrLuAlnUAAABEgrDuMktpHQAAAA4R1l0QuBoMUR0AAABOEdZdELgaDGkdAAAAThHWXRDcs05aBwAAgDOEdRewGgwAAAAiQVh3QVBlnbAOAAAAhwjrLgh+gilpHQAAAM4Q1l1AZR0AAACRIKy7gqUbAQAAUH+EdRdQWQcAAEAkCOsuCFpnndo6AAAAHCKsuyDoCaZkdQAAADhEWHdB8GowAAAAgDOEdRcE96wT1wEAAOAMYd0FCYbVYAAAAFB/hHU3BFTWPaR1AAAAOERYd0FQzzptMAAAAHCIsO6CwNVgAAAAAKcI6y4IrqzHbBoAAABoZAjrLghaDYZbTAEAAOAQYd0FRjwUCQAAAPVHWHdB8DrrsZsHAAAAGhfCuguCn2BKWgcAAIAzhHU3UFkHAABABAjrLgjqWY/hPAAAANC4ENZdYIL7YAAAAABHCOsuoGcdAAAAkSCsuyDwCab0rAMAAMApwroLgh+KBAAAADhDWHdBUBsMpXUAAAA4FLWwbozpYox50RizwxhTaozJNsY8boxpWY8x5hpjbC3/pEVrvm6isg4AAIBIJEVjEGNMb0mLJLWT9L6kNZJGSbpV0mRjzDhrbW49hrwnzP6KBk00ZuhZBwAAQP1FJaxLelreoH6LtfbJqp3GmOmSbpP0gKQbnA5mrf1DlOYVFxKCHopEWgcAAIAzDW6DMcb0kjRJUrakp6odvltSoaSpxpiMhr5WYxW0GkwM5wEAAIDGJRqV9Qm+7SxrrSfwgLU23xizUN4wP0bSbCcDGmMukdRTUpmk1ZI+t9aWRmGuMRF8g2nMpgEAAIBGJhphvZ9vuy7M8fXyhvW+chjWJf2r2vd7jDE3WWvfdnKxMSYrzKH+Dl8/qoJvMCWtAwAAwJlorAaT6dseCnO8an8LB2O9L+kcSV0kpcsbrh/yXfuGMeasBswzZgw3mAIAACAC0brBtDZVSbXOmGqt/XO1XWsl/dYYs0PSk5IelPSRg3FGhJyIt+I+vK7roy2wsu4hrAMAAMChaFTWqyrnmWGON692XiSel3fZxqHGmGYNGCcmEgOWg/GQ1gEAAOBQNML6Wt+2b5jjx/m24Xra62StLZGU7/u20a0qExjWK+mDAQAAgEPRCOtzfNtJxpig8XxV8HGSiiUtjvQFjDH9JLWUN7Dvi3ScWAkK61TWAQAA4FCDw7q1dqOkWZJ6SLqp2uF75K2Ev2ytLazaaYzpb4wJWpnFGNPLGNO5+vjGmDaSXvJ9+y9rbaN7immiIawDAACg/qJ1g+mNkhZJesIYc4a8a6OPljRe3vaXO6udv9q3DVyC/FRJzxtj5knaKGm/pG6Szpa3H36ppF9Fab6uorIOAACASEQlrFtrNxpjRkq6V9JkeQP2TklPSLrHWrvfwTBZkl6VNELSUHlvTM2XtFLSm5KetdaWRWO+bksIvMGUnnUAAAA4FLWlG621OZKmOTzXhNi3UtJV0ZpPPKENBgAAAJGIxg2mqENCwG+ZyjoAAACcIqy7gMo6AAAAIkFYdwE3mAIAACAShHUXBN9gGsOJAAAAoFEhrLuANhgAAABEgrDugqA2GG4wBQAAgEOEdRckBFTWPVTWAQAA4BBh3QVU1gEAABAJwroLArK6rJUsgR0AAAAOENZdYIwJCuzcZAoAAAAnCOsuoRUGAAAA9UVYdwkPRgIAAEB9EdZdwlrrAAAAqC/CukuCnmLqieFEAAAA0GgQ1l2SX1Lh/3pvQWkMZwIAAIDGgrAeA28uzYn1FAAAANAIENZjYF8+lXUAAADUjbDukpTEw7/qgZ2ax3AmAAAAaCwI6y6ZNq6H/+sKVoMBAACAA4R1lyQHVNbLK1gOBgAAAHUjrLskKfHw0o3lVNYBAADgAGHdJUGV9Uoq6wAAAKgbYd0lyQGV9QrCOgAAABwgrLskuLJOGwwAAADqRlh3SRJtMAAAAKgnwrpLUgJvMCWsAwAAwAHCukuSEg7/qitogwEAAIADhHWXJCcd/lWXUVkHAACAA4R1lyQnBK4GQ2UdAAAAdSOsuyQtOdH/dXF5ZQxnAgAAgMaCsO6SpmlJ/q8LSitiOBMAAAA0FoR1lzRNPRzW80vKYzgTAAAANBaEdZc0C6ysl1BZBwAAQN0I6y5plprs/zqfsA4AAAAHCOsuCepZL6uQtawIAwAAgNoR1l2SmGCU7HuKqbWstQ4AAIC6EdZdlJZ0ePnG0grCOgAAAGpHWHdRavLhX3cJa60DAACgDoR1F6UGVtbLqawDAACgdoR1FwVW1ksrqKwDAACgdoR1FwX2rJdQWQcAAEAdCOsuorIOAACA+iCsuyiNnnUAAADUA2HdRUGrwVBZBwAAQB0I6y6iZx0AAAD1QVh3ET3rAAAAqA/CuouorAMAAKA+COsuCqqs8wRTAAAA1IGw7qK05IDKegWVdQAAANSOsO6i1KTAyjphHQAAALUjrLsouLJOGwwAAABqR1h3EZV1AAAA1Adh3UWpVNYBAABQD4R1F1FZBwAAQH0Q1l1EzzoAAADqg7DuIirrAAAAqA/CuosCK+ulVNYBAABQB8K6i9KorAMAAKAeohbWjTFdjDEvGmN2GGNKjTHZxpjHjTEtGzDmqcaYSmOMNcbcH625xgqrwQAAAKA+kqIxiDGmt6RFktpJel/SGkmjJN0qabIxZpy1NreeYzaT9A9JRZKaRmOesZaWTGUdAAAAzkWrsv60vEH9Fmvt+dbaO6y1EyT9WVI/SQ9EMOZfJGVKeihKc4y51CQq6wAAAHCuwWHdGNNL0iRJ2ZKeqnb4bkmFkqYaYzLqMeZ5kqZJukXSjobOMV5QWQcAAEB9RKOyPsG3nWWtDUqg1tp8SQslNZE0xslgxph2kv4u6T1r7atRmF/coLIOAACA+ohGz3o/33ZdmOPr5a2895U028F4z8n7JuKGSCdkjMkKc6h/pGNGA5V1AAAA1Ec0wnqmb3sozPGq/S3qGsgYc7Wk8yRdYq3dHYW5xZXqlXVrrYwxMZwRAAAA4llUVoOpQ1UatbWeZEwPSY9Lesta+2ZDXtBaOyLMa2RJGt6QsRsiMcEoOdGovNLKWqm80iolibAOAACA0KLRs15VOc8Mc7x5tfPCeVFSsaQbozCnuEXfOgAAAJyKRlhf69v2DXP8ON82XE97leHyLv+41/cQJGuMsZJe8h2/07fvvYZNN7YC+9ZLygnrAAAACC8abTBzfNtJxpiEwBVhfA82GidvxXxxHeO8LO+qMdUdJ+lUScslZUn6usEzjqHAyjo3mQIAAKA2DQ7r1tqNxphZ8q74cpOkJwMO3yMpQ9Kz1trCqp3GmP6+a9cEjHNLqPGNMVfJG9ZnWmt/19D5xlpq4IowtMEAAACgFtG6wfRGSYskPWGMOUPSakmjJY2Xt/3lzmrnr/Ztj7m7K4N61qmsAwAAoBbR6FmXtXajpJGSZsgb0m+X1FvSE5LGWmtzo/E6R4M0KusAAABwKGpLN1prcyRNc3iu44q6tXaGvG8CjgqpSTwYCQAAAM5EpbIO59KSWboRAAAAzhDWXUZlHQAAAE4R1l1GZR0AAABOEdZdRmUdAAAAThHWXRZUWecJpgAAAKgFYd1lQZX1CirrAAAACI+w7rLgyjphHQAAAOER1l0WXFmnDQYAAADhEdZdRmUdAAAAThHWXUZlHQAAAE4R1l2WSmUdAAAADhHWXRbYBkNlHQAAALUhrLsssA2GyjoAAABqQ1h3GZV1AAAAOEVYd1nQDaZU1gEAAFALwrrLgpZupLIOAACAWhDWXUZlHQAAAE4R1l1GZR0AAABOEdZdRmUdAAAAThHWXUZlHQAAAE4R1l1GZR0AAABOEdZdFvRQpIpKWWtjOBsAAADEM8K6y5ISE5SUYCRJ1krllYR1AAAAhEZYjwH61gEAAOAEYT0G6FsHAACAE4T1GAiqrJdTWQcAAEBohPUYCKqsV1BZBwAAQGiE9RhIpbIOAAAABwjrMUBlHQAAAE4Q1mMgLTnwBlMq6wAAAAiNsB4DqUmH22CorAMAACAcwnoMBFbW6VkHAABAOIT1GKCyDgAAACcI6zFAZR0AAABOENZjgMo6AAAAnCCsxwCVdQAAADhBWI8BKusAAABwgrAeA1TWAQAA4ARhPQYCK+sl5VTWAQAAEBphPQaCnmBaQWUdAAAAoRHWYyA1mco6AAAA6kZYj4G0gLBeXF4Rw5kAAAAgnhHWY6Blk2T/1/sLy2I4EwAAAMQzwnoMtGma6v86t4CwDgAAgNAI6zHQummK/+t9BaUxnAkAAADiGWE9BpqlHm6DKSpjNRgAAACERliPgeRE4/+6vJLVYAAAABAaYT0GEhOMjC+ve6xU6bGxnRAAAADiEmE9BowxSk48/Kunug4AAIBQCOsxkhIQ1ssI6wAAAAiBsB4jgX3rFZW0wQAAAKAmwnqM0AYDAACAuhDWYyQwrJdVENYBAABQE2E9Rli+EQAAAHUhrMdIcBsMPesAAACoibAeI/SsAwAAoC6E9RhJTiKsAwAAoHaE9RhJCehZL+UGUwAAAIRAWI+RzPQU/9f7C8tiOBMAAADEq6iFdWNMF2PMi8aYHcaYUmNMtjHmcWNMy3qM8UtjzIe+awuMMXnGmJXGmOnGmC7Rmms86JiZ5v9656GSGM4EAAAA8SopGoMYY3pLWiSpnaT3Ja2RNErSrZImG2PGWWtzHQz1Y0kFkuZJ2i0pWdIwSbdJusYYc7q19utozDnWOgSE9T15hHUAAADUFJWwLulpeYP6LdbaJ6t2GmOmyxu0H5B0g4NxBllrayRXY8x1kp7zjXN2VGYcY5npyf6v80rKYzgTAAAAxKsGt8EYY3pJmiQpW9JT1Q7fLalQ0lRjTEZdY4UK6j5v+rbHRTjNuNMs7fD7pLySihjOBAAAAPEqGj3rE3zbWdbaoGVNrLX5khZKaiJpTANe4xzf9psGjBFXmqcFVNaLqawDAACgpmi0wfTzbdeFOb5e3sp7X0mznQxojLlWUhdJTSUNljRR0hZJdzi8PivMof5OrndDYGU9n8o6AAAAQohGWM/0bQ+FOV61v0U9xrxW0uiA77+SdJm1dkM95xa3mgVU1vPpWQcAAEAI0brBtDZVT/+xTi+w1o6RJGNMa0nD5b2xNMsYc4m19mMH148IORFvxX2403kcSVTWAQAAUJdo9KxXVc4zwxxvXu08x6y1udbaT+VtoymW9LIxJr3+U4w/hHUAAADUJRphfa1v2zfM8aoVXML1tNfJWntQ0v8ktZV0fKTjxJOMlCQZ32cOxeWVKq/01H4BAAAAjjnRCOtzfNtJxpig8YwxzSSNk7cqvriBr9PZtz0qytAJCUZNUw9X1wuorgMAAKCaBod1a+1GSbMk9ZB0U7XD90jKkPSytbawaqcxpr8xJmhlFmNMd9+a7TUYY34s6URJOZJWNnTO8SJw+caDLN8IAACAaqJ1g+mNkhZJesIYc4ak1fKu5jJe3vaXO6udv9q3NQH7hkl61xizyHfNbkmt5V2ffbCkAklTrbWVUZpzzHVuma7tB4slSV+s26uebep8bhQAAACOIdFog6mqro+UNEPekH67pN6SnpA01lqb62CYZZL+LClF0hRJv5D0Q3lXkfmTpIHW2nnRmG+86NOuqf/ru//znax1vGAOAAAAjgFRW7rRWpsjaZrDc02IfVvlDfnHjDMHttfrS7b6v88vrQhqjQEAAMCxLSqVdURmfL92Qd/n0bcOAACAAIT1GOvXvpn/a9ZbBwAAQCDCeow1Tz/ciURlHQAAAIEI6zHWLKBHPY/KOgAAAAIQ1mOsedrhynp+CZV1AAAAHEZYj7GgyjptMAAAAAhAWI+xwJ71D77ZyVrrAAAA8COsx1hgZX3plgOavXpPDGcDAACAeEJYj7GMlMSg73/9zjcxmgkAAADiDWE9xiYd3yHoe9ZaBwAAQBXCeoy1b54W9H1ZpSdGMwEAAEC8IazHmVOOaxPrKQAAACBOENbjwO+mDPB/PX/9PlV6WBEGAAAAhPW40LVVk6Dvr3rpyxjNBAAAAPGEsB4H0pKDV4SZv36fDvGAJAAAgGMeYT0OpCbV/GMYef+nBHYAAIBjHGE9DlSvrEtSeaXV/R+sisFsAAAAEC8I63GgaWpSyP1vZW1zeSYAAACIJ4T1ONA6IyXWUwAAAEAcIqzHgcz05FhPAQAAAHGIsB4HEhJM2GMl5ZUuzgQAAADxhLAeJ/5wzsCQ+7/K3u/yTAAAABAvCOtx4qpxPbXi95Nq7N92oDgGswEAAEA8IKzHkcwmNXvXf/PuSuUWlMZgNgAAAIg1wnqc+culQ2vse2zWuhjMBAAAALFGWI8z5w3trLur9a//88utWp5zUMu2HpC1NkYzAwAAgNsI63GoRYh2mPOfWqgLn16kL9bvi8GMAAAAEAuE9TjUs03TsMdufDXLxZkAAAAglgjrcWhAx2ZhjxWWse46AADAsYKwHodSkxL1uykDwh6nbx0AAODYQFiPU1eM6R722PaD3rXXCe0AAABHt6RYTwChpSUn6syB7fXpqt01jt313rfafrBY63YXaOKAdnpu6kglJJgYzBIAAABHEpX1OPbTCX1C7p+zdq/W7S6QJH22eo9mhQj0AAAAaPwI63HshC4tdO95x2to1xa1nrd5X6Ekb3uMx0NrDAAAwNGCsB7nfjS2h967aZyuHtcz7DlNUhL14IerNe7hz3Xp3xfTyw4AAHCUIKw3ErecEbolRpK+3npAz32xSZL05eb9+m5HnlvTAgAAwBFEWG8kWjRJUfbDU0Iee2/5jqDvs3ML3ZgSAAAAjjDC+lFoX35prKcAAACAKCCsH4UKSitiPQUAAABEAWG9kencIr3Oc/IJ6wAAAEcFwnoj87crRiglqfY/tkLCOgAAwFGBsN7IDO6Sqa9+O1EvXz0q7Dnvf71Dg+7+RFNfWKKS8koXZwcAAIBoIqw3QplNkjWmV2t1a9Uk5PH80goVlFZo/vp9+seibHcnBwAAgKghrDdSKUkJ+vDWU/TOT06q9byHPlqjN5fmaOL0eZqzZo9LswMAAEA0ENYbsaapSRrRvWWd5/3q7W+0YU+1InegAAAgAElEQVSBps34SrvzSlyYGQAAAKKBsH4UOLlPG8fnvp217QjOBAAAANFEWD8K3D6pr0b1bOXo3Ec/WavSCm46BQAAaAwI60eBYd1a6s0fj1W7ZqlB+8Mt8Xj9y1luTAsAAAANRFg/ilQP5x/dekrI8+at26vVO/NU6bHafrDYjakBAAAgAoT1o8ikgR38X4/o3jLs0o6S9OXm/brwmUUa9/Dn+stn692YHgAAAOqJsH4U+dmZx2lIl0x1b91ED184WMmJCbrulJ4hz737P99pRc5BSdKfP1tHHzsAAEAcSor1BBA9zdOS9f7NJ8taK2OMJOnOKQPVo02G7vz3t7Vee+0/luqX3+unwZ0z/dcCAAAgtqisH4Wqh+3LR3fXlWO713rN/PX7dO5fF+qPH689klMDAABAPRDWjxGpyYmOzvvbvI1HeCYAAABwirB+jLj25NC966GUlHv71z0eq6wtB1RYWnGkpgUAAIBaENaPEe2ap+nPlwxxdG7/uz6Wx2N1z3+/00XPLNKUJ+arotJzhGcIAACA6gjrx5ALhnXR69eNdnTuT//1tf7xvy2SpOzcIn2Zvf9ITg0AAAAhENaPMSf1bqNffq9fnefN/GZn0PfllfZITQkAAABhRC2sG2O6GGNeNMbsMMaUGmOyjTGPG2NaOrw+wxhzuTHmdWPMGmNMoTEm3xiz1BhzuzEmJVpzPdbdNL6Plt11pu4593i9fq2zSrvHY5VbUKrHPlmr/6zYcYRnCAAAAClK66wbY3pLWiSpnaT3Ja2RNErSrZImG2PGWWtz6xjmFEmvStovaY6k9yS1knSOpMckXWiMOcNaWxKNOR/rWmWk6MqTekiSfn5mX03/dF2t57+dtU1z1+7xt8b0adtUAzs1P9LTBAAAOKZF66FIT8sb1G+x1j5ZtdMYM13SbZIekHRDHWPsknSFpLestWUBYzSTNFfSSZJukvSnKM0ZPj+d0EcT+rdTjzYZGnT3JyHPmbkyuC3m7CfmS5JevGqkJvRvf8TnCAAAcCxqcBuMMaaXpEmSsiU9Ve3w3ZIKJU01xmTUNo61drm19rXAoO7bn6/DAf30hs4XNRljNKhzppqmJml0z1b1uvbqGUuVW1Cq3Xl84AEAABBt0ehZn+DbzrLWBq3v5wvaCyU1kTSmAa9R7tuy4PcRdt/5g+p9zYj7P9PoB2fr/g9W6VBxed0XhFBaUanSisqIrgUAADhaRSOsVy0tEq7peb1v27cBr3G1b/uxk5ONMVmh/pHUvwFzOCb0bd9Mb98wNqJrn1+wWUPumaU7/72yXtdtyS3UuIc/1+gHZ2v97vyIXhsAAOBoFI2e9Uzf9lCY41X7W0QyuDHmZkmTJS2X9GIkY6B+RnRvqfvOO153vf9dRNe/tmSrJGntrnyVe6wGdmyuu88ZqLTkxJDn3/fBKu0r8HY/3fjaMn3689MimzgAAMBRJlo3mNbG+Lb1XqjbGHOhpMflvfn0Imutox4La+2IMONlSRpe33kca4wxmjq2h7YfLNHf5m2MaIyqwC5JK3IOqkPzNN068biQ567Ydvh93vo9BRG9HgAAwNEoGm0wVUkrM8zx5tXOc8QYc76kf0naI+l0a+2myKaHSJ07pFPQ90O6hPsjrtufP1unS5/7nzye4Pds89fv1d780hrnezxWb3y1VS8u2KyScnrZAQDAsSkalfW1vm24nvSqcmrtC3kHMMb8n6TX5a2oT7DWrq/jEhwBAzs11yMXnaCV2w/phtN7q3OLdPW4Y2bE4y3etF/Xv5Kle847XrkFperbvplue2NFyHPnrtujX7/j7X23kq45uWfErwsAANBYGWsb9hh53wORNsi7dGPvwBVhfGuk75S3gt/WWlvoYLzLJL0sabuk8dGsqBtjsoYPHz48KysrWkMecz74Zodufv3rI/oavdtmaOPe4H9Vsh+eoopKj+Zv2KfurZqoV9umR3QOAAAADTFixAgtW7ZsWbj2bKcaXFm31m40xsySd631myQ9GXD4HkkZkp4NDOrGmP6+a9cEjmWMuVLem0i3yBvUtzR0foiuswZ11JM/lJISjL53fAcVl1fq7axt6pCZph+/Ep03QdWDepXn5m/SIx+vVUpigr741XjtKyjV3+Zt1Ph+7XTRiC7K2V+kJimJat00NSrzAAAAiLVo3WB6o6RFkp4wxpwhabWk0ZLGy9v+cme181f7tlU3n8oYM17eoJ4gaY6kacaYapfpoLX28SjNGRFITDA6J6CXPSM1SVee1EOSNLJ7Sy3dcuCIvG55pUePfOztuCqr9GjMQ7PVtVW6cvYX64NvdqqovFJ3vfetUhITNOu2U9WjTehncL339XbNXLlT15/aSyf2qN8DoAAAANzW4DYY/0DGdJV0r7zLLLaWt/3lPUn3WGv3VzvXSpK1NjCsXyXppTpeZou1tkcD5kgbzBH07fZD+v6TC2I9DUnSwjsm6K2lOSqv9OiG03qrWVqyDhSWadh9n/rPWXf/Wdq6v1C92zZViDeGR1xZhUcea8MuaQkAABqvuGmDqWKtzZE0zeG5NZKRtXaGpBnRmg/cN6hzpt75yUm66JlFsZ6Kxj38uf/rxZv2652fnKTs3OD2msF/+ESlFR5dd0pP3TlloCRpb36p3lyaoxHdW2pMr9ZHbH67DpXonL8uUHFZpV67drSGdI3oMQQAAOAoF42lGwG/Ed1b6sNbTlHPMG0oktSheZqLM5KythzQ3+ZtrBHWSyu890L/ff5m/777PlilRz9Zqytf/FL7C70Paiouq5THY2Wt1b6CUlV6Gv5p1GOz1mpvfqkKSis0bcZX/v2b9xVq2dYDitYnXgAAoHFz46FIOMYM7NRcn99+mv786To98fkG//47zuqvFunJGtenjcY/NlcVAaH3gmGd9e+vtx+xOT380Zpaj0+ftVY/n9RP/1mxQ5I3yM9evVsdM9N1/StLZSQVlh1e7331vZPlsVYZqTX/E8rZXyRJ6tqqSdjXm7t2r//rqjcFm/YW6Izp82St9OdLhuiCYV0c/3wAAODoRFjHEWGMUWaTlKB9N5zW2//1R7eeovnr92ntrnyN799Wkwd11K1nHKer//GVNoVZDeZIeuLzDUFvLCTpl29/E/b8Ab//WMZI14zrqd99f6B/f9aWA7r4b4tkrXT56G76wciuNVpc7v3vKu0rqPkgqAc/XK2qgvptb6zQBcO6aO2ufN32xnJ1bpmupy8fruTExv1h2P825mpPfonOGtRRKUmN+2cBAMAN/G2JI+ayUd3UxreM4t3nDAw6dlz7Zrr65J7648UnaPKgjpKkHm0y9OEtp6h32wz/Nc//aKS7k64Ha6XnF2zWO1nbtDuvRLe9sVwXPbPIH7hfW7JVF/9tkXYdKvFf4/FYvbhwc42xnp23UdsPltTYf+u/vtaqnXn6dNVuPfLxGq3dla+bXlumVxZvUWFphe7/YJUe+XiNyis9Na6tsmlvgV5YsFk7DhaHPF7psVq0YZ9259V8/Wj6bsch/fDvi3Xrv5brlcWsygoAgBNRWw2mMWA1GPcVlVVox8ES9Wnn/CFGFZUe7S0oVcfMdElS3999pLKK8GE00MYHz9Yv3lpxRFtqIjFxQHst2ZSr/NIKx9dkPzylxhNjkxONyiu9/81eNLyL3lm2zX9sTK9W+nZ7nn44qqt+NLaHurZqokqP1fjH5mrr/iIN6dpCT146TIs27tOZA9v716Of/uk6PTF7vZqlJmnhbyaoeVpyFH7imm56fZlmfrMz6OfzeKwe+mi1tu4v0u+mDKy1daixKiyt0MHicnVukR7rqQAAXBR3q8EAoTRJSapXUJekpMQEf1CXpDMHtg8KebVJTDB69OITNKZXK81cuUtfrNtb90Uu+Gz17npfUz2oS/IHdUlBQV3yrnojeW+Y/fv8zbrjrP46d0gnbfX10K/IOahTH50jSXpv+Xb96/qxkqQnZq+XJOWXVujNr3J07tBOuu+D1UpONHp/+Q5Veqy6tWqiz28/TUlh2nC2HyxWaXll2CfL/nfFjpB/hh9/t8t/g++BonK9+eOxyi8p1zNzN6pVRoquOblnjWU1c/YXqdJjw66lH08OFZXr9Mfm6EBRuf5y6VCdN7RzrKcEAGhkCOuIe3efM1AbdheowuPRjGmj1Lppit78Kkd/+O+qkOcnJSbokhO76ZITu+nBD1fruS82hR3bGKl9szTtOsItILHw8EdrlBuiN17yBvvsfYU1Am9JeaX++NFa/dd3o22VrfuL1OfOj9SnXVP9+NRe+r+RXSVJ1lot23rQ36d/9zkDdf7QzmqZkaLcglK9t3yHRnZvqXs/qPlnddPry/w340rSl5u9bzb+Nm+jnp67UZK0akeefjtlgN74KkcDOzVXyyYpOv+phZKkN64fo9G9WqvSY7Ult1A922T4g/2h4nL9b+M+je3dRpnpNT8pKKvw6PM1u9WjTYb6d2he4/ievBKlpSQ2+FOGGYuydaCoXJJ067+W+8N6pceqoLQiaG7W2pis9x9t1lo9+OFqrdqZp7u+PzDk7xcA4BxtMGgUqv49DQwzW3ILddPry/Tt9jz/vuyHp9S47vpXsvTpqsOV7WtO7qkbT++tVxZv0cCOzTWhfzs9+fkG/cVXYT6W9G6boY0R3NA7Y9qJOrFHK13+/BItzzkYdCwjJVFf/Gq87nr/W324cle9xg3V+lMlwUjVV828Ykw3vbp4qyTpwmGdNf2SoZKki55ZpCzf03R/NvE4vb5kq35yem9NG9dTkvTk7PX606frlJxo9MWvxqtN01S9k7VNCQlGvwq4sfg/N4/Tqh15GtenTb1bdDbsKdDE6fOC9q2+d7KMkc76y3xtO1Ckxy8ZpikndNTW3CJd+/JXSk9O1Ixpo9QyI/jm7H0FpSopr1SXlvHfJrRowz5d9vwSSVLnFulaeMeEGM/IHW8tzdGyrQd0w2m91b11/H/qA+DIi1YbDGEdjdpNry3TzJXBfdDVWWt1qLhcLaqtThPKjIWbw1bsceS9es1oXfHCkoiv7966iWZMG6Xxj80NeXzNfZOVlpwY9IbgulN6qnfbprrj3ZVhx+3cIl1zf3m6fzWewtIKXfuPpTpQVKZHLx6iAR2bKSkxQdZardtdoK6t0nXdy0u1cENu0DipSQk6oUumvso+4N+X/fAUTX1hieav3yfJ+wbq45+dqrlr96prq3QlJyZo8uNfqLzS6vkfjdTEge21aMM+/fOrHP3fiC46tW9bSdKC9fs0Y1G2LhreWa2bpurNpTm6cHhnndS7jSRp494C/eadlercMl2PXnxCUEvTZ6t261fvfKNzTuioG8f30by1ezWqZ6t6txpZa3X1jK80J2Bp0qr/Jt9cmqOvtx7QT07ro26tm8jjsZq5cqespO8P7qiEhMg+VSgqq1CTlNh+SJyzv0inPOJtMRvSJVPv33xyTOfjlg178rVsy0F9b1CHkJ9gAcc6wnoECOtHn625RTr9sTnyWOnBCwbrstHdGjTeoeJynfzHz5VfUqGpY7rr7nMGqrC0Uje8mqXySo/OHdpJc9fu1edr9ig50ejMge0dVY9/OKqb/vnl1qB9p/ZtGzc99ceS0/u1DVrnPiUpwdENzK9eM1pje7fWoeJyXfXSl/pm2yH/sU6Zafrw1lP02pKtevSTtfWaT6hPEzpmpmnnoRKlJCWoXbNUbTsQeiUfSXp26gh97/gOIT+RaNkkWUt+O1EpSQk6768LtMI35z9eNFiTB3XUb99dGfRmN1Cbpila8OsJSktO9O/beahYe/NLNbhzZsiWnSdmr9f0T9cF7bvm5J46tW9bXfnil5KkUT1b6c0fj9XH3+7SDa9m+edzyYmH/9utqPTorve/0978Ut173vEqKa9UzoFiHSgs09jerdXe92C1Z+Zu1GOz1mrK4I760w+G6NvthzSgY3P/nD0eq0Ubc9UyI1nHd8oMmpfHY/XJd7vksdLkQR2UGOGbhbyScp3314XavO/wJ1Sr7v2emqQkadPeAq3YdlBnDuygpqlJ8nisXl2yRUVllbrqpB5Bv9v6qPTYiOcbLYWlFRrz0Gzll1TogmGd9Wffp1pHu5nf7NSnq3bp2lN6aVDnzLovwDGNsB4BwvrRafO+Qu3OK9Honq2i0vO7eV+hvttxSBMHtA/7l+mBwjKlpyQqLTlRM7/ZqZteXxZ2vN5tM/Tfn54sI6PrX1mqpdkH9Mvv9dPVJ/fUZ6t26453Vwatu/7TCX30ZLU132vz2c9P1fKcQ/rFWyuc/5CISN/2TZWzv1jF5ZV1n+zQ9af2qvW+CieGdMn0B/HqLhzWWfecd7wG/2GWf9/EAe01qHNzPf5Z7a1f/7l5nE7o4n1OwNbcIp0xfa7KK60GdW6ua07uqfOHdtbW/UX648dr1L9D8xpBPZzsh6dowF0fB/0ebx7fR29l5ahzi3SdNaijHvhwddjrP7/9NPVq2zToDUqP1k2UnVukIV0y9e8bxykhweitpTn+5yV8dOspGtCxub7cvF95xeW69uWl/mvPH9pJX27er9P6tdVDF54Q9Fob9hQoe1+hTu/XNuQN1n/4z3easSi7xv5ffq+fnp23UXklFbpkZFf98eITgt6g/G7KAF17Si//+dZaPT9/s7JzC/WziX21O69Er/xvi7K2HtB5Qzrpp2ccJ8nbbnPPf1fpzIHtNf0HQ7TtQLE6tUj3h/e9+aW6451vlJaSqEcuOiHowW3llR798aM1qvBY/XxSX+UWlKljZlq93zTsOlSis/7yhf9+DElaeMcEdW6RrkUb9mnuur26bFQ3/ycza3bl6UBhuUb3bOXoExSPxwadZ63V8pyD6t2u6RFbrcqJ/YVlGn7fp5KkVhkpWnbXmf5j1e85qaj0qKi8MqbzjZYXFmzW21nbdPP4PppyQsdYT6dRIaxHgLCOIyWvpFx/+Wy9XlgQvIb6hgfOqvEXfKiq2IY9+Vq8ab+mDO6olhkpOuWRz5WzP3w1NVD11p/b31xRY6UYIFIf/+wU/fXzDfogxGo+7Zunande6JuYa7P4N2dozEOzGzSv4d1aaNnWgyGPVd18HBjmJw5op2tP6aVLn1tc67gf/PRkf8V0d16JTv7j5/5VmEZ0b6m//2ikissr9eDM1WqenlzjE7Nwsh+eorEPzdbOgOcuXDGmmxZtyFWXVk105djuuuYfS8NeP2lgez1+6VAN/P0n/n3N05KUV1KhYd1a6J0bTlJCgtG5f13g/9Tntol9devE47Rpb4E+W71bD354+EnOxnifFdG5Rbpm335aUGCfsXCzlucc1M0T+qhrqyZKTQoO8794a4Xezqr5/5jLR3fTW0u3qazSo+HdWujdG8dp3e58TfrzF5Kk28/s63/TIXkD7v0zV2v1zjz9/pyB+mrzfj02a50OFZdreLcWevma0WqamqRHP1mjp+Z4bzp//drR+uucDRrfr52uO9X7ZudQUblue3O5JOlP/zck6J6PtbvydcULSzS8Wwv9dMJxemHBZg3s2Nx/baCS8sqwb1yW5xz039xeZd39ZyklKUH3f7BKby/bptsn9dPUMd2VV1Kusx6fr70FpXr2ihEa379d0HWb9hb4HgjYTqlJCSqt8Gjtrnz1bJvhD/dVN+/3bJOhVhmhWzhLyitVVukJ+YbgUHG5cgtKw67Q5dSqHXk6+4n5/u+r/r7ZV1CqhRv26dTj2vp/38u2HtDGPQX6/gmdlJ4S2adGDVFSXqn9hWXqFEfL5BLWI0BYx5EWWDmTQvfQO/H8/E26f6a3sti5RbpO7tNGa3blhayeVn+N3XklGv1gcBB65OITtG5Xvp5fsFmJCUZ/OGeg7nr/u4jm5lTXVumO33AA8aJ/h2b6z80n682lOfrde99GbdzV907WgN9/HLXxqvvLpUM1oX+7oE9QOrdI10vTTvSH5XCe+OEwnTukk6SaobRVRoreuH6MWmWk6IGZq5WanKB/fpnjaE7ZD0/RVS99GdR2NnVMd32745CSExN0wbDO+k0t94pI0l8vG6abX/865LEvfjle3Vo30XlPLdQK343u08b10N3nHK8v1u3VL99eEfbN5Ps3jfM/Xdpaqx+9+KUWbvDeN3LGgPZ64PxBatssVd9uz1OT1ETd9NoyrdmVX2Oc6p+MZT88RY98vMa/opUkTf/BELVokiyPRxrevaVOfWSOCkq97UOb9hX65y5JD184WBeN6KJn523UY7O8n1Q9c/lwvbJ4i0b3bK1bJ3rf7OTsL9Jpj85RWnKi/nndGP/PUumxejsrR79+x/t7/dXkflq3K1/dW2foZxOPq/Hpc0WlJ+ySvF+s26sf+drXqpw/tJOuPKmHfvPuSq3Zla/RPVvpjR+P1bYDRTrt0bmq9K0C8OaPx2pUz1b+6+7890rNW7dXd59zvI5r11QvLNisj77dpZHdW+rpy4crIcEoa8t+/eyN5TquXTM9N3VEyHkt23pA+wvKNL5/u6Ci1+tLtuq3//b+zI9efILOG9pZSQkm4nthooWwHgHCOo60So/VuX9doNU783TPucdr6tgeEY1TVuHR/TNXKa+4XL/7/kC1aZqq4rJK3fHuN3p/efCyiqHeELyyeIteW7xFV4/rqR+c2NU/5uzVu9W1VRMN6pwpj8dqZ16JlmzK1c/fPNxCc/c5A3VS7zY658kFKqv0KCMlUYVllerVNkO3nnGcnp6zUWt31/xLq7rPfn6a9uaX6od/91Yyh3ZtoZeuOlH3frCqxkOr2jZL1d585xXaTQ+erXnr9mrajK8cXwMgvNevG6284nLd8Gr4lr76+uznp9VYESma+ndopmnjeviDqeS9B+Xdn5yk7z+5oNZrh3VroX/fOE55JeW677+r9Fa1TwrOG9pJk4/voJ+8Vr/fx9r7J+vcJxeG/X/ksG4t9HWYT4SqTB3TPexTnqvauQI/NerfoZnev3mcLvv7Ev8qWKG8ft1o/w3nkvcJ2Z+u2q2ebTJ0at+2uuHU3mqenqRPvtulxIQE3ffBKv9zOgJVfSpTZf0DZ+mhD9fUeDr3hcO8S9V+f0hHXT0j/KdG/do306vXjtZpj85RUZm3Na532wy1zkjVoM6Zuuv7A2SM0X9W7NAt//S+cXv4wsG6dFQ3lVZUavbqPbqx2p9T87QktW6aqvduHKfMJrFrRSKsR4CwDjd4PFYHisr8TwiNtqplB6tEWr0P9M8vt2rOmj26eUIff49ybkGpisoq1bVVExWWVvh7X621evjjNXp2Xug+6/vOH6Tzh3ZSM99HszsOFuuLdXs1oX87tWuepvJKj5ZmH1BxeYXKKqzOGNDOW2V7emGdf4mF+pkrKj36dkdejY+ojzaXjOyqN5Y6q2gCqNtDFw7Wv7/e7n/GQ2Nx4fDOendZcMHj15P7648frwlzhVdyotH6B84O27Y5cUA7dW+dUWN/XV6adqJ++dY3QfdeBQq17G59vHbtaJ3Uu7V6/ubDoP2r7v2ezpz+hbYfDP8J7nWn9NSdUwZG/uINRFiPAGEdR4Oisgqd/Zf52nmoRH+9bLjOHNg+JvOw1mrhhlzNXbtHzwf8zz3SNw9vfLU1qEImeZc6bJ6eXKPqXv01PB6rsQ/P9n/kfVLv1vrV5P4qKq3QNf9Yqsz0ZM285WQ9M3ejnl+wWSmJCSqrPLwCzPBuLfTj03rrpteWqcJj9ZPTe8taaX9hqZbnHNS63QWOfoa/XTFcy3MO6W/zDn8EPucXp+uhD1dr1qrgp9i+NO1ETXvJ+ScDGx88WweKynTDK1lauuWAhnRtEfTx+dGoXbNU7anHJy4Aate5Rbo6ZqZpaS0V+Maif4dmIVuTqvv5mX118/g+MWmJIaxHgLCOo0Wlx6qorMJfvY6lDXvyNXH64Z7YSMN6pcfqvg9WaU9+iX5z1gAdKCpTvw7NlJqUqIc+Wh1UyQ/1Gos35er1JVt1wfDOGt/v8A1dBaUVSktKqNH/+Jt3V+o/y7frjrMHaOqY7pKkbQeKlFtQphO6ZNZ4ANcv3loRtD56oMnHd9DlY7rp5D5tZIzR52t26/UlW3Xpid00cWB7HSwq00ff7tKyLQf07Y483Xh6b50zpJNueCVLH3/n7MFR1X/mkvJK9b8ruAd6YMfmGt+/rf9mvOemjtDna/boX1+Frsj/5PTeesbXW9sqI0W92mTU+y/x357dX+9kbQ/62P+1a0dr5sqden1J8M2X9Q3fC++YoOx9hbrc95ClC4Z1rtFCBQB1+cHILnrk4iGuvy5hPQKEdeDImD5rrT5dvUe/ntxPp/drV/cF9ZSzv0inP+a9eWnqmO667/xBURm3vutVF5RW6L7/rtKyrQe0fs/hanukb1AClxasMqpnK505oH2NpQtDvcYDM1fp7/M3q1VGij752alq28x7b8OMRdlqnp6ky0Z1kzFGX2Xv14crd+qlhdn+azc9eLYSEoxeW7JFG/cU6ien91bbZt7WrUNF5Xp+waZalxDt36GZrhjTXZee2FVllR5d/Mz/tGpnnk7q3VqvXTtapRUeLVi/T0/N3aCvtx7UZaO76cELBuvOf6/Ua0ucraCy+t7JQatKWGtrfBTetlmq2jdP9T/J+HvHt9fiTft1qLhcoTRNTVJBaYWj1w/noQsH69l5G5Wde7ifd+YtJ2t3XkmN3txJA9vX+FSlNrNuO1X78kt1+QtLZK30wpUja10l5mjw6MUn6I53V/pvTgSOhCd/OEzn+G6kdgthPQKEdaDxmrdur77bcUg/PLFb0NJssbAi56DOC+iRjzSsezxWd7z7jTbvK9T95w+WMVKP1hlKSUrQO1nbdHvA2vnhXmNvfqnaNE1x9IyB5+dv0j+/3KqbJ/TRBcO61Hl+YWmF/vVVju77oOZTfUMF6Q17CtSrbdOgN0AeT9UThJNl/r+9e4+uojz3OP59kpCEW4BwkYtC5BqLSgVBTVsgCAWserTFUj0qp19xGNcAABIHSURBVJe11FZb6oVaW4/aVnu8tLXV03JOPdWu2lYt1lLrBatcRBFBkSJCgCAhCiExQUkgBHJ5zx8ziXvHvWHvJDt7kvw+a82atWfPOzN77ydvnpl5533N+Pk/t/Orl8L7dx+clcnIgT1ZszN8xNdIn/nu5wv4zcqdZGWm8fKifPr2SGfvR4e56g9v0i3V+O2VZ9I7sxvLC0pZt+vDsIfe3vjhTAb0ymB5QSnv7Knk388e0dQtXmVNLQ8uLzxmn/d5o/pz7YzR5I0aQMmBw/zH79azrbSKWZ86gf+9YhJmRn2D47xfrmZbaRU5/Xuw/Ibp3LRkU8zdqW64dRbZPdOpb3AYkJJiXLJ4TdhdnV/Mn8DmPZVNbYsvnXISk0ZkRx1r4ayTs3ndb5fdLdWauqKMx/fm5FK8/1BYTzAPXnYGDY6mh/4afeH0ITwToavPaP5x3WfJ7pnOoiWbyO6Zzj3zTmf2/S+zu+KTDzd2FvfMO50fPb2l1SeOHUljV6PJ3P+m22e36z6VrLeAknURaQuHjtQx/javv+sR/Xuw6qb8Nt9HTW090+9dyb7KGhbOHMPCmWPbfB+xcM7xwpZSCssOho3Ouuun57VoELKPqo/y2bu9rusWzRnH3FOHMLRvJhlpqawoKAvr4SfaCcp7+6vDBgI6luUFpfx53XtcdtbwsOZR0Wwo/pDV28t5pfAD3tz9YdiDcet+cC6DemeGrV9VU/uJ5millTW8tLWM/NyBDOnTnaUb9/CdxzY2vZ/ZLYX8cYOYOnYgr+2s4O//+riHp8Y7HqG27K3k/he3U1PXwMKZY5g4vB9VNbUsWrKJ2nrHPfNOJ7tnOlU1tWwo/ogbnthI+cGjDOqdwZqbZ5CWmsJ7+6vZWlLJtHEDSU9Nwcw4dKSOh1/d1dRFYCQPXXkm/Xqmc8ZJfUlJMR5a/S73PL+NqWMH8Nsrz2yKgSv+73VW7yhnhH+C8tRbe2IeqG3t989lcJ/w7zW05w+AaWMHMn3cQO542jtxvGb6KL5z7hgWLdnEsnf2caTZKMQvXj+VhY9vZPMe727Pno8Ox5X8pxj85epzWPZOadMJXFqKseLG6dQ3OL792FthoxivuHE6FzzwSszJ91+/mceoAb24delmzLy7NV99eH3TSVUk3bulMn5oVkLbm889dTDPbQ5vmnfDrLEUVVS3evyOxZdPxDmaetgZPzSLd/ZWtmqb8WqLDhnioWS9BZSsi0hbWbOznJe2lnHplOGMHtS6gUeiOXikjqLyQ4wfmtUmo/O2RlVNbVgf3q35p1dWVUNxRTWTRvQL+1xH6uqZdo93gnLplOH89IunteqYW+vw0fqwvtHfuWN22Iigsaqrb+DiX69ha0kld13s9aPdeKKxu+IQ0+5d2bRuWyQTW/ZW8uSG9znvtCFMGtHvuOs3NDj2HjjMKzvK2fnBQX67+uO7EStvnN40EmmjSIMHVdXU8mphOZNzsunfy+uKdfKdLwIwtE8mE0f04+yR/Tl7ZH+2llRyXUgivuPOuXSL0Kd2XX0DRRXVZGWmMSgrk/oGx69XFFJ1pI7rZoxuOkmqq2/grxv2cN8L27h44jC+P/cUwGvmtq+yhmHNBskpq6rh2j+9FbUXmMZBtRoVlh3k8fXFnJmTzezxg5uWP795H/cuK2D2+MEsmpPLjtIqZkXo0757t9RPjHq8elE+J2X3CFtWsK+Sax7dwC5/xNwHL5uIc455v3mNikNHeGjBZD59Ul9e3FLKH9buZtX2j/uv/+b0USyak8vSjXtY9s4+Lp0ynE3vHyAjLYWMbqksefP9Yz6QPmpgT7406US+Mnk4j67d3TQa8YUThnL//E9z6Ggddz9fwKNrvSZsg3pnsOqmfP60rjjinbdInrwmj4nD+7Jq+wekmPG5MQO485mtYR0UAHx/bi4/fc7r2WbhzDFkpKVG7elm7qmD2bavinfLDwHew6THGkn5rVtnteudWSXrLaBkXUSk5Rav2smf1xXzrfzRfPnMkxKyj4qDR9i8t5JzRvYnPS3yYC3tpb7BMeqWj9vIR7rqHSvnHJU1dfTp/smHwm956m3+vnEvN8/N5XL/YedkCu3De83NM1o8IuSq7R+wZmc5l581IiwxDT0JSksxCu86r3UH3ALOOUoO1FBZU0th2cGwgZeevCYvppOcSL63ZBPPvF3Ct/JHk5ZijB+a1dRz02X+g9IABT+eE3W01EjHWt/gPvGQ/EtbS7n7ee9k4YbPjzvudp544z0eebWILSWV9EhPberTHMIHxoLoo7k2NDhe37Wf3MG96dcznQOHa5lwxwth6wzolc7MU07gmbdLqApp9hLpBOVoXQOPry8mo1sq08cNZGCvDMyMNYXlFFVUc/EZw+ienkphWRWPri1mxbaypjskL14/ldGDerPvQA1rdpYzI3cQfXt4iXh9g2NrSSU/eOrtpsEE/3L1OUw4sW+71itK1ltAybqIiMTj3mUFPLR6F1dNHcn1MSRELdXQ4JI+2mKj0GQ9nqQyHv/cUsrfNu7hyrNHhF3FTpbQz/z0tZ/ltBP7tHhbkR5cd84xb/FrvLn7w6ZRP5Op+Yno4ssnMufUIS3a1uPri3nyzT1cNW0kYwb1pn+v9KY7UKfetqypadC2n8whI611sVRTW88rO8oZPyyLIX1iO4k8UF2btIGR2ipZj/9+noiISBdx0+xcvjtzbNQh2dtKUBJ18K5A/nHtbi46Y1hCEnWAWZ86IWljRERybu4gXiooY1jf7pwypHerthXpWQoz4/dfm8K6XRVMOTn5JyfNj7ElzbsazZ88nPmTh0d87+GvTuZPrxdz0RnDWp2oA2R2S2VmnHGTzBFM24qSdRERkWNIdKIeNJNzspmck53sw2hX910ygRe27CNv1ICE/d69MtKYkRucE5QF54zg96/tJqd/D/JGDUjIPrpiLCWCknURERHp0vr1TI96dbizuu2C8Vz46aHkDs6Ka7wJaX9K1kVERES6mJQUY9IIXfXuCLrWvT0RERERkQ5EybqIiIiISEApWRcRERERCSgl6yIiIiIiAaVkXUREREQkoJSsi4iIiIgElJJ1EREREZGAUrIuIiIiIhJQStZFRERERAJKybqIiIiISEApWRcRERERCSgl6yIiIiIiAaVkXUREREQkoJSsi4iIiIgElJJ1EREREZGAUrIuIiIiIhJQ5pxL9jG0GzOr6N69e/Ypp5yS7EMRERERkU5s69atHD58eL9zrn9rttPVkvVdQBZQlITd5/rzgiTsWzofxZO0JcWTtCXFk7SljhxPOUClc+7k1mykSyXryWRmbwI45yYl+1ik41M8SVtSPElbUjxJW1I8qc26iIiIiEhgKVkXEREREQkoJesiIiIiIgGlZF1EREREJKCUrIuIiIiIBJR6gxERERERCShdWRcRERERCSgl6yIiIiIiAaVkXUREREQkoJSsi4iIiIgElJJ1EREREZGAUrIuIiIiIhJQStZFRERERAJKyXqCmdmJZvY7M9trZkfMrMjM7jezfsk+NkkePw5clGlflDJ5Zvasme03s2oz22RmC80s9Rj7Od/MVprZATM7aGavm9mCxH0ySSQzm2dmD5jZajOr9OPl0eOUaZe4MbMFZrbOX/+AX/78ln5WSbx44snMco5RZzkze+wY+4krNsws1Y/RTWZ22I/dZ80sry0+t7Q9M+tvZt8ws6fMrND/3Q6Y2Stm9nUzi5hvqn6KjQZFSiAzGwWsAQYBS4ECYAqQD2wDPuOcq0jeEUqymFkR0Be4P8LbB51z9zVb/9+AJ4Ea4HFgP3ABMA5Y4py7JMI+rgUeACr8MkeBecCJwM+ccze21eeR9mFmG4EJwEHgfSAX+KNz7vIo67dL3JjZfcAN/jEtAdKBrwDZwHXOuQdb/qklUeKJJzPLAXYB/wL+FmFzm51zSyKUiys2zMyAJ/BibhvwtL/ufCAT+JJzbmn8n1YSycyuBn4DlAArgGLgBOCLQB+8eugSF5J0qn6Kg3NOU4ImYBng8IIhdPnP/eWLk32MmpIWG0VAUYzrZgFlwBHgzJDlmXgngw74SrMyOXgVYAWQE7K8H1Dolzkn2d+DprjjJh8YAxgw3f8dH01m3AB5/vJCoF+zbVX428tpzefWFIh4yvHffySO7ccdG8ClfplXgcyQ5ZP9WC4Deif7u9P0id96Bl6indJs+WC8xN3hnWg1Llf9FMekZjAJYmYjgc/jJWX/3ezt24BDwBVm1rOdD006nnnAQOAx59wbjQudczXAD/2X1zQr8zUgA3jQOVcUUuZD4C7/5dWJOmBJDOfcCufcDuf/tzmO9oqbxtd3+us1linCq/sygK/GcLzSzuKMp5ZoSWw0xuQP/VhtLLMe70rqQLzYlgBxzi13zj3tnGtotnwfsNh/OT3kLdVPcVCynjgz/PkLEYK3Cu+qQQ/g7PY+MAmMDDO73MxuMbPvmFl+lHZ6jbH0fIT3XgaqgTwzy4ixzHPN1pHOqb3iRrHWtQw1s6v8eusqMzv9GOvGFRt+LObhxebqWMpIh1Drz+tClql+ioOS9cQZ58+3R3l/hz8f2w7HIsE0GPgDcCde2/XlwA4zm9Zsvaix5Jyrw2tHmgaMjLFMCd6dnRPNrEdrPoAEWsLjxr8zOAzvOYuSCMegeq7zmYV3pfROf/4vM1thZsNDV2phbIwGUoF3/RiNpYwEmJmlAVf6L0MTZtVPcVCynjh9/PmBKO83Lu/bDsciwfMwcC5ewt4TOA34H7x2dM+Z2YSQdVsSS7GW6RPlfen42iNuVM91HdXAj4FJeG2E+wHT8B4mnA681KxZZyLjT/HUcfwXcCrwrHNuWchy1U9xULKePObP1R1PF+Scu8Nv41fqnKt2zm12zl2N9/Bxd+D2ODbXklhS/El7xo3irINzzpU55/7TObfBOfeRP72M92zW63hXxb/Rkk3Hsa7qrQ7EzL6N1wtLAXBFvMX9ueonlKwn0vGuXGY1W08EPn4QZ2rIspbEUqxlKuM6OulI2iNujrf+8a5sSQfnN1l4yH8ZT70VKTb0f7OTMLNvAb8EtgD5zrn9zVZR/RQHJeuJs82fR2sLNcafR2vTLl1TmT8PvZ0cNZb89oAn4z24826MZYb423/fOVfd2gOWwEp43DjnDgF7gF7++82pnusaPvDnTfVWC2OjEKgHRvoxGksZCRgzWwg8CGzGS9QjDfSn+ikOStYTZ4U//3zzkbvMrDfwGeAwsLa9D0wC7Rx/HlpBLffncyKsPxWvV6E1zrkjMZaZ22wd6ZzaK24Ua9LYq9m7zZbHFRt+LK7Bi83PxVJGgsXMvgf8AtiIl6iXRVlV9VM8kt3Re2ee0KBImiLHxXggO8LyEXhPpzvglpDlWXhXruIZPOJkNChSp56IbVCkhMcNnWTQka4+xRBPZwHpEZbP8H9jB+S1NjaIbVCkrGR/X5oixsit/m/3RqT/cc3WVf0Ux2T+QUsCmNkovKAbBCwFtuJVePl4t13ynHMVyTtCSQYzux24Ge/uyy6gChgFfAGvonoWuNg5dzSkzEV4wyTXAI/hDct8If6wzMCXXbM/ZjO7DvgVcQzLLMHmx8FF/svBwGy8q5mNfVKXh/6u7RU3ZvYz4HrCh/OeD/Snowzn3QXFE09mthLvQsNKvN8Z4HQ+7qP6VufcTyLsI67YMDMDnsCLuQLgaX/d+Xj145ecc0tb8bElAcxsAfAIXjOmB4jcDrzIOfdISBnVT7FK9tlCZ5+Ak/C66SvBC6rdeA9dHPOsU1PnnfC6O/sz3j+ij/AGjPgA+Cdef7QWpdxn8BL5D/GaUL0NfBdIPca+LgBW4Z0QHALWAwuS/R1oanHs3I53lSjaVJSsuAEW+Osd8sutAs5P9nemqW3iCfg68A+8UbkP4l0RLcZLmD7XlrGB17/2d/1YPezH7rM0u3KvKThTDLHkgJURyql+imHSlXURERERkYDSA6YiIiIiIgGlZF1EREREJKCUrIuIiIiIBJSSdRERERGRgFKyLiIiIiISUErWRUREREQCSsm6iIiIiEhAKVkXEREREQkoJesiIiIiIgGlZF1EREREJKCUrIuIiIiIBJSSdRERERGRgFKyLiIiIiISUErWRUREREQCSsm6iIiIiEhAKVkXEREREQkoJesiIiIiIgH1//M4zKqR7QK3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f9f0a7e6fd0>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 373
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "epoch_count = 10\n",
    "print_every = 100\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3, weight_decay=1e-5)\n",
    "scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.99)\n",
    "loss_function = nn.CrossEntropyLoss() # composes nll_loss(log_softmax(x))\n",
    "hidden = model.create_hidden(batch_size)\n",
    "\n",
    "model = model.to(gpu)\n",
    "hidden = hidden.to(gpu)\n",
    "batches = batches.to(gpu)\n",
    "\n",
    "start = time.time()\n",
    "all_losses = []\n",
    "\n",
    "try:\n",
    "    for epoch in range(1, epoch_count + 1):\n",
    "        \n",
    "        for batch, batch_tensor in enumerate(batches):            \n",
    "            # reset the model\n",
    "            model.zero_grad()\n",
    "                \n",
    "            # everything except the last\n",
    "            # size: seq_length, batch_size, cols\n",
    "            input_tensor = batch_tensor[:-1]\n",
    "                \n",
    "            # prediction and calculate loss\n",
    "            output, _ = model(input_tensor, hidden)\n",
    "            \n",
    "            # everything except the first, flattened\n",
    "            # size: seq_length, batch_size, cols\n",
    "            target_tensor = batch_tensor[1:]\n",
    "            \n",
    "            loss = loss_function(output, target_tensor.view(-1))\n",
    "            \n",
    "            # gradient clip betwen loss calculation and backprop\n",
    "            clip_grad_norm_(model.parameters(), 0.5)\n",
    "            \n",
    "            # backprop and optimize\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            \n",
    "            loss = loss.item()\n",
    "            all_losses.append(loss)\n",
    "\n",
    "            if print_every > 0 and batch % print_every == 0:\n",
    "                batch_count = len(batches)\n",
    "                total_batch_count = ((batch_count * (epoch - 1)) + batch)\n",
    "                rate = (batch_size * total_batch_count) / (time.time() - start)\n",
    "                for temperature in np.linspace(0.1, 1.0, 10):\n",
    "                    show_prediction(temperature=temperature)\n",
    "                duration = time_since(start)\n",
    "                print(\\\n",
    "f\"\"\"\n",
    "Speed: {rate:.2f}/s ({total_batch_count} in {duration})\n",
    "Epoch: {epoch}/{epoch_count}\n",
    "Batch: {batch}/{batch_count}\n",
    "Loss: {loss:.2f}\n",
    "\"\"\")\n",
    "                plt.plot(all_losses)\n",
    "                plt.show()\n",
    "                clear_output(wait=True)\n",
    "        \n",
    "        scheduler.step()\n",
    "                \n",
    "except KeyboardInterrupt:\n",
    "    pass"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}