Pong-Playing TensorFlow Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pong-Playing TensorFlow Neural Network\n",
    "\n",
    "## Import modules needed to train neural network in Pong environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from IPython import display\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "\n",
    "config = tf.ConfigProto()\n",
    "#config.gpu_options.allow_growth = True\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Investigate the environment and set up data preprocessing functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-11-15 18:11:43,569] Making new env: Pong-v0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Discrete(6)\n",
      "['NOOP', 'FIRE', 'RIGHT', 'LEFT', 'RIGHTFIRE', 'LEFTFIRE']\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsUAAAJNCAYAAADZKAitAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xn8HVV9//HXmwRIgLCE0BgSIGARBQtBI6ACgoAspQT7\na2kQaRBqxKIFxCrBVrGKUqqiVkWDYqKsYROKimwixbIYICIhIAECSchCwpIAAQ18fn+cc2Fyc+93\nvVu+834+HvdxZ85sZ+bOmfnMmTNzFRGYmZmZmZXZeu3OgJmZmZlZuzkoNjMzM7PSc1BsZmZmZqXn\noNjMzMzMSs9BsZmZmZmVnoNiMzMzMyu9AREUSzpT0oXtzkdPSdpW0guSBrU7L2ZmZma2jgTFOYCs\nfF6TtKrQf0yTlnmmpJC0Z1X6cZJu78+8I+LJiNgkIl6ts+x5hXVcImmapE36s8y+kDRK0vmSnsp5\neSzn5a2tzkuNvI2RdKWkZZKel/SApOPanS8zM+sZSb+UNKkFyxmbz+eDm72sssqxwZfbnY/+WieC\n4hxAbhIRmwBPAn9TSLuo0cuTJOAfgWfydyPn3dNC+Td5fd8BjAf+rZH56I6kLYH/AzYC9gGG5bz8\nBjiolXmp46fAfGA7YEvgWGBJb2fi2nqrpxEXwHk+35f0743IUx+XP0/Sge1afi2tCFI6cb0Hut5W\n6ETEoRExvUHL3i9Xmr0gaaWkhyV9pBHz7mKZlfVdKek5Sf8n6URJ60RsZWsbSD/cBpJ+knfO2ZLG\nVwZI2jrXKj4t6XFJ/9LNvPYBRgH/AkyUtEGez9uA7wPvzgXvuZz+15Luk7RC0nxJZxaWXTn4nyDp\nSeCW3pwQImIh8Evg7YV1uVbSM5LmSvpoYVlnSprRxXZ4R87nSkmXS7qsiyu7U4EVwLER8Wgkz0XE\njyPivwvzvFzS4lxbe5ukXQrDpkn6Xq4NeEHSbyW9SdI3JT0r6SFJuxfG783v9C5gWkS8GBGrI+K+\niPhlYV575wPUc/k3Oa6Qp/Mk/ULSi8D+kjaU9DVJT+YD+fclDS3M63BJswoHvV0Lw+ZJ+rSk+/M2\nuEzSkC7yXTqSPiRpZt4HFuX9Ye9256uRcgD9qta8q/UdgIg4MSK+1Mf5dhvYSdo0l6kn83Ifzf0j\n+rLMTpG36R8kvZSPMedJ2rzd+bJudVuho6TP8UcX586n8rI3BT4LnC9p574up4f+JiKGkSpozs7L\n/VGTl2lNMpCC4iOAS4HNgWuB7wDkgvc/wO+B0cABwCmSDu5iXpPyNDNy/98ARMQc4ETgjlxLXTlA\nv0iqUd4c+Gvg45KOrJrn+4C3AV0tdy2StgEOA+7LSZcCC4Ctgb8DviLp/YVJ6m2HDYCrgWnAcOAS\n4INdLPpA4OqIeK2bLP4S2BH4C+BeoLrm/ijSQXEE8ApwRx5vBHAF8I2cv97+TncC35U0UdK2xQGS\ntsv5+m9gK2AcMKswyoeAs0i137eTDmRvyeP9ZV7+5/O8dgcuAD5GqpH+AXCtpA2r1vEQYHtgV+C4\nOnkuHUmfAr4JfAUYCWwLfJe0n9Yaf12+vXlH8a5WRHyiuwn6u765XN8M7ELaBzcF3g0sA/boz7zb\nSdJpwH8C/wpsBuxFCjpuzOtsHa5Ghc6tks6S9FvgJWCHnPZPefh6kv5N0hOSlubKnc3ysLUql7pZ\ndkTEz4BngbWC4uqLTRWeS5I0RNKFkpbnipDfSRrZg/V9PiKuBf4BmCSpst6b5XV5Oq/bv1UuCHL/\nO3P3MXkdd8n9J0j6WSF/XVV4fVbSQr1RQ35ATt9D0h15PRZJ+k6x/OTl/bOkR/K0X5L05lz5syIv\ns1IpuJ+kBZLOUGq2OE9dNF9VF5VJHS0i1qkPMA84sCrtTOCmQv/OwKrcvSfwZNX4U4Af15n/RqQa\n0iNz/w+AawrDjwNu7yaP3wTOzd1jgQB2KAyvpA3uYh1fAJ4DngC+BwwFtgFeBYYVxv0qqca0u+2w\nL7AQUGH47cCX6+RhLnBiof+InJ+VwA11ptk8r9dmuX8acH5h+CeBOYX+vwKe6+PvtAUpmJ2dt8ks\n4F2F6a6uM9004CeFfpEuat5cSHs38HjuPg/4UtU8HgbeV/itPlwYdg7w/XaXk074kIKZF4C/72Kc\nM0kXRxfmcvdPwIa5DD2VP98ENszj70e6KDyDFPjNA46p+n2/D9yY99XfANsVhr81D3sm/45HFYZt\nSbqQXAHcDXyJbsp6Ydrj6o2b8/Tlqvx/FlhMagY0Argul69ngP8lVVj8FHgNWJW342dqzPufSM2G\nNukib/OATwP3A88DlwFDCuXoOuBpUgBxHTCmMO2teTv8Nm/PG4ARedhYUnmfRGrWtgz4XGHa9YDT\ngUeB5aRKhuFV0651DCQF9i8Uf5ucvknO5/GFfWcG8JOct9nA+Kr1PhB4EykI27Iw7B15Xuu3u5wM\npA+F8zPpfDWbfPzM+9KTpAu4wcD6Oe2f8vDjSeedHfJvfRXw06r95SfAxsDQGsveD1hQ2Pc+CPwZ\n2Kl6f6Mqjsj70oW5+2OkCpqNgEHAO4FNu1vfqvQngY/n7p8A15AqYcYCfwROKAw7LXdPzWWlON2p\nhfy9TKogG0Q679+Zh+1Eakq4dWFbvTl3v5N0QTk4p88BTinkM3LeNs2/yyuki+wdSMfvB4FJhe27\nmlSRtSGpou9FYKc8fBpvHOd2B5aSzuuDSMeIeeTjeCd/BlJN8eJC90vAkFwLsx2wdb5aeU6pycMZ\npFqrWj5I+uF/kfsvAg6VtFW9BUvaU9Kv85Xg86Ta5Opbl/N7uT5HRsTmEbFdRPxzRKwi1Q4/ExEr\nC+M9QarZrKi3HbYGFkbeY3uQp+WkJiQARMS1kWrGTwUqV46DJJ2db9euIO30sOa6F9v5rqrRX2lv\n1qvfKSKejYjTI2KXPM4s4GeSRDoYP9rFuhXXeyvSwe+ewnKvz+mVfJ1Wla9tSNuzonqbt/yhyA71\nbmAI6Q5FVyaQAuPNSeXtc6QD+ThgN1KNZ/EW7JtI+9ho0sF2qqSdCsOPIQVyI0j7xUUAkjYmBcQX\nk+5sTAS+V7i9+l3SiWcU6QR9fG9XuIfeRLpbsx0wGTiNFChvRdqXzyBVdh3Lms9QnFNjXgcC10fE\nC90ss97djPWAH+e8bEsqk9+pmvZDwEdI22wDUoBdtDfpxHwA8HmlZmaQLoKPJJ08tyYF3d/tJp8A\n7yHtN1cVE/M6/oI1n2moeWesarrFpODrqELyscClEfHnHuTHeudn+Th5O+mi9CuFYdMiYnakJm/V\n2/4Y4BsR8Vj+raeQmi8W76acGanJ3Ko6y946L3sZ8AVS87+He5n/P5MukP8yIl6NiHsiYkUv5/EU\nMFzpmZWJwJSIWBkR84Cvk/Y/SNvnfbl7H1KwW+l/Xx5ecXtE/CLSA/o/JR0bIVUKbQjsLGn9iJgX\nEY8C5Lzfmbf3PFIl3/tY0zkRsSIiZgMPkCq9HouI50k1/btXjf/vEfFKRPwG+DlrlquKycAPIuKu\nvA2nkwLuvbrebO03kILieuaTav02L3yGRcRhdcafRApqnpS0GLicdEX7oTw8akxzMemAvE1EbEaq\nqVLVOLWm661KQRtWSNuWVAPcnUXA6Bw0VmzTxfg3A0eq63ZfHyIFNAeSrirH5vTqde+J3v5Or4uI\nZcDXSCfe4Xleb+5qkkL3MlIgsEthuZtFapdWyddZVfnaKCIu6cM6ls2WwLKIWN3NeHdExM8i4rV8\nsjsG+I+IWBoRTwNf5I2TSEVXB+afR8RtEfEKKcB+t1IzpMOBeZHaxa+OiPuAK4G/zyev/wd8Pp90\nHwB6+wDQXsWLJ0n1TgCvAV/I+V9FOgmPItVo/zki/rfq4rUrW5LKdne+HRFPRcQzpFqwcQARsTwi\nroyIl/LF9lmsfdL8cUT8Med1RmXagi9GxKqI+D2p+VPlZH0iqeZ4Qf4tzgT+rgdNRkZQf79ZxJoX\n3fUChWrTgQ/D6w/XHp3Ht8arVaFT0VVFzNakSp6KJ0g1nMWKke4ql57Kyx4eEeMi4tLeZR1I+8Wv\ngEuV3rx0jqT1ezmP0aS7PiNI8UP1elUqsn4D7CNpFKlGdQbwXkljSefUYrO/mhVeETEXOIVUvpZK\nulTS1gCS3iLpOqU2+StIFyjVFXY9rbgCeDYiXqxal2IFUUVPKpM6UhmC4ruBlbnNzdBcu/l2Se+q\nHlFSpS3r4aQDf6Wm6j954y0US4AxWrNd2zBSDe7LkvbgjQC6oSJiPumNEF9Vave0K3AC6dZzd+4g\nXVF+QtJgSRPous3hN0i3Vn+a2xgpB+PFE+Iw0tXfclJt61fWnk2P9fh3ApD0n3n44JyvjwNzI2I5\nqWbwQElH5eFbSqo+kQMQqc30+cC5kv4iz3u03mjLfD5wYr4bIEkbKz1YOazW/GwNy4ERPQiCqk90\ntU6OxYNpdwfm1+eXa5yeycO3A/asOlAfQ6q53Yp0Ai7mpZiHnriz6uLpzjrjPR0RLxf6/4t02/gG\npdcent6LZa5xR6cLNe9mSNpI0g+U2jauAG4DNteab2Xp7k5IveHbAVcXtvUc0jGou/aZy6i/34zK\nw+ste0id6a4h1aRtT6ppfj4i7u4mH9Z4XV3sPUXaZyq2Jd21LQZpjahcgnTbf6NC/5teX0C6MP1i\nROxMumtxOL14C1U+Z40m1ZQvI130Vq/XwrysuaT99pPAbblGejGppvX26P6ZnkqeL46IvfNyghSz\nQGr+9xCwY0RsSroL1ZdKq4ot8h234ro8VWO8dbYyacAHxbkGoRLkPk7aSX9IugqrdiwwKyJuiIjF\nlQ/wbWBXpYbzt5DaSS2WVDk4/zPwH5JWkh7QmlFj3o1yNKlG9inSbekvRMRN3U0UEX8C/pYURD9H\nqjW5jhTU1hp/GelWx8ukwr2SdNVaCUAhtXl6glTAHyQ9/NYnvfydIB3Qrs7r8hjpYHBEnteTpLZX\np5EColnUr0GC1L5zLnBnDgxuIt0OJiJmAh8l3ZZ9No93XN/WsnTuIO1f1Q+dVqs+0dU6ORYPvN0d\nmF+/A6L0Oqjhefh84DdVB+pNIuLjpPalq1nz7skaD3A20BrrG+m26mkRsQNpH/6U8oMy1ePWcBNw\ncNX26I3TSPv6nvmkuW9O78+Js2I+cGjV9h4S6QGsrlT2m78tJubf8lDSXaxeyRchM0jHvWNxLXEn\nugQ4VdL2+bf+CnBZD+409cUsUtOM9ZUeWPu7ygBJ+0v6q3xhuIIU1HYbnCq9BeZwUnOeCyPiD/m8\nNgM4S9IwpYfAP8WaFVm/AT7BG00lbq3q7265O0l6v9LD3y+Tancr+R2W1+EFpf8X+Hid2fTGFyVt\nIGkf0jn78hrjrLuVSdEBDZv9ac8HuAv4SLvz4c/A/ZCCriWkwHgj0q3EQ0nt2KDwgEthmi+T7ohs\nRbrV9/oDobzxsMfXSO1b9yHV+rw1D59GOgnsnYefC/w2DxtGuog7NudjfdKr/d6Wh19GOqFtRHpI\ndQGFh+dIJ6sz66zncfTiQbuq4YeT3npSaQ+/CNg/D7sTmNzF9t0Q+B2pHfxbSRUdW5JqhA7L48yj\n/kNF55DaDQ4hXTxczZoPJN1KfhCqej2p8bAcaz44dWru3y73bwVMqDdt1Xp9hrTfHJJ/p7Gk9sT3\n8sZDl6+vR6151ljv95KeNVhJ4eFLfxpa3tfY5lXD1tiXauwv65EqleaTLlIvBLboyf6Sx1mrbHWx\nb+xAOv+9QGp+9e1CmTia9BDui3kf/HYX++k8UhC6kvQQ6x3AScCgwjhb5HV5Oq/b54H1CsM/lvO2\nXe4/PPfvWRin7r5Oekbg7pyHZ0iVXZWH7vYl1RS/QHqA9z9Y85gWpLbTlf7bgeMK/V8GfljcvqQm\nactIzzscWxh3GoUH90ll93ekiqtFpOB5WK3t2EmftmfAnxb+2Kmt4JtyQZqUC/OodufLn4H9ITVR\nmJlPMovzSeg9edgaB/ucNiSfiBblz7d5420JPTkwV94+8QKpOcD2heE75eU/TWp6cAswLg/bKp9Q\nar59ghRQHVRnHY+j70HxqaST64t53f69MGxCXsfngE/Xmf9mpDd0zM/r/Cip+dOWefg86gfFW5MC\nkxdIT8VXTtCNCIrXI9WKPUw6YT8KfKXetDXW6wTSgz+Vdo4/IAdJtfad6nlWr3dOe4R0t6Dt5cIf\nf9alT61j10D8KK+slYCkyaQT/cakJgdTIuLn7c2VWc9J2o8UCI2pM3wa6cDd0H+AlDQGmBER72nk\nfK21JN0CXBwRP2x3XszWJd0deweKprUplnSI0kuk5/bywRFrkoiYGhEjI7Wj3NUBsVnPRHqDggPi\ndVh+AOodpGYyZmZraUpQnBuof5fUdnBn4Gg1/68WzczM1iJpOumhxFNizfe8m1kPRMStA72WGGhO\n8wlJ7yY9kHJw7p8CEBFfrTX+0I0Gx6ab9vY1gO0xYtjQPk23bGW9d41bT7wWb+nTdOvpjw3OSWMs\nXfLysoio+4cwzSbpEOBbpHdj/jAizm5XXsysay6vZq3R3ftD+2o0a77vcwHp7/5el9u3TgYYtun6\n/MOkrv5roXN8dL9d+jTd+bfObnBOyuXlV37R/Ug1DNnw4O5HaoP/Pmd2b9+B2zCFOzkHkcrm7yRd\nGxEPtitPZlaby6tZ6zQrKO5WREwl/dc3I9801E/7mbXOHqQ/OnkMQNKlpLcc1DzJbqANYwh9fQ2u\n2cCzkmdbeaenV+UVXGbNqvW0zDYrKF7Imi/BH0PP/orYzJqvV3dyhrARe77+XxJmdlNc0co7Pd2W\nV3CZNetKT8tss4Li3wE75r/UXAhMpEl/fWxmjVe8k7OphvtOjlmHK5bZ8bsNiV/9alabc2TWOQaN\n6tl4TQmKI2K1pE8AvyI9GHBBRAzIRrW12gr3td2x9U51e+GXX/lVm3KyzunVnZy37PoSPsGavaGn\nJ9gG8Z1XsxZpWpviiPgF6W85zayz+E6O2brD5dWsRdr2oJ2ZtUeZ7uSYretcXs1ax0GxWQn5To7Z\nusPl1aw1mvY3z2ZmZmZm6woHxWZmZmZWeg6KzczMzKz0HBSbmZmZWek5KDYzMzOz0nNQbGZmZmal\n56DYzMzMzErPQbGZmZmZlZ6DYjMzMzMrPQfFZmZmZlZ6/pvnfvrofru0Owul9fIrv2p3FszMzGyA\ncE2xmZmZmZWeg2IzMzMzKz0HxWZmZmZWeg6KzczMzKz0/KBdL51/6+x2Z6GUhmx4cLuzYGZmZgOY\na4rNzMzMrPQcFJuZmZlZ6fW5+YSkbYCfACOBAKZGxLcknQl8FHg6j3pGRPyiq3m9sHo1dy5Z3tes\nmJmZmZn1S3/aFK8GTouIeyUNA+6RdGMedm5EfK3/2TMzMzMza74+B8URsQhYlLtXSpoDjG5UxszM\nzMzMWqUhbYoljQV2B+7KSZ+UdL+kCyRtUWeayZJmSpq5+uXXGpENM8skbSPp15IelDRb0sk5fbik\nGyU9kr9rlk8za618vlwq6YFCmsurWQv1OyiWtAlwJXBKRKwAzgN2AMaRapK/Xmu6iJgaEeMjYvzg\nIX7ez6zBKs2bdgb2Ak6StDNwOnBzROwI3Jz7zaz9pgGHVKW5vJq1UL+iUUnrkwLiiyLiKoCIWBIR\nr0bEa8D5wB79z6aZ9UZELIqIe3P3SqDSvGkCMD2PNh04sj05NLOiiLgNeKYq2eXVrIX6HBRLEvAj\nYE5EfKOQPqow2geBB6qnNbPWqWreNDI/DwCwmPT2mFrTvN686enlr7Ykn2a2lh6VV3CZNWuE/rx9\n4r3AscAfJM3KaWcAR0saR3pN2zzgY/3KoZn1WXXzpnQtm0RESIpa00XEVGAqwPjdhtQcx8xap6vy\nmoe7zJr1U3/ePnE7oBqDunwnsZm1Rq3mTcASSaMiYlG+q7O0fTk0s264vJq1kJ9wMxuA6jVvAq4F\nJuXuScA1rc6bmfWYy6tZCzkoNhuYKs2b3i9pVv4cBpwNHCTpEeDA3G9mbSbpEuAOYCdJCySdgMur\nWUv1p02xmXWoLpo3ARzQyryYWfci4ug6g1xezVqkI4LiTQYPZq+RW7Y7G2Yt8zsWtzsLZmZmVuDm\nE2ZmZmZWeg6KzczMzKz0HBSbmZmZWek5KDYzMzOz0nNQbGZmZmal56DYzMzMzErPQbGZmZmZlZ6D\nYjMzMzMrPQfFZmZmZlZ6DorNzMzMrPQcFJuZmZlZ6TkoNjMzM7PSG9zuDBTduWT56917jdyyjTkx\nMzMzszJxTbGZmZmZlZ6DYjMzMzMrPQfFZmZmZlZ6DorNzMzMrPT69aCdpHnASuBVYHVEjJc0HLgM\nGAvMA46KiGd7Mj8/XGdmZmZm7dCImuL9I2JcRIzP/acDN0fEjsDNud/MzMzMrGM1o/nEBGB67p4O\nHNmEZZiZmZmZNUx/g+IAbpJ0j6TJOW1kRCzK3YuBkbUmlDRZ0kxJM1eterWf2TCzWiQNknSfpOty\n/3BJN0p6JH9v0e48mhlI2kbSryU9KGm2pJNzususWYv0NyjeOyLGAYcCJ0natzgwIoIUOK8lIqZG\nxPiIGD906KB+ZsPM6jgZmFPod/Mms860GjgtInYG9iKdU3fGZdasZfoVFEfEwvy9FLga2ANYImkU\nQP5e2t9MmlnvSRoD/DXww0KymzeZdaCIWBQR9+bulaSL2dG4zJq1TJ+DYkkbSxpW6QY+ADwAXAtM\nyqNNAq7pbybNrE++CXwGeK2Q1uvmTU8vd/Mms1aSNBbYHbgLl1mzlulPTfFI4HZJvwfuBn4eEdcD\nZwMHSXoEODD3m1kLSTocWBoR99Qbp6fNm7ba0s2bzFpF0ibAlcApEbGiOMxl1qy5+vye4oh4DNit\nRvpy4ID+ZMrM+u29wBGSDgOGAJtKupDcvCkiFrl5k1lnkbQ+KSC+KCKuyskus2Yt4n+0MxuAImJK\nRIyJiLHAROCWiPgwbt5k1pEkCfgRMCcivlEY5DJr1iL9+kc7M1vnnA3MkHQC8ARwVJvzY2bJe4Fj\ngT9ImpXTzsBl1qxlHBSbDXARcStwa+528yazDhQRtwOqM9hl1qwF3HzCzMzMzErPQbGZmZmZlZ6D\nYjMzMzMrPQfFZmZmZlZ6DorNzMzMrPQcFJuZmZlZ6TkoNjMzM7PSc1BsZmZmZqXnoNjMzMzMSs9B\nsZmZmZmVnoNiMzMzMys9B8VmZmZmVnoOis3MzMys9BwUm5mZmVnpOSg2MzMzs9JzUGxmZmZmpeeg\n2MzMzMxKb3BfJ5S0E3BZIWkH4PPA5sBHgadz+hkR8Ys+59DMzMzMrMn6HBRHxMPAOABJg4CFwNXA\nR4BzI+JrDcmhmZmZmVmTNar5xAHAoxHxRIPmZ2ZmZmbWMo0KiicClxT6PynpfkkXSNqi1gSSJkua\nKWnmqlWvNigbZlYhaXNJV0h6SNIcSe+WNFzSjZIeyd81y6eZtZakIZLulvR7SbMlfTGnu8yatUi/\ng2JJGwBHAJfnpPNI7YvHAYuAr9eaLiKmRsT4iBg/dOig/mbDzNb2LeD6iHgrsBswBzgduDkidgRu\nzv1m1n6vAO+PiN1I589DJO2Fy6xZyzSipvhQ4N6IWAIQEUsi4tWIeA04H9ijAcsws16QtBmwL/Aj\ngIj4U0Q8B0wApufRpgNHtieHZlYUyQu5d/38CVxmzVqmEUHx0RSaTkgaVRj2QeCBBizDzHpne9Ib\nYH4s6T5JP5S0MTAyIhblcRYDI9uWQzNbg6RBkmYBS4EbI+IuXGbNWqZfQXE+yR4EXFVIPkfSHyTd\nD+wPnNqfZZhZnwwG3gGcFxG7Ay9Sdds1IoJUE7WWYpv/p5e7zb9ZK+S7rOOAMcAekt5eNdxl1qyJ\n+hUUR8SLEbFlRDxfSDs2Iv4qInaNiCMKV7hm1joLgAW5pgngClKQvKRyNyd/L601cbHN/1Zbus2/\nWSvlpk6/Bg7BZdasZfyPdmYDUEQsBubnP9mB9NrEB4FrgUk5bRJwTRuyZ2ZVJG0lafPcPZR0F/Yh\nXGbNWqbPf95hZh3vk8BF+Q0xj5H+WGc9YIakE4AngKPamD8ze8MoYHr+M6z1gBkRcZ2kO3CZNWsJ\nB8VmA1REzALG1xh0QKvzYmZdi4j7gd1rpC/HZdasrp+sGFEz/R83XdbreTkoNjMzyxp5gjWzdYvb\nFJuZmZlZ6TkoNjMzM7PSc1BsZmZmZqXnoNjMzMzMSs9BsZmZmZmVnoNiMzMzMys9B8VmZmZmVnoO\nis3MzMys9BwUm5mZmVnpOSg2MzMzs9JzUGxmZmZmpeeg2MzMzMxKz0GxmZmZmZXe4HZnwKwTvfzK\nr9ZKG7LhwW3IiZmZmbWCa4rNzMzMrPQcFJuZmZlZ6XUbFEu6QNJSSQ8U0oZLulHSI/l7i8KwKZLm\nSnpYku83m5mZmVnH60lN8TTgkKq004GbI2JH4Obcj6SdgYnALnma70ka1LDcmpmZmZk1QbdBcUTc\nBjxTlTwBmJ67pwNHFtIvjYhXIuJxYC6wR4PyamZmZmbWFH1tUzwyIhbl7sXAyNw9GphfGG9BTjMz\nMzMz61j9ftAuIgKI3k4nabKkmZJmrlr1an+zYWZVJJ0qabakByRdImlIV88DmFl7SRok6T5J1+V+\nl1ezFuprULxE0iiA/L00py8EtimMNyanrSUipkbE+IgYP3Somx2bNZKk0cC/AOMj4u3AIFJ7/5rP\nA5hZRzgZmFPod3k1a6G+/nnHtcAk4Oz8fU0h/WJJ3wC2BnYE7u5vJs2sTwYDQyX9GdgIeAqYAuyX\nh08HbgU+247MmdkbJI0B/ho4C/hUTp6Ay6tZl/5x02UNm1e3QbGkS0iFcoSkBcAXSMHwDEknAE8A\nRwFExGxJM4AHgdXASRHhthFmLRYRCyV9DXgSWAXcEBE3SKr3PMAaJE0GJgNsO9p/fGnl0cgTbC99\nE/gMMKyQ1qPyCi6zZo3QbcmJiKPrDDqgzvhnka50zaxNctvDCcD2wHPA5ZI+XBwnIkJSzecBImIq\nMBVg/G5Dev3MgJn1nKTDgaURcY+k/WqN01V5zcNdZs36yZeTZgPTgcDjEfE0gKSrgPeQnweIiEVV\nzwOYWfsw8OzFAAAgAElEQVS8FzhC0mHAEGBTSRfi8mrWUv6bZ7OB6UlgL0kbSRLpzs4c3ngeANZ8\nHsDM2iQipkTEmIgYS3og9paI+DAur2Yt5ZpiswEoIu6SdAVwL6l9/32kW6ubUON5ADPrSDWf3zGz\n5nBQbDZARcQXSA/GFr1CnecBzKz9IuJW0lsmiIjluLyatYybT5iZmZlZ6TkoNjMzM7PSc1BsZmZm\nZqXnNsVmNQzZ8OB2Z8HMzMxayDXFZmZmZlZ6DorNzMzMrPQcFJuZmZlZ6TkoNjMzM7PSc1BsZmZm\nZqXnoNjMzMzMSs9BsZmZmZmVnoNiMzMzMys9B8VmZmZmVnoOis3MzMys9BwUm5mZmVnpOSg2MzMz\ns9JzUGxmZmZmpddtUCzpAklLJT1QSPsvSQ9Jul/S1ZI2z+ljJa2SNCt/vt/MzJuZmZmZNUJPaoqn\nAYdUpd0IvD0idgX+CEwpDHs0Isblz4mNyaaZmZmZWfN0GxRHxG3AM1VpN0TE6tx7JzCmCXkzMzMz\nM2uJRrQpPh74ZaF/+9x04jeS9qk3kaTJkmZKmrlq1asNyIaZmZmZWd/0KyiW9DlgNXBRTloEbBsR\n44BPARdL2rTWtBExNSLGR8T4oUMH9ScbZqVVp83/cEk3Snokf29RGDZF0lxJD0s6uD25NrNaJM2T\n9IdcsTQzp9Utz2bWWH0OiiUdBxwOHBMRARARr0TE8tx9D/Ao8JYG5NPMapvG2m3+TwdujogdgZtz\nP5J2BiYCu+RpvifJV6RmnWX//EzO+NxfszybWeP1KSiWdAjwGeCIiHipkL5V5SQraQdgR+CxRmTU\nzNZWq80/MAGYnrunA0cW0i/NF6+PA3OBPVqSUTPrq3rl2cwarCevZLsEuAPYSdICSScA3wGGATdW\nvXptX+B+SbOAK4ATI6L6hG1mzTUyIhbl7sXAyNw9GphfGG9BTltLsc3/08vd5t+sRQK4SdI9kibn\ntHrleQ0us2b9N7i7ESLi6BrJP6oz7pXAlf3NlJk1RkSEpOjDdFOBqQDjdxvS6+nNrE/2joiFkv6C\nVOn0UHFgV+XZZdas//yPdmYDzxJJowDy99KcvhDYpjDemJxmZh0gIhbm76XA1aTmTfXKs5k1mINi\ns4HnWmBS7p4EXFNInyhpQ0nbk9r8392G/JlZFUkbSxpW6QY+ADxA/fJsZg3WbfMJM+tcuc3/fsAI\nSQuALwBnAzNy+/8ngKMAImK2pBnAg6RXKZ4UEW58aNYZRgJXS4J0br44Iq6X9DtqlGczazwHxWbr\nsDpt/gEOqDP+WcBZzcuRmfVFRDwG7FYjfTl1yrOZNZabT5iZmZlZ6TkoNjMzM7PSc1BsZmZmZqXn\noNjMzMzMSs9BsZmZmZmVnoNiMzMzMys9B8VmZmZmVnoOis3MzMys9BwUm5mZmVnpOSg2MzMzs9Jz\nUGxmZmZmpeeg2MzMzMxKz0GxmZmZmZWeg2IzMzMzKz0HxWZmZmZWeg6KzczMzKz0ug2KJV0gaamk\nBwppZ0paKGlW/hxWGDZF0lxJD0s6uFkZNzMzMzNrlJ7UFE8DDqmRfm5EjMufXwBI2hmYCOySp/me\npEGNyqyZmZmZWTN0GxRHxG3AMz2c3wTg0oh4JSIeB+YCe/Qjf2ZmZmZmTdefNsWflHR/bl6xRU4b\nDcwvjLMgp61F0mRJMyXNXLXq1X5kw8zMzMysf/oaFJ8H7ACMAxYBX+/tDCJiakSMj4jxQ4e6hYWZ\nmZmZtU+fguKIWBIRr0bEa8D5vNFEYiGwTWHUMTnNzJqgzoOw/yXpoXwn52pJmxeG+UFYsw4laXNJ\nV+TyO0fSuyUNl3SjpEfy9xbdz8nM+qJPQbGkUYXeDwKVE/K1wERJG0raHtgRuLt/WTSzLkxj7Qdh\nbwTeHhG7An8EpoAfhDVbB3wLuD4i3grsBswBTgdujogdgZtzv5k1weDuRpB0CbAfMELSAuALwH6S\nxgEBzAM+BhARsyXNAB4EVgMnRYQbDJs1SUTcJmlsVdoNhd47gb/L3a8/CAs8LqnyIOwdLciqmXVB\n0mbAvsBxABHxJ+BPkiaQzsEA04Fbgc+2PodmA1+3QXFEHF0j+UddjH8WcFZ/MmVmDXM8cFnuHk0K\nkiu6fBAWmAyw7ehuDxNm1n/bA08DP5a0G3APcDIwMiIW5XEWAyNrTewya9Z//kc7swFK0udId2wu\n6u20xQdht9rSLSzMWmAw8A7gvIjYHXiRqqYSERGkO7RrcZk16z8HxWYDkKTjgMOBY/KJFPwgrFkn\nWwAsiIi7cv8VpCB5SeU5nvy9tE35MxvwHBSbDTCSDgE+AxwRES8VBvlBWLMOFRGLgfmSdspJB5Ce\nz7kWmJTTJgHXtCF7ZqXghkdm67A6D8JOATYEbpQEcGdEnOgHYc063ieBiyRtADwGfIRUeTVD0gnA\nE8BRbcyf2YDmoNhsHeYHYc0GjoiYBYyvMeiAVufFrIzcfMLMzMzMSs9BsZmZmZmVnoNiMzMzMys9\nB8VmZmZmVnoOis3MzMys9BwUm5mZmVnpOSg2MzMzs9JzUGxmZmZmpeeg2MzMzMxKz0GxmZmZmZWe\ng2IzMzMzKz0HxWZmZmZWeg6KzczMzKz0HBSbmZmZWek5KDYzMzOz0nNQbGZmZmal121QLOkCSUsl\nPVBIu0zSrPyZJ2lWTh8raVVh2PebmXkzMzMzs0YY3INxpgHfAX5SSYiIf6h0S/o68Hxh/EcjYlyj\nMmhmZmZm1mzdBsURcZuksbWGSRJwFPD+xmbLzMzMzKx1+tumeB9gSUQ8UkjbPjed+I2kfepNKGmy\npJmSZq5a9Wo/s2FmZmZm1nc9aT7RlaOBSwr9i4BtI2K5pHcCP5O0S0SsqJ4wIqYCUwFGvmlo9DMf\nZmZmZmZ91ueaYkmDgb8FLqukRcQrEbE8d98DPAq8pb+ZNLPaaj0IWxh2mqSQNKKQNkXSXEkPSzq4\ntbk1s3ok7VR4SH2WpBWSTpE0XNKNkh7J31u0O69mA1V/mk8cCDwUEQsqCZK2kjQod+8A7Ag81r8s\nmlkXpgGHVCdK2gb4APBkIW1nYCKwS57me5XyambtFREPR8S4/KD6O4GXgKuB04GbI2JH4Obcb2ZN\n0JNXsl0C3AHsJGmBpBPyoIms2XQCYF/g/vyKtiuAEyPimUZm2MzeEBG3AbXK2LnAZ4Bi06QJwKX5\njs7jwFxgj+bn0sx66QDSm5yeIJXb6Tl9OnBk23JlNsD15O0TR9dJP65G2pXAlf3Plpn1laQJwMKI\n+H16QczrRgN3FvoX5LRa85gMTAbYdnR/Hz0ws14qVjqNjIhFuXsxMLLWBC6zZv3nf7QzG0AkbQSc\nAXy+P/OJiKkRMT4ixm+1pVtYmLWKpA2AI4DLq4dFRLDm3Z/iMJdZs35yUGw2sLwZ2B74vaR5wBjg\nXklvAhYC2xTGHZPTzKxzHArcGxFLcv8SSaMA8vfStuXMbIBzUGw2gETEHyLiLyJibESMJTWReEdE\nLAauBSZK2lDS9qQHYe9uY3bNbG3Vrzq9FpiUuycB17Q8R2Yl4aDYbB3WxYOwa4mI2cAM4EHgeuCk\niPA/55h1CEkbAwcBVxWSzwYOkvQI6a1PZ7cjb2Zl4Nb4Zuuweg/CFoaPreo/CzirmXkys76JiBeB\nLavSlpPeRmFmTeaaYjMzMzMrPQfFZmZmZlZ6DorNzMzMrPQcFJuZmZlZ6TkoNjMzM7PSc1BsZmZm\nZqXnoNjMzMzMSs9BsZmZmZmVnoNiMzMzMys9B8VmZmZmVnod8TfPL6xezZ1Llrc7G2ZmZmZWUq4p\nNjMzM7PSc1BsZmZmZqXnoNjMzMzMSs9BsZmZmZmVXrdBsaRtJP1a0oOSZks6OacPl3SjpEfy9xaF\naaZImivpYUkHN3MFzMzMzMz6qyc1xauB0yJiZ2Av4CRJOwOnAzdHxI7AzbmfPGwisAtwCPA9SYOa\nkXkzMzMzs0boNiiOiEURcW/uXgnMAUYDE4DpebTpwJG5ewJwaUS8EhGPA3OBPRqdcTMzMzOzRulV\nm2JJY4HdgbuAkRGxKA9aDIzM3aOB+YXJFuS06nlNljRT0szVL7/Wy2ybmZmZmTVOj4NiSZsAVwKn\nRMSK4rCICCB6s+CImBoR4yNi/OAhft7PzMzMzNqnR9GopPVJAfFFEXFVTl4iaVQePgpYmtMXAtsU\nJh+T08zMzMzMOlJP3j4h4EfAnIj4RmHQtcCk3D0JuKaQPlHShpK2B3YE7m5cls2sQtIFkpZKeqAq\n/ZOSHspvjDmnkO43w5h1KEmn5jL7gKRLJA3p6k1PZtZYPakpfi9wLPB+SbPy5zDgbOAgSY8AB+Z+\nImI2MAN4ELgeOCkiXm1K7s1sGuktL6+TtD/pgdfdImIX4Gs53W+GMetQkkYD/wKMj4i3A4NI5bXm\nm57MrPEGdzdCRNwOqM7gA+pMcxZwVj/yZWY9EBG35Qdgiz4OnB0Rr+RxKk2bXn8zDPC4pMqbYe5o\nUXbNrGuDgaGS/gxsBDwFTAH2y8OnA7cCn21H5swGOj/hZjbwvAXYR9Jdkn4j6V05vUdvhoE13w7z\n9HLf6DFrtohYSLqr8ySwCHg+Im6g/pue1uAya9Z/3dYUm9k6ZzAwnPRnO+8CZkjaoTcziIipwFSA\n8bsN6dWbZcys93Jb4QnA9sBzwOWSPlwcJyJCUs3yWCyzm2p4HLz1uCbn2GxdMrdHY7mm2GzgWQBc\nFcndwGvACPxmGLNOdiDweEQ8HRF/Bq4C3kP9Nz2ZWYO5pths4PkZsD/wa0lvATYAlpHeDHOxpG8A\nW9PDN8P88f6NcK2TWVHPap166UlgL0kbAatIz+zMBF4kveHpbNZ805OZNZiDYrN1mKRLSA/hjJC0\nAPgCcAFwQX5N25+ASfkPdmZLqrwZZjU9fDPMSp5ddlNc8SIpsO4kI+i8PEFn5st56rme5Gu7Ri80\nIu6SdAVwL6l83kdqDrEJqQnUCcATwFHdzauDy2wzder+1AxlWldozPr2qMwqnSvbS9LTpKvhsvzI\nZdqhy7KuvV3P7SJiq2ZlptEkzYyI8e3OR1En5gk6M1/OU891ar56a6CsR0+VaX3LtK7Q2vXtiJri\niNiqTD+y13XgKct6mpmZDVR+0M7MzMzMSs9BsZn1xNR2Z6CGTswTdGa+nKee69R89dZAWY+eKtP6\nlmldoYXr2xFtiiG9eDy/Z3HA87oOPGVZTzMzs4GqY4JiMzMzM7N2cfMJM6tL0iGSHpY0V9LpbczH\nNpJ+LelBSbMlnZzTz5S0UNKs/DmsxfmaJ+kPedkzc9pwSTdKeiR/b9HC/OxU2BazJK2QdEo7tpOk\nCyQtza8GrKTV3TaSpuT97GFJB7cwT/8l6SFJ90u6WtLmOX2spFWFbfb9ZuTJzDqHg2Izq0nSIOC7\nwKHAzsDRknZuU3ZWA6dFxM6kv68+qZCXcyNiXP78og152z8vu/L2kdOBmyNiR+Dm3N8SEfFwZVsA\n7wReAq7Og1u9naYBh1Sl1dw2+becCOySp/le3v9akacbgbdHxK7AH4EphWGPFrbZiU3IT1N0ysVs\nM3Rxgdy2i9FWkDRI0n2Srsv9A3Z9JW0u6Yp8sTpH0rtbtb5tD4oHcuGFzqtJaqROrAlqljrrWrf2\nbV1e14I9gLkR8VhE/Am4FJjQjoxExKKIuDd3rwTmAKPbkZcemABMz93TgSPblI8DSEHdE+1YeETc\nBjxTlVxv20wALo2IVyLicdJfxu3RijxFxA0RsTr33kn6+/N1VoddzDZDvQvktl2MtsjJpONexUBe\n328B10fEW4HdSOvdkvVta1BcgsJb0TE1SQ02jc6rCWqWaay9rlCj9m0ArGvFaGB+oX8BHRCIShoL\n7A7clZM+mW99X9CGC8wAbpJ0j6TJOW1kRCzK3YuBkS3OU8VE4JJCfzu3U0W9bdMp+9rxwC8L/dvn\nC97fSNqnDfnpi465mG2GLi6QO+VitOEkjQH+GvhhIXlArq+kzYB9gR8BRMSfIuI5WrS+7a4pHtCF\ntwsDYmfuxJqgZqmzrvWs0+vaySRtAlwJnBIRK4DzgB2AccAi4OstztLeuanCoaQaq32LA/Pfa7f8\naWZJGwBHAJfnpHZvp7W0a9vUI+lzpFrIi3LSImDb/Pt+CrhY0qbtyl8vdMoFRtNVXSB3ysVoM3wT\n+AzwWiFtoK7v9sDTwI9zc5EfStqYFq1vu4PiMhTeTq5JaoZOrwlqtFq1bwNlXRcC2xT6x+S0tpC0\nPikgvigirgKIiCUR8WpEvAacT4svPiJiYf5eSmq7uwewRNKonOdRwNJW5ik7FLg3Ipbk/LV1OxXU\n2zZt3dckHQccDhyTg3XyRe3y3H0P8CjwllblybpW4wL5dZ12wdUfkg4HluZ9sKaBtL6kf1p+B3Be\nROwOvEjV3fRmrm+7g+Iy6MiapFYYyOuWdVztW4P9DthR0va55nEicG07MiJJpNtpcyLiG4X0UYXR\nPgg8UD1tE/O0saRhlW7gA3n51wKT8miTgGtalaeCoyk0nWjndqpSb9tcC0yUtKGk7YEdgbtbkSFJ\nh5Bq4Y6IiJcK6VtVmj1J2iHn6bFW5KmfOupithlqXSDTGRejzfBe4AhJ80h3098v6UIG7vouABZE\nRKV53BWkILkl69vuoHjAF94Orklqlo6sCWqGLmrfBsS65oePPgH8itRub0ZEzG5Tdt4LHEs6IRQf\nbDwnP8h6P7A/cGoL8zQSuF3S70kB3M8j4nrgbOAgSY8AB+b+lskB+kHAVYXklm8nSZcAdwA7SVog\n6QTqbJu8X80AHgSuB06KiFdblKfvAMOAG7Xmq9f2Be6XNIt0Yj4xInrahKqdOuZithnqXSDTGRej\nDRcRUyJiTESMJf2Wt0TEhxm467sYmC9pp5x0AOm40JL1beufd0gaTHoFzgGkoOF3wIfaeOJtqHxy\nWi8iVubuG4H/IK3v8og4W+mNG8Mj4jPtzGtf5TZd10XE23P/f1Fj3STtAlxMChy3Jj2Et2MzTnzN\nUmNdR1Waikg6FdgzIiYOhHU1s3VXvmD8JjAIuCAizmpzlhpG0t7A/wJ/4I02tmeQ2hXPALYFngCO\nWkcuYnpM0n7ApyPicElbMkDXV9I40kOFG5DuznyEVInb9PVt+z/aDfDCuwNvvCN0MHBxRJw1UHbm\nXOuyHzACWAJ8AfgZddYtP8hyPOlhllMi4pc1ZtuR6qzrfqSmEwHMAz5WCJLX2XU1MzMro7YHxWZm\nZmZm7dbuNsVmZmZmZm3noNjMzMzMSs9BsZmZmZmVnoNiMzMzMys9B8VmZmZmVnoOis3MzMys9BwU\nm5mZmVnpOSg2MzMzs9JzUGxmZmZmpeeg2MzMzMxKz0GxmZmZmZWeg2IzMzMzKz0HxWZmZmZWeg6K\nzczMzKz0HBSbmZmZWek5KDYzMzOz0nNQbGZmZmal56DYzMzMzErPQbGZmZmZlV7bgmJJQyX9j6Tn\nJV3ernwMFJLOlHRhG5c/VlJIGtyuPPSWpGmSvtzufJSVjwGtI+kYSTe0Ox+27nJ5bR2X1/ZpelAs\n6VZJz0rasGrQ3wEjgS0j4u8lHSfp9mbnpy8cPJn1XTuPAXmer0p6QdIKSbMkHd7IZawLIuKiiPhA\nM+YtaZ6kVXkbVz5bN2NZ/SFpf0m/zkHdvHbnp1O5vLafyytI+ldJD0haKelxSf/aiuU2NSiWNBbY\nBwjgiKrB2wF/jIjVDVrWOlND2Um83ayZOuQYcEdEbAJsDvwImCFpi15M3+j8DER/ExGbFD5PVY/Q\nAdvjReACoCUn13WRy2tprAvlVcA/AlsAhwCfkDSx6UuNiKZ9gM8DvwW+AVxXSP8i8Cfgz8ALwEnA\ny8Cruf+5PN6GwNeAJ4ElwPeBoXnYfsAC4LPAYuCnNZb/ZuAWYDmwDLgI2LxOXgWcCywFVgB/AN4O\nTM75/FPO2//k8bcGrgSeBh4H/qUwrzOBK4DLgJXAvcBuvdhuE4BZOR+PAocUlnkt8AwwF/ho1TIv\nLPQfAcwGngNuBd5WGDYvb7f7gVeAwT3ZFnnYUODrwBPA88DtOW0s6UA6Kf9ey4DPFea5HnB6Xp/l\nwAxgeB5WmfYjwHzgWeBE4F05j88B36nK4/HAnDzur4DtutieewP/l+czHzgup08Dvgv8PP9OdwFv\nLkz3rTz+CuAeYJ+q7T0D+EmedjYwvjD8HcB9edjleV/4cmH44fk3fi7nbdfCsM8CC/O0DwMHNLOc\nDvBjwHHA7YX+jfO+Nr7e9N38NvOAKcCDed/7MTCkq/wAHyWV12dI5Xfrwvx2AW7Mw5YAZ/SgvAwB\nLszpzwG/A0YW1vexvO88DhxTZzsEqYw9kufxXUB52CBSGV+W5/GJPP7gOr/xPODAGulj83Qn5N/v\ntpx+ed4+zwO3AbsUppkGfA/4Zd4Pfgu8Cfhm3t4PAbsXxq97HO5inzwQmNfustGJH1xeweW1o8pr\nYdpvA//d9DLQ5AI2F/hn4J2kwjSyMOxM1gzi1tgJctq5eaccDgwD/gf4amGHXg38J6kgDq2x/L8E\nDsrDt8o/6Dfr5PVgUuCzOSkofBswqvDDFwOa9fK4nwc2AHbIO/bBhXX7M+l20/rAp/MOsH4Pttke\neec7KC9nNPDWPOy2vAMOAcblHev91dsTeAupVuSgvPzP5N9ig0KhmAVsU2e7dbUtvksKskeTCuN7\n8vYdSypQ55OC5N1IAffb8nQnA3cCY/L4PwAuqSqM38/r9gHSAfdnwF/kZS0F3pfHn5DX523AYODf\ngP+rsz23Ix1wjs7bYktgXOF3XZ63+WDSRdOlhWk/nMcfDJxGOjAMKWzvl4HD8nb4KnBnHrYB6aLh\n5LzMvyWdUL6ch++e12fPPO2k/JtsCOxECsS3LmybN3e1z3Tyh/YfA16fZ/4dT877w2a1pu/qtymU\nnQdIZWc46STw5Xr5Ad5POlm9I6f9N2+cbIYBi/K+NST379mD8vKxvB02ynl8J7ApKYBYAeyUxxtF\nPoFVb1tSebuOVMa3JR1LKhffJ5KCiDGkWpqb6N9J9ic5b5Xg6Pi8rhuSTp6zCtNMy9vrnXmb3EI6\ndv5jXtcvA7/uyXG4i33SQbHLa8384PLaceU1TytSJdOJTS8DTSxce5MK1Yjc/xBwak8LWN4IL7Jm\nzd27gccLO/SfyEFKD/N0JHBfnWHvB/4I7AWsVzVsGmsGxXsCT1aNMwX4cWHd7iwMW49UmPbpQR5/\nAJxbI30b0lX5sELaV4Fp1dsT+HdgRtXyFwL7FQrF8V3koea2yPNZRY1a70KBGlNIuxuYmLvnUKjx\nJB0A/kw68FWmHV0Yvhz4h0L/lcApufuXwAlV+XqJGrXF+Xe5us56TgN+WOg/DHioi+3ybGXd8/a+\nqTBsZ2BV7t43b28Vht/OGwfj84AvVc37YeB9pAu5paQTd7cXUZ38oQOOAXmeq0m1K8tIJ64D603f\n1W9TKDsnFoYdBjzaxfx+BJxT6N8kb5OxpAu1esejrsrL8VTViOVxNs7r+f+oCjhqbNsA9i70zwBO\nz923AB8rDDuQ7k+yL+RlPwf8LKePzdPt0MXvs3keZ7N4o0yeXxj+SWBOof+veKNWssvjcBfLdFBc\ne7u4vLq8dlx5zeN9Efg9+WKnmZ9mtimeBNwQEcty/8U5rae2Il1Z3SPpOUnPAdfn9IqnI+LlejOQ\nNFLSpZIWSlpBuoUxota4EXEL8B1STehSSVMlbVpn1tsBW1fylfN2BukhhIr5hXm/RrpN05PG7NuQ\nbsFU2xp4JiJWFtKeINWi1hr3iarlz68ad371RIXx622LEaSrwVr5q1hc6H6JdFCBtM2uLmyvOaQg\nv7jNlhS6V9XoL87rW4V5PUM6INfaFvW2Z3f5RdKnJc3JD+Y8R6qtGNHFtENyO6ytgYWRS3NW3N7b\nAadV7T/bkGqH5wKnkE5AS/P+23EPQfRQ248B2Z0RsXlEjIiIvSLipi6mr/vbFMYp/pZPVA2rnl91\nWXyBdME3mq73za7Ky09JTYYulfSUpHMkrR8RLwL/QKo5WiTp55Le2sV2qbfvb/3/27v7YLnu+r7j\n7w+SbMvCYIuQi7AFdlvF1CFYBIXnJgTZ5THIw7Qe05DRUHfUB6AQyAQ7nWnKTCHupENC0jaMwpM6\n4aEuD7WHYTBGQJpOwCDATQBDBARjGz0Y81jHGBu+/eMcwfXlXmv33r333N3f+zWj2T1nz9nz++3e\nz+q75/zO2QV9XPKzYp5L+tf4zKq6ZMFjP14/yYYkVyX5cv+Z/NX+ofm5Gudz4GSfwxqdeTWv91l/\nPeQ1yUvo9jw/p6ruHqFvK7IqRXGSzcClwK8kOZrkKPCbwIVJLlxitVow/Q26F/Tn5715D65uAP5S\n6yz02n6ZX6iqB9EdDs9SC1fVH1XV4+j2+v0cPzkhY+F2bqH79nvmvH9nVNWz5y2z/cSdJA+gO7Tx\nU4PZF3EL3Vjohb4ObE1yxrx5j6DbI7nYso+ct/307Zm/7P2+dku8Ft+gGzKwWPtO5hbgWQtes9Oq\narH2j/Jc/3LBc22uqr9cYtmx25vkH9ENO7kUOKuqzqQb1rLk3888R4Cz+9f9hO3z7t8CvGZB+0+v\nqncAVNXbq+qpdO9h0R3emyrr6DPgZBbL9pLvTW/+e/kI7pvrhc+3MItb6Ibk3NZv6+8t0a4l81JV\n91TVq6vqArrhS8+l+0+Dqrquqi6m21P1BbrhTOM6Qvd5dcL2pRYc0fzX5J/RDX+6iO5L5rn9/FFy\ntdAon8MagXn9MfO6jvKa5J/TjdXeXVW3LmObY1utPcWX0H1LuoBu7OtOuvGff0H/x7CIY8A5SU6B\nH+/d/FPgD5L8LECSs5M8Y4x2nEF3mOA7Sc7mfs46TvJLSZ6QZBPdIaDvAz+a17b5YfgE8L0kr0p3\n7cYNSR6d5JfmLfO4JM/v9xy+nG587cf7bb01yVuXaMqbgBcl2Z3kAX2fH1VVt9Adgvm9JKcleQzd\ngIL6Lc4AABffSURBVPjFrk18NfCc/jk20Y2Burtf/6SWei369+TNwOuSPLzv95Py05fuWcwbgNck\neWS/jYcm2TNKe5Z4riuT/Hz/XA9O8k+XWPZtwEVJLk2yMclDkuwcYRtn0B3Gux3YmOTf040DG8XH\n6P7+X9Jvcw/duOUT/hT4V/1rnCRbkjwnyRlJzk/y9P41/T7dfzI/+ulNrHvr5TNgXEu+N/OWeXGS\nc5JsBf4d3UmUS3kHXZ539u/pa4EbquqrdGMEtyV5eZJT+/f/Cf16S+Yl3aXFfiHJBroxifcAP0p3\nZGxP/x/53XSffcv527kaeFn/Wp9JdyLSpJzRt+0Our2Kr13Bc43yOfxj/efpaXTj/NN/jp6ygu3P\nEvPaMa/3NWRef73f3sVV9ZUVbHcsq1UU76UbJ/K1qjp64h/dIflfz+KX+vgw3Rn8R5OcOHzzKrqB\n/x9Pt+v+Q3QnIo3q1XQD5r9Dd4WB99zPsg+iC9i36A6f3AH8fv/Ym4AL0u3y/19V9UO6b3s76QaV\nfwN4I903qROuoTs08i3gN4DnV9U9/WPb6Qb8/5Sq+gTdVRj+oG/3n/OTb64voPum9nXgvcDvLji0\ndOI5vki3V/yP+7b9Gt0lWH5wP/0f9bX4LbqrUXySbtjCf2K0v6PX052A8cEk36P7gvCE+19lcVX1\n3n677+z/Lj4LPGuJZb9GN47slX17b6Q7CfBkrqM79Pc3dK/B9xntsBT96/x8ui8t36Z7L95H9+FC\nVR2iO8P5v9C9xl+iG0MG3ckMV9G9b0fpTjS8cpTtrjPr5TNgLCd5b054O/BBupNEvkx3MslSz/ch\nujH+76bbo/P3gcv6x75HdzLsr9G914eBX+1Xvb+8PIzu6jbfpTtM++d0h2gfALyC7vPhm3Rj1P/1\nuK8BXfY/SHfll88A76f7gvjDZTzXQv+dLk+30Z0c9PHlPtGIn8Pz/TLdl8z30+0xvIuunzKvJ57P\nvN7XkHn9j3R76T+Zn1xP+Q3L3f6oTlzSQxOU5D8A/6CqXrjIY6fQDRh/zLwiWTMuyQ3AG6rqLUO3\nRcuX7kcf/sViX0ZnVZJn0f3tPvKkC0vriHnVuAb7medWVdUPquofWhDPtiS/kuRh/fCJvcBj6PY8\nS+taf2jz2f3f7tnA79IdmZK0zpjXybIollbH+XRHBL5NN3Tjn1TVkWGbJI0kdEPPvkV3OPYmumuL\nSlp/zOsETWT4RJJn0o2p2UB33derVvykklaNmZWmh3mV1saKi+L+jMq/oRuAfivdCVgvqKrPr7x5\nkibNzErTw7xKa2exM0rH9XjgSycumZHknXTXtVsysD+zdUOdu33TBDYtzYZP/dXd36iqh558yYkY\nK7PmVbqv9ZxXMLPSQqNmdhJF8dnc91JVt3KSS22du30Tn7hupdeXlmbHhm1fuvnkS03MWJk1r9J9\nree8gpmVFho1s2t2ol2SfUkOJTl0+x2TuHyepNViXqXpYmallZvEnuLbuO/PCp7DIj89XFX7gf0A\np89tr52/928msGlpVrxiLTd20szOz+uDsrWe8fBRfgRQasWX1nJjY/8fa2alhUbL7CT2FH8S2JHk\nvP6HKS6j+2UXSeuTmZWmh3mV1siK9xRX1b1JXkL3s7gbgDdX1edW3DJJq8LMStPDvEprZxLDJ6iq\n99P93rakKWBmpelhXqW14S/aSZIkqXkWxZIkSWqeRbEkSZKaZ1EsSZKk5lkUS5IkqXkWxZIkSWqe\nRbEkSZKaZ1EsSZKk5lkUS5IkqXkWxZIkSWqeRbEkSZKaZ1EsSZKk5lkUS5IkqXkWxZIkSWqeRbEk\nSZKaZ1EsSZKk5lkUS5IkqXkWxZIkSWqeRbEkSZKaZ1EsSZKk5lkUS5IkqXkjF8VJ3pzkeJLPzpu3\nNcn1SQ73t2etTjMljcvMStPDvErDG2dP8VuBZy6YdwVwsKp2AAf7aUnrw1sxs9K0eCvmVRrUyEVx\nVf1v4JsLZu8BDvT3DwCXTKhdklbIzErTw7xKw9u4wvXnqupIf/8oMLfUgkn2AfsANp3hESBpICNl\ndn5eT+P0NWqapAWW9X+smZWWZ2In2lVVAXU/j++vql1VtWvj5i2T2qykZbq/zM7P6yZOXeOWSVpo\nnP9jzay0PCstio8l2QbQ3x5feZMkrSIzK00P8yqtoZUWxdcCe/v7e4FrVvh8klaXmZWmh3mV1tA4\nl2R7B/Ax4Pwktya5HLgKuDjJYeCiflrSOmBmpelhXqXhjXyiXVW9YImHdk+oLZImyMxK08O8SsPz\nF+0kSZLUPItiSZIkNc+iWJIkSc2zKJYkSVLzLIolSZLUPItiSZIkNc+iWJIkSc2zKJYkSVLzLIol\nSZLUPItiSZIkNc+iWJIkSc2zKJYkSVLzLIolSZLUPItiSZIkNc+iWJIkSc2zKJYkSVLzLIolSZLU\nPItiSZIkNc+iWJIkSc2zKJYkSVLzLIolSZLUvJGL4iTbk3wkyeeTfC7Jy/r5W5Ncn+Rwf3vW6jVX\n0ijMqzRdzKw0vHH2FN8LvLKqLgCeCLw4yQXAFcDBqtoBHOynJQ3LvErTxcxKAxu5KK6qI1X16f7+\n94CbgLOBPcCBfrEDwCWTbqSk8ZhXabqYWWl4G5ezUpJzgccCNwBzVXWkf+goMLfEOvuAfQCbzvDo\nj7RWVprX0zh99Rsp6cfMrDSMsU+0S/JA4N3Ay6vqu/Mfq6oCarH1qmp/Ve2qql0bN29ZVmMljWcS\ned3EqWvQUklgZqUhjVUUJ9lEF9a3VdV7+tnHkmzrH98GHJ9sEyUth3mVpouZlYY1ztUnArwJuKmq\nXjfvoWuBvf39vcA1k2uepOUwr9J0MbPS8MYZU/wU4DeAv05yYz/vd4CrgKuTXA7cDFw62SZKWgbz\nKk0XMysNbOSiuKr+D5AlHt49meZImgTzKk0XMysNz1+0kyRJUvMsiiVJktQ8i2JJkiQ1z6JYkiRJ\nzbMoliRJUvMsiiVJktQ8i2JJkiQ1z6JYkiRJzbMoliRJUvMsiiVJktQ8i2JJkiQ1z6JYkiRJzbMo\nliRJUvMsiiVJktQ8i2JJkiQ1z6JYkiRJzbMoliRJUvMsiiVJktQ8i2JJkiQ1z6JYkiRJzbMoliRJ\nUvNGLoqTnJbkE0n+b5LPJXl1P39rkuuTHO5vz1q95koahXmVpouZlYY3zp7iu4GnV9WFwE7gmUme\nCFwBHKyqHcDBflrSsMyrNF3MrDSwjaMuWFUF/L9+clP/r4A9wNP6+QeAjwKvmlgLpSk398d/uej8\nYy998qpt07xK08XMSstz3ddvXHT+Mx6+c+znGmtMcZINSW4EjgPXV9UNwFxVHekXOQrMLbHuviSH\nkhy69647x26opPFMKq/3cPcatVhqm5mVhjVWUVxVP6yqncA5wOOTPHrB40X3zXaxdfdX1a6q2rVx\n85ZlN1jSaCaV102cugatlWRmpWEt6+oTVfVt4CPAM4FjSbYB9LfHJ9c8SStlXqXpYmalYYxz9YmH\nJjmzv78ZuBj4AnAtsLdfbC9wzaQbKWk85lWaLmZWGt7IJ9oB24ADSTbQFdNXV9X7knwMuDrJ5cDN\nwKWr0E5J4zGv0nQxs9LAxrn6xF8Bj11k/h3A7kk2StLKmFdpeSZ5Jvs4zKw0PH/RTpIkSc2zKJYk\nSVLzLIolSZLUPItiSZIkNc+iWJIkSc2zKJYkSVLzLIolSZLUPItiSZIkNc+iWJIkSc2zKJYkSVLz\nLIolSZLUPItiSZIkNc+iWJIkSc2zKJYkSVLzLIolSZLUPItiSZIkNc+iWJIkSc3bOHQDpFl37KVP\nHroJkiTNpGc8fOfEnss9xZIkSWqeRbEkSZKaZ1EsSZKk5o1dFCfZkOQzSd7XT29Ncn2Sw/3tWZNv\npqTlMK/S9DCv0rCWs6f4ZcBN86avAA5W1Q7gYD8taX0wr9L0MK/SgMa6+kSSc4DnAK8BXtHP3gM8\nrb9/APgo8KrJNE/ScplXaXyTPJN9HOZVGt64e4r/EPht4Efz5s1V1ZH+/lFgbrEVk+xLcijJoXvv\nunP8lkoa10Tyeg93r3IzJbGCvIKZlSZh5KI4yXOB41X1qaWWqaoCaonH9lfVrqratXHzlvFbKmlk\nk8zrJk5drWZKYuV57R83s9IKjTN84inA85I8GzgNeFCSPwOOJdlWVUeSbAOOr0ZDJY3FvErTw7xK\n68DIe4qr6sqqOqeqzgUuAz5cVS8ErgX29ovtBa6ZeCsljcW8StPDvErrwySuU3wVcHGSw8BF/bSk\n9cm8StPDvEpraKyrT5xQVR+lOwuWqroD2D25JkmaJPMqTQ/zKg3HX7STJElS8yyKJUmS1DyLYkmS\nJDXPoliSJEnNsyiWJElS8yyKJUmS1DyLYkmSJDXPoliSJEnNsyiWJElS8yyKJUmS1DyLYkmSJDXP\noliSJEnNsyiWJElS8yyKJUmS1DyLYkmSJDXPoliSJEnNsyiWJElS8yyKJUmS1DyLYkmSJDXPoliS\nJEnNsyiWJElS8zaOs3CSrwLfA34I3FtVu5JsBf4HcC7wVeDSqvrWZJspaVzmVZouZlYa1nL2FP9q\nVe2sql399BXAwaraARzspyWtD+ZVmi5mVhrIJIZP7AEO9PcPAJdM4DklrQ7zKk0XMyutkXGL4gI+\nlORTSfb18+aq6kh//ygwt9iKSfYlOZTk0L133bnM5koaw0Tyeg93r0VbJZlZaVBjjSkGnlpVtyX5\nWeD6JF+Y/2BVVZJabMWq2g/sBzh9bvuiy0iaqInk9UHZal6ltWFmpQGNtae4qm7rb48D7wUeDxxL\nsg2gvz0+6UZKGp95laaLmZWGNXJRnGRLkjNO3Af+MfBZ4Fpgb7/YXuCaSTdS0njMqzRdzKw0vHGG\nT8wB701yYr23V9UHknwSuDrJ5cDNwKWTb6akMZlXabqYWWlgIxfFVfUV4MJF5t8B7J5koyStjHmV\npouZlYbnL9pJkiSpeRbFkiRJap5FsSRJkppnUSxJkqTmWRRLkiSpeRbFkiRJap5FsSRJkppnUSxJ\nkqTmWRRLkiSpeRbFkiRJap5FsSRJkppnUSxJkqTmWRRLkiSpeRbFkiRJap5FsSRJkppnUSxJkqTm\nWRRLkiSpeRbFkiRJap5FsSRJkppnUSxJkqTmjVUUJzkzybuSfCHJTUmelGRrkuuTHO5vz1qtxkoa\nnXmVpouZlYY17p7i1wMfqKpHARcCNwFXAAeragdwsJ+WNDzzKk0XMysNaOSiOMmDgV8G3gRQVT+o\nqm8De4AD/WIHgEsm3UhJ4zGv0nQxs9LwxtlTfB5wO/CWJJ9J8sYkW4C5qjrSL3MUmJt0IyWNzbxK\n08XMSgMbpyjeCPwi8CdV9VjgThYcxqmqAmqxlZPsS3IoyaF777pzue2VNJqJ5fUe7l71xkoys9LQ\nximKbwVuraob+ul30QX4WJJtAP3t8cVWrqr9VbWrqnZt3LxlJW2WdHITy+smTl2TBkuNM7PSwEYu\niqvqKHBLkvP7WbuBzwPXAnv7eXuBaybaQkljM6/SdDGz0vA2jrn8S4G3JTkF+ArwIrrC+uoklwM3\nA5dOtomSlsm8StPFzEoDGqsorqobgV2LPLR7Ms2RNCnmVZouZlYalr9oJ0mSpOZZFEuSJKl5FsWS\nJElqnkWxJEmSmmdRLEmSpOZZFEuSJKl5FsWSJElqnkWxJEmSmmdRLEmSpOZZFEuSJKl5FsWSJElq\nnkWxJEmSmmdRLEmSpOZZFEuSJKl5FsWSJElqnkWxJEmSmmdRLEmSpOZZFEuSJKl5FsWSJElqnkWx\nJEmSmmdRLEmSpOaNXBQnOT/JjfP+fTfJy5NsTXJ9ksP97Vmr2WBJJ2depeliZqXhjVwUV9UXq2pn\nVe0EHgf8HfBe4ArgYFXtAA7205IGZF6l6WJmpeEtd/jEbuDLVXUzsAc40M8/AFwyiYZJmhjzKk0X\nMysNYOMy17sMeEd/f66qjvT3jwJzi62QZB+wD2DTGR79kdbQivJ6GqevegMl3YeZlQYw9p7iJKcA\nzwP+58LHqqqAWmy9qtpfVbuqatfGzVvGbqik8U0ir5s4dZVbKekEMysNZznDJ54FfLqqjvXTx5Js\nA+hvj0+qcZJWzLxK08XMSgNZTlH8An5yWAfgWmBvf38vcM1KGyVpYsyrNF3MrDSQsYriJFuAi4H3\nzJt9FXBxksPARf20pIGZV2m6mFlpWGOdaFdVdwIPWTDvDrozZSWtI+ZVmi5mVhqWv2gnSZKk5lkU\nS5IkqXkWxZIkSWqeRbEkSZKaZ1EsSZKk5lkUS5IkqXkWxZIkSWqeRbEkSZKaZ1EsSZKk5lkUS5Ik\nqXkWxZIkSWqeRbEkSZKaZ1EsSZKk5lkUS5IkqXkWxZIkSWqeRbEkSZKaZ1EsSZKk5lkUS5IkqXkW\nxZIkSWqeRbEkSZKaZ1EsSZKk5o1VFCf5zSSfS/LZJO9IclqSrUmuT3K4vz1rtRoraXTmVZouZlYa\n1shFcZKzgX8L7KqqRwMbgMuAK4CDVbUDONhPSxqQeZWmi5mVhjfu8ImNwOYkG4HTga8De4AD/eMH\ngEsm1zxJK2BepeliZqUBjVwUV9VtwH8GvgYcAb5TVR8E5qrqSL/YUWBusfWT7EtyKMmhe++6c4XN\nlnR/JpnXe7h7TdostczMSsMbZ/jEWXTfWM8DHg5sSfLC+ctUVQG12PpVtb+qdlXVro2bt6ygyZJO\nZpJ53cSpq95eqXVmVhrexjGWvQj426q6HSDJe4AnA8eSbKuqI0m2AcdP9kQXPOx2PnHlf1tWg6VZ\ntOGPJv6UE8vrzz3m77juuhsn3kBpWm3YtipPa2alVTJqZscZU/w14IlJTk8SYDdwE3AtsLdfZi9w\nzRjPKWl1mFdpuphZaWAj7ymuqhuSvAv4NHAv8BlgP/BA4OoklwM3A5euRkMljc68StPFzErDSzdE\naY03mtwO3Al8Y803Ppyfwf7OspX295FV9dBJNWaS+rzeTFvvaUt9Bfs7rnWbV/D/2Aa01FeYTH9H\nyuwgRTFAkkNVtWuQjQ/A/s62FvrbQh9PaKmvYH9nUQt9nK+l/rbUV1jb/vozz5IkSWqeRbEkSZKa\nN2RRvH/AbQ/B/s62FvrbQh9PaKmvYH9nUQt9nK+l/rbUV1jD/g42pliSJElaLxw+IUmSpOZZFEuS\nJKl5gxTFSZ6Z5ItJvpTkiiHasJqSbE/ykSSfT/K5JC/r529Ncn2Sw/3tWUO3dVKSbEjymSTv66dn\nua9nJnlXki8kuSnJk2a8v+Z19t5T8zqj/YXZzmyLeQUzu1b9XfOiOMkG4L8CzwIuAF6Q5IK1bscq\nuxd4ZVVdADwReHHfxyuAg1W1AzjYT8+Kl9H9JOkJs9zX1wMfqKpHARfS9Xsm+2teZ+897ZnXGe1v\nA5ltMa9gZtemv1W1pv+AJwHXzZu+Erhyrduxxn2+BrgY+CKwrZ+3Dfji0G2bUP/O6f9Inw68r583\nq319MPC39Cepzps/q/01r7P3nprXGe1v35+mMjvree37Y2bXqL9DDJ84G7hl3vSt/byZlORc4LHA\nDcBcVR3pHzoKzA3UrEn7Q+C3gR/NmzerfT0PuB14S38o641JtjC7/TWvnVl6T83r7PYXGspsI3kF\nM7tmmfVEu1WU5IHAu4GXV9V35z9W3dedqb8eXpLnAser6lNLLTMrfe1tBH4R+JOqeixwJwsO48xY\nf5thXjuz0teeeZ1RLeQVzCxrnNkhiuLbgO3zps/p582UJJvoAvu2qnpPP/tYkm3949uA40O1b4Ke\nAjwvyVeBdwJPT/JnzGZfodvrcmtV3dBPv4suwLPaX/PKTL2n5nW28woNZLahvIKZXdPMDlEUfxLY\nkeS8JKcAlwHXDtCOVZMkwJuAm6rqdfMeuhbY29/fSzcWaqpV1ZVVdU5VnUv3Xn64ql7IDPYVoKqO\nArckOb+ftRv4PDPaX8zrTL2n5nXm8wozntmW8gpmljXO7CC/aJfk2XRjZDYAb66q16x5I1ZRkqcC\nfwH8NT8ZA/Q7dOOergYeAdwMXFpV3xykkasgydOA36qq5yZ5CDPa1yQ7gTcCpwBfAV5E9wVzVvtr\nXmfsPQXzyoz2F2Y7s63mFcwsa9Bff+ZZkiRJzfNEO0mSJDXPoliSJEnNsyiWJElS8yyKJUmS1DyL\nYkmSJDXPoliSJEnNsyiWJElS8/4/YPIv0eT4TCAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6a1e72bd68>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gym.make(\"Pong-v0\")\n",
    "env.render(mode='rgb_array').shape\n",
    "env.reset()\n",
    "print(env.action_space)\n",
    "print(env.unwrapped.get_action_meanings())\n",
    "\n",
    "top = 32\n",
    "bottom = 195\n",
    "left = 14\n",
    "right = 146\n",
    "downsampled_height = int(np.rint((bottom-top)/2))\n",
    "downsampled_width = int(np.rint((right-left)/2))\n",
    "input_dim = downsampled_height*downsampled_width\n",
    "\n",
    "def preprocess(img, reshape=False):\n",
    "    #crop, grab only one channel, and downsample by factor of 2\n",
    "    img = img[top:bottom,left:right,0][::2,::2] \n",
    "    #get rid of background color, 109 in first frame, 144 otherwise\n",
    "    img[np.isin(img,[144,109])] = 0 \n",
    "    img[img!=0] = 1\n",
    "    if not reshape:\n",
    "        return img.astype(np.int).ravel()\n",
    "    else:\n",
    "        return img.astype(np.int)\n",
    "\n",
    "def reshape(img):\n",
    "    return img.reshape(downsampled_height,downsampled_width).astype(np.int)\n",
    "\n",
    "#what color pixels are in this image?\n",
    "#print(list(zip(*np.unique(env.render(mode='rgb_array')[top:bottom,left:right,0],return_counts=1))))\n",
    "#print(list(zip(*np.unique(env.render(mode='rgb_array')[top:bottom,left:right,0][::2,::2],return_counts=1))))\n",
    "\n",
    "plt.subplots(2,3, figsize=(12,10))\n",
    "\n",
    "plt.subplot(2,3,1)\n",
    "plt.title(\"The Atari Pong Game Screen\")\n",
    "plt.imshow(env.reset())\n",
    "\n",
    "plt.subplot(2,3,2)\n",
    "plt.title(\"Cropped, First Channel Only\")\n",
    "plt.imshow(env.render(mode='rgb_array')[top:bottom,left:right,0])\n",
    "\n",
    "plt.subplot(2,3,3)\n",
    "plt.title(\"Prior Plus Downsample\")\n",
    "plt.imshow(env.render(mode='rgb_array')[top:bottom,left:right,0][::2,::2])\n",
    "\n",
    "plt.subplot(2,3,4)\n",
    "plt.title(\"After a step, color scheme changes\")\n",
    "plt.imshow(env.step(2)[0][top:bottom,left:right,0][::2,::2])\n",
    "\n",
    "plt.subplot(2,3,5)\n",
    "plt.title(\"After Preprocessing Frame 1\")\n",
    "plt.imshow(reshape(preprocess(env.reset())))\n",
    "\n",
    "plt.subplot(2,3,6)\n",
    "plt.title(\"After Preprocessing Frame 2\")\n",
    "plt.imshow(reshape(preprocess(env.step(2)[0])))\n",
    "plt.show()\n",
    "\n",
    "#print(list(zip(*np.unique(env.render(mode='rgb_array')[top:bottom,left:right,0][::2,::2],return_counts=1))))\n",
    "#print(list(zip(*np.unique(reshape(preprocess(env.reset())),return_counts=1))))\n",
    "#print(list(zip(*np.unique(reshape(preprocess(env.render(mode='rgb_array'))),return_counts=1))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define and initialize the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession(config=config)\n",
    "\n",
    "x = tf.placeholder(tf.float32, shape=[None, input_dim])\n",
    "advantage = tf.placeholder(tf.float32, shape=[None])\n",
    "action_is_down = tf.placeholder(tf.float32, shape=[None])\n",
    "\n",
    "h1_dim = 200\n",
    "l1 = tf.layers.dense(x, h1_dim, activation=tf.nn.relu)\n",
    "#use tf.squeeze() to reshape this from [batch_size,1] to [batch_size]\n",
    "logit = tf.squeeze(tf.layers.dense(l1, 1))\n",
    "sampled_action_negative_log_prob = tf.nn.sigmoid_cross_entropy_with_logits(logits=logit, labels=action_is_down)\n",
    "reward_weighted_neg_likelihood = advantage*sampled_action_negative_log_prob\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)\n",
    "train = optimizer.minimize(tf.reduce_sum(reward_weighted_neg_likelihood))\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "tf.global_variables_initializer().run()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set up an agent class that plays pong using actions chosen by the neural network in the active TensorFlow session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class pong_agent:\n",
    "        \n",
    "    def clean_slate(self):    \n",
    "        self.wins = 0\n",
    "        self.games = 0    \n",
    "        self.p_list = []\n",
    "        self.actions = []\n",
    "        self.frames = []\n",
    "        self.frame_changes = []\n",
    "        self.rewards = []\n",
    "\n",
    "    def make_batch(self, n_sets):\n",
    "        self.clean_slate()\n",
    "        for _ in range(n_sets):\n",
    "            self.play_set()\n",
    "        self.normalize_rewards()\n",
    "        return self.frame_changes, self.actions, self.rewards\n",
    "\n",
    "    def play_set(self):\n",
    "        env.reset()\n",
    "        done = 0\n",
    "        self.frames.append(preprocess(env.render(mode='rgb_array')))\n",
    "        self.frame_changes.append(self.frames[-1] - self.frames[-1])\n",
    "        while not done:\n",
    "            done = self.play_point()\n",
    "\n",
    "    def play_point(self):\n",
    "        frames_played = 0\n",
    "        discount = 0.99\n",
    "        while True:\n",
    "            prob, action, reward, new_frame, done = self.play_frame(self.frame_changes[-1])\n",
    "            self.p_list.append(prob)\n",
    "            self.actions.append(action)\n",
    "            frames_played+= 1\n",
    "            if not done:\n",
    "                self.frames.append(new_frame)\n",
    "                self.frame_changes.append(self.frames[-1] - self.frames[-2])\n",
    "            if reward:\n",
    "                self.rewards+= [reward * discount**k for k in reversed(range(frames_played))]\n",
    "                self.wins+= max(reward,0)\n",
    "                self.games+= 1\n",
    "                break\n",
    "        return done\n",
    "\n",
    "    def play_frame(self, frame_change):\n",
    "        p_down = 1/(1+np.exp(-sess.run(logit, feed_dict={x:np.array([frame_change])})))\n",
    "        #sample an action using p_down, 3=down, 2=up\n",
    "        action = np.random.binomial(1, p_down) + 2 \n",
    "        observation, reward, done = env.step(action)[:3]\n",
    "        return p_down, action, reward, preprocess(observation), done\n",
    "\n",
    "    def normalize_rewards(self):\n",
    "        mean = np.mean(self.rewards)\n",
    "        std_dev = np.std(self.rewards)\n",
    "        self.rewards = (np.array(self.rewards)-mean)/std_dev"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the agent for 3000 updates to reach >50% win rate\n",
    "\n",
    "This part takes a while. My setup processes the earlier batches at a rate of ~100 batches/hour. To monitor progress (the agent's win rate), I output at a .PNG plot every 10 batches. There's a visible shift in the win rate from ~2% to ~4% by batch 300. Later batches take longer to process because the agent is playing more games.\n",
    "\n",
    "To speed up and improve quality in the later stages of training, we could trim the number of games played (because each point takes more frames to complete as the agent gets better), and add a decaying learning rate, respectively."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "ratios = []\n",
    "matches_per_batch = 10\n",
    "epochs = 3001\n",
    "agent = pong_agent()\n",
    "\n",
    "for i in range(epochs):\n",
    "    #play Pong with the network, save frames and associated rewards\n",
    "    frame_changes, actions, rewards = agent.make_batch(matches_per_batch)\n",
    "    train.run(feed_dict={x:frame_changes, action_is_down:np.array(actions)==3, advantage:rewards})\n",
    "    ratios.append(agent.wins/agent.games*100)\n",
    "    if i%10==0:\n",
    "        print(\"{}: batch {} finished after {} hours\".format(time.strftime('%X %x '), \n",
    "                                                    i, round((time.time()-start)/3600,2)))\n",
    "        plt.title(\"Agent Quality over Time\")\n",
    "        plt.plot(range(1,i+2), ratios)\n",
    "        plt.xlabel(\"Number of Updates\")\n",
    "        plt.ylabel(\"Percent of Games Won\")\n",
    "        plt.savefig(\"./pong_agent_quality\")\n",
    "        if i%100==0:\n",
    "            #save out the neural network's weights here\n",
    "            saver.save(sess, \"./pong_agent.ckpt\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Restore the saved variables and let the agent play Pong!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sess.close()\n",
    "tf.reset_default_graph()\n",
    "\n",
    "agent = pong_agent()\n",
    "\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "\n",
    "x = tf.placeholder(tf.float32, shape=[None, input_dim])\n",
    "advantage = tf.placeholder(tf.float32, shape=[None])\n",
    "action_is_down = tf.placeholder(tf.float32, shape=[None])\n",
    "\n",
    "h1_dim = 200\n",
    "l1 = tf.layers.dense(x, h1_dim, activation=tf.nn.relu)\n",
    "#use tf.squeeze() to reshape this from [batch_size,1] to [batch_size]\n",
    "logit = tf.squeeze(tf.layers.dense(l1, 1))\n",
    "sampled_action_negative_log_prob = tf.nn.sigmoid_cross_entropy_with_logits(logits=logit, labels=action_is_down)\n",
    "reward_weighted_neg_likelihood = advantage*sampled_action_negative_log_prob\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)\n",
    "train = optimizer.minimize(tf.reduce_sum(reward_weighted_neg_likelihood))\n",
    "\n",
    "saver = tf.train.Saver()\n",
    "saver.restore(sess, \"./pong_agent.ckpt\")\n",
    "\n",
    "number_of_frames_to_play = 400\n",
    "frame = env.reset()\n",
    "new_frame = np.zeros_like(frame)\n",
    "diff = np.zeros_like(preprocess(np.copy(frame)))\n",
    "img = plt.imshow(frame)\n",
    "for i in range(number_of_frames_to_play):\n",
    "    action = agent.play_frame(diff)[1]\n",
    "    new_frame = env.render(mode='rgb_array')\n",
    "    diff = preprocess(np.copy(new_frame))-preprocess(frame)\n",
    "    frame = new_frame\n",
    "    img.set_data(new_frame)\n",
    "    display.display(plt.gcf())\n",
    "    display.clear_output(wait=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save a GIF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import moviepy.editor as mpy\n",
    "global frame, new_frame, diff\n",
    "frame = env.reset()\n",
    "#resetting a couple of pixels that turned black for some reason\n",
    "frame[0][:8]=frame[0][8]\n",
    "new_frame = np.copy(frame)\n",
    "diff = np.zeros_like(preprocess(np.copy(frame)))\n",
    "def make_frame(t):\n",
    "    global frame, new_frame, diff\n",
    "    action = agent.play_frame(diff)[1]\n",
    "    new_frame = env.render(mode='rgb_array')\n",
    "    new_frame[0][:8]=new_frame[0][8]\n",
    "    diff = preprocess(np.copy(new_frame))-preprocess(np.copy(frame))\n",
    "    frame = new_frame\n",
    "    return frame\n",
    "clip = mpy.VideoClip(make_frame, duration=25)\n",
    "clip.write_gif(\"Pong.gif\",fps=15)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}