Keras Deep Q-Network with Convolutional layers to solve a numpy implementation of the FrozenLake OpenAI gym environment.

keras_deep_q_frozen_lake.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Q-Network Route Finder\n",
    "\n",
    "Here in this jupyter notebook, we will have some fun with using reinforcement learning to find safe routes across the ice.\n",
    "This notebook draws a lot of inspiration from Open AI's [FrozenLake](https://gym.openai.com/envs/FrozenLake8x8-v0/) gym environment.\n",
    "Really, how nice of them to have a well crafted toy problem already thought out for us!\n",
    "\n",
    "Specifically and technically, we will train a Deep Q-Network in Keras/Tensorflow to help us decide safe routes across slippery ice!\n",
    "Later, we will move on to applying this safe navigation tool to our crevasse avoidance problem.\n",
    "\n",
    "References:\n",
    "- Open AI [gym](https://github.com/openai/gym) \n",
    "- [Simple Reinforcement Learning with Tensorflow Part 0: Q-Learning with Tables and Neural Networks](https://medium.com/emergent-future/simple-reinforcement-learning-with-tensorflow-part-0-q-learning-with-tables-and-neural-networks-d195264329d0)\n",
    "- [Simple Reinforcement Learning with Tensorflow Part 4: Deep Q-Networks and Beyond](https://medium.com/@awjuliani/simple-reinforcement-learning-with-tensorflow-part-4-deep-q-networks-and-beyond-8438a3e2b8df)\n",
    "- [Frozen Lake github gist using Keras](https://gist.github.com/ceshine/eeb97564c21a77b8c315179f82b3fc08)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python       : 3.6.5 | packaged by conda-forge | (default, Mar 29 2018, 23:19:37) \n",
      "Numpy        : 1.14.2\n",
      "Keras        : 2.1.5\n",
      "Tensorflow   : 1.4.1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'/device:GPU:0'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import all the necessary libraries\n",
    "import os\n",
    "import sys\n",
    "import warnings\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"   # see issue #152\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"  #set to see only the first gpu\n",
    "os.environ['KERAS_BACKEND'] = 'tensorflow'\n",
    "warnings.simplefilter(action = \"ignore\", category = FutureWarning)\n",
    "\n",
    "#import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import random\n",
    "%matplotlib inline\n",
    "\n",
    "import keras\n",
    "import tensorflow as tf\n",
    "\n",
    "print('Python       :', sys.version.split('\\n')[0])\n",
    "print('Numpy        :', np.__version__)\n",
    "print('Keras        :', keras.__version__)\n",
    "print('Tensorflow   :', tf.__version__)\n",
    "tf.test.gpu_device_name()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Design Keras Model to fit into Q-Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set seed values\n",
    "seed = 42\n",
    "random.seed = seed\n",
    "np.random.seed(seed=seed)\n",
    "tf.set_random_seed(seed=seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import Conv2D, Flatten, Dense, Input, SeparableConv2D\n",
    "from keras.layers.core import Activation, Flatten, Dropout, Reshape\n",
    "#from rl.agents.dqn import DQNAgent\n",
    "#from rl.policy import EpsGreedyQPolicy\n",
    "#from rl.memory import SequentialMemory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def keras_convnet(input_shape=(4,4,2)):\n",
    "    \"\"\"\n",
    "    The Keras Convolutional Network that parses the incoming pixel data.\n",
    "    Consists of 3 Convolutional Layers, and 2 Dense Layers, plus 1 softmax output layer\n",
    "    Note that we use SeparableConv2D layers ala MobileNets for memory efficiency\n",
    "    \n",
    "    Inputs:\n",
    "    input_shape -- shape of the input tensor, (height, width, channels)\n",
    "    \n",
    "    Outputs:\n",
    "    model -- keras.models.Model instance\n",
    "    \n",
    "    \"\"\"\n",
    "    inp = Input(shape=input_shape)\n",
    "    \n",
    "    X = SeparableConv2D(filters=32, kernel_size=(4,4), strides=(1,1), padding='same', activation='relu', use_bias=False)(inp)\n",
    "    X = SeparableConv2D(filters=64, kernel_size=(3,3), strides=(1,1), padding='same', activation='relu', use_bias=False)(X)\n",
    "    X = SeparableConv2D(filters=64, kernel_size=(2,2), strides=(1,1), padding='same', activation='relu', use_bias=False)(X)\n",
    "    X = Flatten()(X)\n",
    "    X = Dense(units=256, activation='relu', use_bias=False)(X)\n",
    "    X = Dense(units=4, activation='softmax')(X)\n",
    "    \n",
    "    model = Model(inputs=inp, outputs=X) \n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize Deep Q-Network parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 4, 4, 2)           0         \n",
      "_________________________________________________________________\n",
      "separable_conv2d_1 (Separabl (None, 4, 4, 32)          96        \n",
      "_________________________________________________________________\n",
      "separable_conv2d_2 (Separabl (None, 4, 4, 64)          2336      \n",
      "_________________________________________________________________\n",
      "separable_conv2d_3 (Separabl (None, 4, 4, 64)          4352      \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 256)               262144    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 4)                 1028      \n",
      "=================================================================\n",
      "Total params: 269,956\n",
      "Trainable params: 269,956\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# DQN model parameters\n",
    "model = keras_convnet()\n",
    "print(model.summary())\n",
    "#policy = EpsGreedyQPolicy(eps=0.1)\n",
    "#nb_actions = env.action_space.n\n",
    "#memory = SequentialMemory(limit=10000, window_length=4)\n",
    "\n",
    "# DQN runtime parameters\n",
    "def sum_squared_loss(yTrue, yPred):\n",
    "    return K.sum(K.square(yTrue - yPred))\n",
    "loss = sum_squared_loss\n",
    "optimizer = keras.optimizers.Adam(lr=0.001)\n",
    "\n",
    "# Compile keras model\n",
    "model.compile(loss=loss, optimizer=optimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pure numpy implementation of Open AI Frozen Lake, plus a Keras Deep Q-network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper function to convert integer to a 2D grid location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 0.],\n",
       "       [0., 1., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def int_to_grid(integer, grid_shape=(4,4)):\n",
    "    \"\"\"\n",
    "    Function to map an integer to it's 2D grid location\n",
    "    \n",
    "    E.g. to place the integer 5 in a 4x4 (height*width) grid\n",
    "    \n",
    "            0000\n",
    "    5   ->  0100\n",
    "            0000\n",
    "            0000\n",
    "            \n",
    "    Steps:\n",
    "    1) np.prod(grid_shape) i.e. 4*4=16\n",
    "    2) np.identity(16)[integer:integer+1] i.e. output something like [0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0]\n",
    "    3) np.reshape(grid_shape) i.e. output the grid form of the above\n",
    "    \"\"\"\n",
    "    return np.identity(np.prod(grid_shape))[integer:integer+1].reshape(grid_shape)\n",
    "int_to_grid(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The player step left, down, right, up dynamics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def player_step(old_position:np.array, action:int):\n",
    "    \"\"\"\n",
    "    Take a step somewhere, unless you'll go through the wall!\n",
    "    \n",
    "    Inputs:\n",
    "    old_position -- a (height, width) shaped numpy array of 0s with a 1 marking the player position\n",
    "    action -- an integer from 0 to 3 representing either Left, Down, Right, Up\n",
    "    \n",
    "    Outputs:\n",
    "    new_position -- a (height, width) shaped numpy array of 0s with a 1 marking the player position\n",
    "    \n",
    "    Example taking a step Right:\n",
    "    \n",
    "           old_position                        new_position  \n",
    "    array([[0., 0., 0., 0.],            array([[0., 0., 0., 0.],\n",
    "           [0., 1., 0., 0.],    ----\\          [0., 0., 1., 0.],\n",
    "           [0., 0., 0., 0.],    ----/          [0., 0., 0., 0.],\n",
    "           [0., 0., 0., 0.]])                  [0., 0., 0., 0.]])\n",
    "    \"\"\"\n",
    "    actionKeys = {0:\"Left <-\", 1:\"Down v\", 2:\"Right ->\", 3:\"Up ^\"}\n",
    "    position = np.argmax(old_position)\n",
    "    \n",
    "    assert(len(old_position.shape) == 2)  #numpy array shape must be (height, width) i.e. 2 dimensional\n",
    "    \n",
    "    if actionKeys[action] == 'Left <-':\n",
    "        #print('Going Left')\n",
    "        #move left unless you are at the left wall\n",
    "        new_pos = position - 1 if (position)%map_grid.shape[1] != 0 else position\n",
    "    \n",
    "    if actionKeys[action] == 'Down v':\n",
    "        #print('Going Down')\n",
    "        #move down unless you are at the bottom wall\n",
    "        new_pos = position + map_grid.shape[0] if position + map_grid.shape[0] < np.prod(map_grid.shape) else position\n",
    "    \n",
    "    if actionKeys[action] == 'Right ->':\n",
    "        #print('Going Right')\n",
    "        #move right unless you are at the right wall\n",
    "        new_pos = position + 1 if (position+1)%map_grid.shape[1] != 0 else position\n",
    "\n",
    "    if actionKeys[action] == 'Up ^':\n",
    "        #print('Going Up')\n",
    "        #move up unless you are at the upper wall\n",
    "        new_pos = position - map_grid.shape[0] if position - map_grid.shape[0] > 0 else position\n",
    "    \n",
    "    assert(new_pos in range(np.prod(map_grid.shape)))  #final check to see that we are within the grid range\n",
    "    new_position = int_to_grid(new_pos)                #change our \n",
    "    \n",
    "    return new_position"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize some variables and train the Deep Q-Network!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Environment variables, replicated from https://gym.openai.com/envs/FrozenLake-v0/\n",
    "actionKeys = {0:\"Left <-\", 1:\"Down v\", 2:\"Right ->\", 3:\"Up ^\"}\n",
    "'''\n",
    "SFFF       (S: starting point, safe)\n",
    "FHFH       (F: frozen surface, safe)\n",
    "FFFH       (H: hole, fall to your doom)\n",
    "HFFG       (G: goal, where the frisbee is located)\n",
    "'''\n",
    "map_grid = np.array([[0.0, 0.0, 0.0, 0.0],\n",
    "                     [0.0, 5.0, 0.0, 5.0],\n",
    "                     [0.0, 0.0, 0.0, 5.0],\n",
    "                     [5.0, 0.0, 0.0, -5.]], dtype=np.float)  #note that our map is inverse coded so -ve values are good and +ve values are bad!\n",
    "\n",
    "\n",
    "#Hyperparameters\n",
    "epsilon = 0.75       #Exploration setting between 0 and 1, where 0 is don't explore and 1 is explore please!\n",
    "gamma = 0.99         #Future discount factor between 0 and 1, where 0 means care only about now, and 1 means I care alot about the future!\n",
    "\n",
    "#Session parameters\n",
    "num_episodes = 2000\n",
    "rewardList = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Begin training Deep Q-Network!!\n",
      "Step: 500, with epsilon: 0.6099999999999999\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGbdJREFUeJzt3W2oZVd5B/D/47xlxolmkrkZ00wmN+IUDNWq3KZKCtpUJVFJoQhNaKu0wkBpIYLFmgaEfiktgkltpe20ih+UpoiKIVjiGEekUKN3kokmnbxMJJo46ty8OUkmmbf79MPd595z1tnr3J2z197Ps9b5/2C455575pxnr732c9Zea6+1RVVBRETleJV1AERElBYTOxFRYZjYiYgKw8RORFQYJnYiosIwsRMRFYaJnYioMEzsRESFYWInIirMRosP3blzp87Pz1t8NBFRtg4dOvSUqs6t9zqTxD4/P4/FxUWLjyYiypaI/KTJ69gVQ0RUGCZ2IqLCMLETERWGiZ2IqDBM7EREhWFiJyIqDBM7EVFhmNhrHHzoOI4999LY8//72NO468Ff4Hs/ftogqrw98cxJfPeRJeswKLF7fvw0jh5/3jSGU2fP4cuLT8DDbT5/9txLOPjQceswbCYoefenX/gBdmzbhPs++d6R52/89++tPn7879/fd1hZe+enDmJZWW6l+cP9K8eE5X79zN2P4rMHH8P2LRtx3ZsuMYsDAK677bs48fJZ83rOFnvEsyfPWIdQlGX7xhQVaun5UwCAEy/bH7MnXj5rHQIAJnYiouIwsRMRFYaJnYioMEzsRESFYWInIioMEzsRUWGY2ImICsPETkRUGCZ2IqLCMLEHPKw3QUTUBhN7gHmdiHLHxE5EVBgm9gAb7ETUlnWXLhN7wHqHEFH+rNNIssQuIhtE5D4RuTPVe1pgWieitqzzSMoW+00AjiR8PyKiLFmf+SdJ7CKyG8D7AfxHivezZH0KRUTUVqoW+20APg5gOdH7mVHzkygiyp11Fmmd2EXkAwCOq+qhdV63T0QWRWRxacnvTY3ZYieitqzzSIoW+9UArheRxwHcDuAaEfli+CJV3a+qC6q6MDc3l+BjiYh8sj7zb53YVfVmVd2tqvMAbgDwbVX949aRERE1MGgdW7eSh1nHwuvYA9Y7hIiorY0p30xVvwPgOynfs2/Wp1ClU1WIiHUYVJBBdWK1WsMWe4At9m6xfGkWWNdzJvYA8063WL40C6zP/JnYiYgKw8QesJ4KXDqWL80C62rOxB5g2ukWy5dmgXU9Z2IPWH/Tlo7lS7PA+syUiZ2IKDHr9gsTe8h6jxTO+moBolnAxB5g4ukWu2JoFljXcyb2gPUOIaICMLH7wrzeLX5x0iywPvNnYiciSsy6AcPEHrC+TKl01i0ZolnAxB5g2ukWvzdpFlhXcyb2ABNPt1i8NAusz/yZ2APsKuiWdYUn6oN1LWdiJ6Ks8dZ445jYQ44qR4lYvETdY2IPMPF0y7olQ+XxeGs86y5dJvZALPGwbzgRFiPNAnbF+BL7pmVeT8O6JUPUB+tazsTekPWOIqJ6HDwdx8QesN4hpWP5EnWPiT0QyzthHzv73KfDUiuHl2OAg6fjmNgDscoaPuukTmfHSzKg9rgr46zLhom9oXBHsU7TrOMxEGddNkzsgejljmBXTAostXJ4OQZ8Dp6yK4ZmiKeDj6hUTOyB+ASl4PfuQymS9aASpeNlT7ocPGUfuy9NE4/1jssWy60YPAb8YmJvaLzFzlpNs43HQJz1l17rxC4il4nIQRE5IiIPishNKQKzYr1DSsfiJerexgTvcRbAx1T1XhE5H8AhETmgqv+X4L17F52gNHZVTPexlIjlVg7uyzjrs5nWLXZV/bmq3ls9fh7AEQCXtn1fK9EJSqzESVhXeKI+WOeLpH3sIjIP4K0A7kn5vn2Kt9iD35mfpsJyKwf3ZZx10SRL7CKyHcBXAHxUVU/U/H2fiCyKyOLS0lKqj+3N2Fox5ruOyJaXY4ATlMYlSewisgkrSf1LqvrVuteo6n5VXVDVhbm5uRQf2wlPlaNELF6i7qW4KkYAfA7AEVX9dPuQrHERsC5Zt2QoHS+70uUEJePPT9FivxrAnwC4RkQOV//el+B9TXDmabe8JANqz9uu9FS3rGNpfbmjqv4PAEfflR0JE7v1niMy5u0YYDRrOPM00Pg69u5DIXLNyzHgc/DU9vOZ2APWO6R0LF+i7jGxB2KXcI31sTNBTcXLJXLUnpdjYDBo6qluWUfCxB6I32hjvSeoCS/JgBJwti891S3rWJjYG+IEJaJR3o4BT9FYlw0Te6Bpi936GzlXLLZyeDkGVuPwEhDsQ2FiD1h/05bO2yVyRCViYg9wglK3WG7l8LIv1wZP/bBuvzCxNzS+HrunapQPFls5vB0DnsKxPvNnYm+KLXaiEV6OgbUJSl4isv+SYWIPcPC0ayy4Ung7BpyFY4qJPWB9ClU6b8mAqERM7IHmg6fMUNNgqZXDyzGwOnjqIxwA9rEwsTc0VokdVSIiE86OAU/hWH/pMbEHoqs7Mq8nYd2SoXS87EoOno5jYg94qhwlsm7JEM0CJvZAfD324Hfmp6mw3MrhZV96uiXegHXRMLEH4oOnXAQsBS/JgNrzdgx4qlvWZ/5M7A1xPXaiUd6OAU9fNNaRMLGPabZLrHdcrjwdfNSOlz3JW+ONY2IPWO+Q0rF8ibrHxB5ofLkjMxTNOC/HgMfVHa2jYWIPxNeKCVd37CGYArHcyuFtX3qKxzoWJvaGrHcUEU3mafzGOhIm9kDs9JLXsafh6eCjdrwcAxw8HcfEHnBUN4pkXeGJZgETe4ATlLrFUiuHl2PA5cxTTlDyJVZZ2RWThnWFp3S87UpPdcs6Eib2hri6I9EoL8cA+9jHMbGHojuEN7NOgaVWDm/HgKdorLupkiR2EblWRB4WkaMi8okU72nFU+UokbNcQFSk1oldRDYA+CyA6wBcCeBGEbmy7ftaaX5rPJoOS64UXvakx1vjWRdOihb7VQCOquqPVfU0gNsB/H6C93WFg6dEo7wdA9bdH8OsI0mR2C8F8MTQ709Wz2UpelXM2NPWuy5Pw+Woqrj1wCN48tmTdgFRCz6Ogdjg6dcP/wzffWSp/4BqYulbisRedxXp2GaJyD4RWRSRxaUlm8JugmvFdGu42B5begH/ePej+PMv3msWD03P2zEQhnPT7Yfxoc9/3yiW/AdPnwRw2dDvuwEcC1+kqvtVdUFVF+bm5hJ8bDec1dXiDCeD5erxy2fO2QRDVKgUif0HAPaKyBUishnADQDuSPC+JqJrxXDwNAlvl8jR9LzsydU4HNUt61A2tn0DVT0rIn8J4C4AGwB8XlUfbB2ZM7w1HtEoL8fAah+7bRgjrGNpndgBQFW/AeAbKd7LWvRGG2Efu/muyxNLrRxejoFBHF6+aAD7M1POPA05qhwl8nTwEZWKiT3Q9HJHJqjpeGnlUXtujoHVrhgvAdm3D5nYA00rq5tKnRuWWzG8HAODMLzEA8C8njOxNzR+VYynWkTUPy/HwKA/20c0K6zLhok9wAlK3WKxlcPLMeCxxW4dCxN7wFHdKJJ1hSeaBUzsgaYTlGg61qeoVB71OHjKFnseuLoj0Sgvx4COPbBnHQoTeyA6QYk3s05idHXH6qdNKNSSl2PA5eApJyj5Eh88bfY6mkxHHrMQc+blGFgbPHUSEOy/ZJjYx1jvkrINH3yDh3XrPhPR9JjYA7w1XreGy23ZUQuLXrmRsy/LfRm50YYl61iY2BsLr2N3VIsyxSLMW93Zl0kc8NfHbh0NE3sgPnja7HW0jprBU8qTRh73LXZrPEvWsTCxBzh42q3hAVMOnuYtvH+tdRye6pN1JEzsAU+Vo0R1t8YjorSY2ANNB0/tv5PzNJrYWYZ505pHFlF4vNGG7eczsTc0NkHJUSXKFcswb3WTzazj8ML6zJ+JPRAdPG34OprMzSVy1JqXyWYuJyixxe5L00XArHdcrkYukTOMg9rz1mL3VJ+sY2Fip16NTFDi6ClRJ5jYGxq/0QaTUlsswbz5OQY8Dp6yj92V6P7gBKUkeFVMOUbHS8zCcHkduzUm9kCscnCCUio+pqFTeyN97C4GT81CGGMdCxN7wHqHlM7LgBuVzborxBoTe6D56o6zXXGmxdUdy6FOzr7qbrRhXbWs8wMTe0NjO4o5qTUWYea09mHv6rpirOuW9RcLE3uAqzt2q27wlGWZJy+TzdY+ei0G67NBJnZnohOUxl7XfSwlGj19ZyHmzEsLua7Fbp7YTT+diX2M9Q4pXd3gKW+NR6nNepuBiT0UHTwNJijxK2Aqo4OnZmFQAu4GTx1dcWV9NtoqsYvIp0TkIRH5oYh8TUQuSBWYN+yKSc+68lM7I7vPwa70dBMX6+Jo22I/AOA3VPXNAB4BcHP7kGxFKwQHT5MYTuZssefNzeqOOvoTcFC3ch48VdVvqurZ6tfvAdjdPiRb8Vvjca2Y9FiGOfN8M2vr49P6jCFlH/ufAfjvhO9ngqmmW3WtKpY5pWbeYje2cb0XiMi3ALyu5k+3qOrXq9fcAuAsgC9NeJ99APYBwJ49e6YKtg/NZ55SU6NrsPu51pja8dLFXtcVY95iN67a6yZ2VX33pL+LyIcBfADA7+mE0lTV/QD2A8DCwkJ2R/TYlmW3BXbW+7Lk5Y6ZcpJI61Z3tE6s1ulh3cQ+iYhcC+CvAbxTVU+mCclW49UdzXddPmJreLDFnrfRq1AcxOGobllX7bZ97P8M4HwAB0TksIj8a4KYTMVbl+HgaQ/BFMLLIBul5eW68bpb41lXM+uGX6sWu6q+IVUgXlhXiNKNTlBiaVM3Zr1uceZpQ5ygNL2mC6tRXvzdaMPJ4jWwr9tM7KHYImC8KmZqsasVZv2StNxp9Jee1XTFWNct66rNxB6I7xBOUJpWbJBt1k+Xczd6GathHDU3szavWzmvFVMiXseeXnRNERZi1mJXO/XN5+CpLSb2AFvi3eIEJerDsnVfjDEm9oY4eJoeizBvrgdPjVmHwsQeaH4Fh59K5F3semfeGi93PuYn1N3MevhssK+ErwafGcPEHohdwTG+umNfEeUvPnjafyyUjperC3XswWjd6utY9VIeABP7mNiAEAdPpxctx+oXrhWTp9Fd6W2tmP6v2PEymAwwsY+xPoUq3ejgqWEgVLTlkdZ7/10x1pjYJ5h0+ZSjfehefNyChZgzN2vF1MZg2/9vXbOZ2CeYNBhivchPTmKLgLHFnjc3x0DNzaxNWuwjIXHw1JWmAyBsbDbH+Ullctdit26lO6rQTOwBN62QUtWcBbHEKbXhVnp/LXY/NZmJfQJeFZNG7LjizNO8jZ59ObgqJtL9YtHlZ121mdgDsdl049exMyk1Fune4q3x8ublBiqri4ANPxeZj9JpHE5m4gJM7GOaXsdOzcX6Pjl4Wg7TCUo1LXbremadL5jYA41Xd2RSaqzpbF7Ki0WreHIckbrFmafEZNOtujMiljilZnO5o5+azMQ+waRvYE870btYSc360qq5i60B1H8c1c/o4Gn/0Vmf0TOxB6KDp+EEJeakxmKDbIOHHDzNk5cxqLrLZi26RTh4momJLXYm9sZik5J4uWPeYlP4rcRmitvMPO3lI6OY2APRVeF4HfvUooOn7GPPmpdEtu6t8XobPPVTk5nYA472TfFWT6FZ6JTY8PhNX0M5nmoxE/sEk2+04Wk3+hbrb9TgJ+XFYs3z2jgGE5RiE+EMorPOD0zsgdjpHJcUaCE6cWT8gKR8eO6KsVhSwMtgMsDEPia2c8b2E5NRY7HW0+CA4yBqppxcBbJ2uWP91Ve9XVbLCUp+xWasjbfYrXddPqJfljr+HOXDepnctc8e/3Cbm2v4KA+AiZ0MDQ5IttgpNesJStaY2APNV3fsK6L8xWYo1s0YpHx46VOuq0cW/f/FTVASkb8SERWRnSnez1J09hoHT6cWXXVvmZc75sz6ypOhDx+LweZGG0OPc++KEZHLALwHwE/bh+NA5BKucD9Z77icxBIAL3fMm+cWu8VAppfLP4E0LfZbAXwc9tuSxOi3brzmWp9q5SR+M2v2sefMyzGwNtFt7bnlkTrXf4vduuXXKrGLyPUAfqaq9yeKh2bIoO5zkUdKbXTZXrs4rGxc7wUi8i0Ar6v50y0A/gbAe5t8kIjsA7APAPbs2fMKQuxXfPZa/HU0WfzmJZyglDN3XTGR1VhtBk9trZvYVfXddc+LyJsAXAHgfhEBgN0A7hWRq1T1FzXvsx/AfgBYWFiw3u6o2LWoHDxNY3TVvfHnKB9eBk/r5kNY32jDukqvm9hjVPVHAC4e/C4ijwNYUNWnEsRlpul67OZ7LiPx2bxVi73neCgRg1ZxbRi19cggttIudywJk0y3hsuXSwpQV2xa7H5M3WIPqep8qvdyY0Kfmaed6F10dcfBVTGzOLpVgEljUH1azdvRq2L6jcfqM4exxR5oOnvNesflZN21YvoNhxKJ3UDFKo7o+FhPNczT4CkTeyBeOYLXMbM3Fhtk47K9efM0IQeIX8fe3+KOPsYcACb2cQ0HTz1U5FxEb2bNq2KyNumMtk91N7Ou+3v3cQw95uApzaq1wVPbOKg8Fi12T5jYA01bIWxkNhcbZFu7TI2FmaPRY8Cwj33wMzpganBVDLtifIn1G3KC0vQ0UpBcUiBvfrpiqp9Dz1ksKeBpzIGJPRAb6efNrNuor/Brg6csyxx5SWR1N7Me6YrpKbN7uUoIYGIfE2uFMPdMb93LHVm22fPWYrdYtnfk49kVQ7OKy/ZSV3hrPBoRXYmw4etoXOw69poJg5QRL10Pq58c6xrqrY+994+MYmIPNF/d0XrX5SPeFcMJSjmL3cu29zhqB085QYmGRFd3HBs87Sui/EVvZu2kxUfT8TMGNd5AsF9SgIOnbnE99jRi5Tjrk0hyF+ti6z2O1Rb7cIvZosU+HpMVJnYyY7G0Ks2GWa9bYnEKvLCwoIuLi6/4//3T3Y/ijvuPdRDRmuPPn8KvXjoDALj8om3YvGHlu++ZF0/j6RdPr75u5/bN2LFtc6exlOLU2WX89JmTAIAd2zZh5/YtAIBjz72EF0+fAwC84eLtELMIaRrPnjyNp15YOSYuvWArtm3eYBLHY0svYFmBzRtfhcsv3FbFdgZPvXAKAPBrrz0Pr96SbIXyqNPnlvGTp1fq+Wu3bsLF52+pfd3f/cGb8FvzF071GSJySFUX1ntd91ub0Nz5W7B31/ZOP2Pvru3Y9Zrz8NzJMzh19tzI33a95jxs3vAqnD63jF+eeLnTOEqzcPkOvGbrJhx/fq3c9u7ajksv2IpfnDiFc8vLhtHRtC569Ra8fOYcXjx91iyGX991PnZfuBVPVI2HgR3bNuPsOcXzp870Fsvb9uzABds2TcwPWzd1/wWYVWK/4ao9uOEqvzfCJiLygH3sRESFYWInIioMEzsRUWGY2ImICsPETkRUGCZ2IqLCMLETERWGiZ2IqDAmSwqIyBKAn0z533cCeCphODngNs8GbvNsaLPNl6vq3HovMknsbYjIYpO1EkrCbZ4N3ObZ0Mc2syuGiKgwTOxERIXJMbHvtw7AALd5NnCbZ0Pn25xdHzsREU2WY4udiIgmyCqxi8i1IvKwiBwVkU9Yx5OKiHxeRI6LyANDz10oIgdE5NHq547qeRGRz1Rl8EMReZtd5NMRkctE5KCIHBGRB0Xkpur5YrcZAETkPBH5vojcX23331bPXyEi91Tb/V8isrl6fkv1+9Hq7/OW8U9LRDaIyH0icmf1e9HbCwAi8riI/EhEDovIYvVcb/U7m8QuIhsAfBbAdQCuBHCjiFxpG1UyXwBwbfDcJwDcrap7Adxd/Q6sbP/e6t8+AP/SU4wpnQXwMVV9I4C3A/iLal+WvM0AcArANar6mwDeAuBaEXk7gH8AcGu13c8C+Ej1+o8AeFZV3wDg1up1OboJwJGh30vf3oHfVdW3DF3a2F/9VtUs/gF4B4C7hn6/GcDN1nEl3L55AA8M/f4wgEuqx5cAeLh6/G8Abqx7Xa7/AHwdwHtmbJu3AbgXwG9jZbLKxur51XoO4C4A76geb6xeJ9axv8Lt3F0lsWsA3AlASt7eoe1+HMDO4Lne6nc2LXYAlwJ4Yuj3J6vnSrVLVX8OANXPi6vniyqH6nT7rQDuwQxsc9UtcRjAcQAHADwG4DlVHdw0dHjbVre7+vuvAFzUb8St3Qbg4wAGN7W9CGVv74AC+KaIHBKRfdVzvdXvnO55WncT+1m8pKeYchCR7QC+AuCjqnpCpG7TVl5a81yW26yq5wC8RUQuAPA1AG+se1n1M+vtFpEPADiuqodE5F2Dp2teWsT2Bq5W1WMicjGAAyLy0ITXJt/unFrsTwK4bOj33QCOGcXSh1+KyCUAUP08Xj1fRDmIyCasJPUvqepXq6eL3uZhqvocgO9gZYzhAhEZNLKGt211u6u/vxbAM/1G2srVAK4XkccB3I6V7pjbUO72rlLVY9XP41j5Ar8KPdbvnBL7DwDsrUbUNwO4AcAdxjF16Q4AH64efxgr/dCD5z9UjaS/HcCvBqd3uZCVpvnnABxR1U8P/anYbQYAEZmrWuoQka0A3o2VQcWDAD5YvSzc7kF5fBDAt7XqhM2Bqt6sqrtVdR4rx+u3VfWPUOj2DojIq0Xk/MFjAO8F8AD6rN/WgwyvcEDifQAewUq/5C3W8STcrv8E8HMAZ7Dy7f0RrPQt3g3g0ernhdVrBStXBz0G4EcAFqzjn2J7fwcrp5o/BHC4+ve+kre52o43A7iv2u4HAHyyev71AL4P4CiALwPYUj1/XvX70ervr7fehhbb/i4Ad87C9lbbd3/178FBruqzfnPmKRFRYXLqiiEiogaY2ImICsPETkRUGCZ2IqLCMLETERWGiZ2IqDBM7EREhWFiJyIqzP8DkM98xsttJyEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1000, with epsilon: -5.369999999999974\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJztnXmwJld53p/3ziJpJFlipGEQmhEjzMSx2PE1xhGpYLGUDBhCFa5AJTZVIZ5yle1AFS4ZGZcrW5WTSgKUjZN4Krj8B9hQbEalQLCMhJ14kRgtIwkPQhIepNGIzGgZScMw6z354+u9z+k+29d9+/Tzq5rp7tNn6+/e+/T7ve9ZRCkFQggh6bAydgcIIYTEhcJOCCGJQWEnhJDEoLATQkhiUNgJISQxKOyEEJIYFHZCCEkMCjshhCQGhZ0QQhJj4xiNXn755WrXrl1jNE0IIZPlzjvvfEIpta0v3yjCvmvXLuzbt2+MpgkhZLKIyPds8tEVQwghiUFhJ4SQxKCwE0JIYlDYCSEkMSjshBCSGBR2QghJDAo7IYQkxijj2JfJ/77/cfzkrq247KLzrPLf/t0nsfXCzdi9/WLvNpVS+OJdj+GtL78CF2ze4Fz+9Nk1/Ok9j+Hnf2IHRMS7H318+/vP4vjJs1jdtRUA8MyJM/jLB4/i5175Qqvya2sKn7/rEN716iuxcUXwxbsew9tecQXO36R/5u8ePY7vP3sS/+hHLzfWefO9h7Fpwwp+dNtFuHTLJvzx7Y/gZ37s+Xj5jkuKPF+973EcePxZvGT7xbhg0wa8+Zrt+Ow3H8GaAjasCN7wY9vw2TsexZlzawCAS7dsxoYVwZPHT9l+NIQMxrteswNXX37hUttIStiPnTiNX/7UXXjlzkvx5V+51qrMP9v7twCAg//xbd7t/vXDT+JDn9uPux99Gv/hn77cufwnbn0Qv3vrQ7hg0wZrkfXh+o//HwDls37ws3fjtgeO4mVXXmL1i3bT/sO44fP34vFjJ/GaF12KD31uP/YfOoZ/986XafNf91//otZek0eePIFf/eO7i+t//86X4qO3fAd3/P1T+NS/+qki/cYv3YdjJ84U1/f+m7fgN75wX3G967ItOPjkCW0bS3xPEuLFa170PAq7C2fOLTbmfuxp/R/5snju5FkAwJFn/SzEo8dPAwCePXmmJ2dcDh87CQA4dfacVf5jJxb9fOoHp3A88JkB4GSj3bNrStufc+dU5/WR5xZ9uP0334g7/v4p/NqfLF4W/+XnX4l3/8QO7/4RMlXoYyfrBqUM6X3XhnKEzBUKO5kcyqDkgrrrhV4YMlco7GTdYDK8m0LetuCzFAGkIuf0r5O5QmEn6waTJd5MXWsKfaHrUrfYKexkplDYybqnqfet6+G6QsgkoLCTdYM5eFq/0bTYC0+M1P3qQi87mSkUdjI5TC+AVvCUuk5mCoWdrBualnmR3uN6ycstZu1SzQmhsJN1g+049rU1U/C0abFT5Mk8obCTdYMxCMrgKSFOUNjJusE3eJoPk2wHTwmZJxR2MjlMFvpiHDsnKBFCYSfrBuvgqWkmKoc7EgIgMWE3CQOZBtbBU4PPXYTDHQkBIgq7iGwQkbtF5OZYdTpDXU+SloXO5RwJ6SSmxf4BAAci1kdmhq1gm33sXN2RECCSsIvIDgBvA/A/Y9TnC+24NLFdf11EuLojIYhnsX8cwA0A1iLV5wW/oU8bo4+9kd5aKyZDiv9qKYTMjmBhF5G3AziilLqzJ98eEdknIvuOHj0a2qwWBk+nje1Pzyjs0n1NyFyIYbFfC+AdInIQwGcAXCcin2pmUkrtVUqtKqVWt23bFqHZNrTYp43u56fzu3f9nKnlhEQQdqXUjUqpHUqpXQDeA+BWpdS/CO5ZUJ9Gaje0/Fj9dmxXIU48o700gLLqj3l1x8YEpZDOETJhEhvHTlJDa8UbftKtJQXoiyEzZWPMypRS3wDwjZh1OrYPYDzfamizo/Xbsd1Yi+M2261ONCrSNOWaE5RMdVLWyVxJy2KnyT5p8h9f9eeo87EzeEpIN0kJO5k4mWBXZVsn4V0Tmbg+DCEUdjJBuoOnlWtqPJkpSQk7XTHTwrRhRtUi1wdP9XAza0IWpCXsHBfjxXjDQ/Vb3KmOPEB7a7yc1sxT6jqZKWkJO3Xdi/Uyfl43jl3Xt65RMYSQxISd+DHWN532BhrZsa+ccRx7YxGwgL4RMmWSEnYacjPBGDxtbrRBaSfzJC1hpy/Gi3XjYy9OuoOnJlcMN7MmZEFawj52B4gTNq4YbfDUOEFJGtchvSNkuqQl7FR2L9bL52YfPO2YoEQ1JyQtYSd+rJfgKQqLvbs/XXfra8VQ5Mk8SUzY14npSaKh/YlqEnNBF00aIXMjKWFfLy6FqbHegqc+i4DlGs7VHQlJTdjH7sBEGetzawdPNT52TTndqBitb53KTmZKWsJOZfdi2cNETfW3XOy6UTGOwVOqOSGJCXsOt8ZzbNcjv0sZ03M1BV+3CJhtfXpXDEWezJOkhJ2LgCWIZfSUwVNCStISds3WakMy1a3xXL8puG6NZ6re5Iqp59EFT3V9WvSIm1kTkqiwE1dG8rF7r+7Y4YupJtFkJzMlLWGnK8aLZb8QzdWb1mNXhhwLOjezdukYIYmSlLDnTCUI2So/kX7HC566pZf3LcexU+XJTElK2OmKSY9cxFd6RLoMntLHTkhSwp7D4Kkbyw+e2o5jV630/Hyl8uHoZ562e0SLncyVpISdFrsfy5+gZJeuG8euG+m0ttauq7DYa2JOZSfzJClhd4UbcywYb0kBU/C0TdUi50+NkG6SEnbdcLnO/JEVYrLBU8d2owVPW9dtZc/TqpY4g6eEdJOWsI9mytGGXBrZR1v1sWuHsWf3GTwlJDVhz462llosOS78wIH1jBY8dfwkXLtpDJ62VnfM81fLZm1WfewdFnstjSY7mSlpCbujyR7Lxz55e93jAVw+OrMrpn8RMN1Ls3OCErWckLSE3ZXJC3IkfD4HFyvfmNPCYs+puWK0WyhlB260QUi4sIvIThG5TUQOiMi3ROQDMTrmw3ijO0ZqeAbog6ftfEXwtOpjp7KTmbIxQh1nAXxIKXWXiFwM4E4RuUUp9XcR6nbCeXRHJEGe+ho1Pp+DmyvGboJSnqJbBGxlpRo81S3bm6/uWEmjzU5mSrDFrpR6XCl1V3b+HIADAK4MrdezN465I/nYp63rXp+D03BHU7rRFdNeBKzPx67dGY+6TmZKVB+7iOwC8GoAt8es15bxLPZpYZoYFFJHd15Duml1x5rFnrtiupcUyKGWExJR2EXkIgBfAPBBpdSzmvt7RGSfiOw7evRorGa1TGWVxFb5gfptmspvXd65Qdt+mCeYrdj62KnshMQRdhHZhIWof1op9UVdHqXUXqXUqlJqddu2bTGabbexlFot2p26L2YdU64V072kQHmfwVNCYoyKEQCfBHBAKfXR8C7547o1Xmw9nsoEJdOqirYIHIOn1qs7muuoW+yWSwrQMUNmSgyL/VoAvwDgOhG5J/v31gj1OuM8QWmmwdOWj92nDpdx7EZXjMsEpZ5lexk8JaQgeLijUur/YqIxq6kJcixaj73s4Y6W6Z2rO/b42It89t0iJFmSmnlaWnxu+cPbVVHqGy946vpNJ8449tbMU9049ixtpcfHnkt61RdPi53MlbSEfazRMDO1/IdAu9FGhyumquX0sZO5kpawa6afd+aPtQiYZqEqr3qCe2LbTtg4dkGkCUptk72Vnp/1LturqZ8WO5krSQm7qzLGc8XEqmgYaTfN+HSrI0bwtHGtSS8nKJVpa5qpp7qt8ajrZK6kJeyOrDcXymjj8Jdcxn49dnOsoirSXW3T/UJIYsLuGjyNpaRdguRWT3hffNqxtb5V9RjBF9MaFVP0p53W74rRLAJGjSczJS1hn+hSAlNhjM/XNXjaSF1KnwhZ76Ql7K7B02gm++IQHDwdysdu2Lmov9wC0dRhU66VblyMrJquGe5ovTWedRcJSYq0hN01eBpN1yONrolSi0U7nsFT3YxQn/aKdMO1bj32usXerku/Hjsh8yQpYc8ZeoJSrPoG87FbpPSVHz542rM1Xp6PZjohaQm7qy7GHsceXE+cavrbibIee4y8epdQ7ZtBdvRZUoAiT+ZKWsI+UvSUwdPlt1n1sXftoERXDCGpCXt2tA+eRmp3csHT7mtzuSw4jW53SF97RbqhI/XQaRY8rfym6vc8bddPg53MlaSE3ZX1NkFpKOLMPHXJa/Cxt66Vse66j918v5qPk5XIXElL2HOLzzp4Gms0y7QmKJlWVewtVrGonYKnplExhheMdj32no02cmilE5KYsEcbl+7a7kws/zEeU7c1XqePXZNGyNxIS9g11l13gUjtZsfw1R1HmqBkPY59cRSXQh790PrY+2aeBveGkHRIUtit84/V8HKrcW7HNXjqUkbXnrkfbWXXBaa1PniNyU6LncyVpITdlfXmQhnJxe41GmeZm1lrg6M9SwoU+Wi7E5KWsOumpHfnj7sUwFSCp77DKmvB0yjrsfdPUMqpumL0o2KyY81ip8iTeZKWsI81QWmdWf4poQ+easdDVg+tc0LmRFrCnh3tt8aL1G6+y09oPYMFTxvX1sHT6gQl//bM9bfz64On7bJc3ZGQkrSEfaTgaewZrMvGGLR0KBdlglIruT1BqQye9izbW6zuyAlKhCQl7K6M5boZm9DNrBd1hOc1D3fsXjLAdhEwQuZKYsJunpKuzR3NFVNtPaSeoUz2ZrtuxeIFTw31q3Za/9Z42ZHDHQlJS9i5Nd5yGWd1x8yvb7k1Xn2tGELmSVrCnh2HttSiBU/HMdi9VncMazFLbX1zUK3cOotdHzzVLe/o0EVCEiItYXcNnq4zU3uwCUoGQXUpt8yt8bQzT3sWASvu19Zjp7KTeZKUsOcMPUGprC+w/GAWu35ikH15t8/OGDw17OSkDZ5a1AfQr04IkJiwuwp17OBpcD1DjWP39MVUg5tDbWadp65YLylQOafIk5kSRdhF5HoReUBEHhKRD8eo04fxgqfrzKezLEb4gO2X7W2rOHWdzJVgYReRDQB+H8DPArgGwHtF5JrQen1wnnkaq91iEk2cepZN21K29LFX8ruNY7cz2buDp2Va17K9tQlKNNnJTIlhsb8WwENKqe8qpU4D+AyAd0aod+mstwlKwwVPwyYoxXPFmEbL9ExQ0pTRbrRh1z1CkiOGsF8J4NHK9aEsbXAKi8/RZ5xz5twafucrB/DMiTNu7Rrqc65hqM2sDRODbMs5B09tJyhpJnrplhToappGOiFxhF33p9T60xORPSKyT0T2HT16NEKz4TSF5X/d+zj+4C+/i9/56oGgenz7Mdb3B9dRRM4Wu+167Nq1YrLgaeU31XqCEkWezJQYwn4IwM7K9Q4Ah5uZlFJ7lVKrSqnVbdu2RWi2TajAns2icqfPrbm1GyjJ68wjZGSUmafZsW/ZXt2YdY5jJ3MlhrB/E8BuEblaRDYDeA+AmyLU60wxM9L67zmOUoUGT3WW6jLxXt1Rc+bTnrEfmnx5Wu/MU6kfFxf2fSQkJTaGVqCUOisivwrgawA2APhDpdS3gns2AOvNUl7vm1lX88cIPHf1QynVWIJXn093v0ijsJOZEizsAKCU+gqAr8SoK6wf9WNv/tjt+5YbNnYaHjyNNSqm45uDUgth1n8L65igRDEnJLGZp2NNUApseIhuL8PCHgRLV0yu6FzdkZDUhD07Dr81XtZuYPllyqZujXPtzc5K8kOcCUqtUTGaPup+pl0TlGppNN/JTElL2B2VOpYFGlrLEMHT+thwZbxnU0csV0zzhq6PunHstsFTyjqZK0kJuyvzDZ42rl2Dp3CcoGSb3vGtom/Z3iKfda8ISZekhL1qUVrlj+yK8a6u6XdYAlUx9F+P3XOCkiFzX/C0mrbSo9j6tWLs+0hISiQl7GNN3QyeoBSpH8tuY5yt8RbHlb4JSkXwtJJG+53MlKSE3XWCUjQfe3DwNLOEo/TG1EbtCuarjjqKo6qd25Zr96ljHDvqn0lt5qlmYjDHsRNSkpSwuxLbAg0Ooo61CJhrs8qtkPPWeJoy9dUddRZ7Ox8hcyUpYQ+Z6KOUKoXVI5jo226M8nZtVPzXhvZ766h8PG59tvSxayrTbRTe1SaHOBKSmrC75lf6c/eGw6Kfg49j9w2eFsMylVOfQyz2PK2+NV67Lu0iYNR4MlPSEvZAZSysPUdBCB/HPg26FulaXqOLQ98OSrqfGYOnZK6kJewBwdMYBru3K8ZxgxC/NirnEUbxVMOnNvn1fbIJnuY/075le9vQYidzJSlhd0W3oUNQfcHlpxE8jTbztJmvo4zt1ni1NLtmCUmOpIQ9ZHXHIIu94nv2K58dhwqedkwM6qyjMizTpc+2E5SqCc36q26V9TZjmJD1RlrC7pq/YzamWz1+7ZcV+Ldt3USHK8b1RegcPDWmK2O+pnuq6mPXbnatDZ7SZifzJClhH8uUm4ILJkYL+uBpvL5r68+OKytVH3s7H10xhJQkJez537t98LR67i9Q4cHT/Lg8ge9cK8a2jsps0ObM0O627dJ1wexiHHvfsr06Yaeyk5mSlLC7Em0ce17HyOW923X2YcGps7brsVeXCmgFT3t87HTFEFKSlLC7W85xpDQ4eBpo8Vu1oWmvvGcbPC3z9wVPa5+FpcW+pimj+xbG2Ckh3SQm7G5/8vFmngaURXvM9jKIGzytj5Dpb89Un7kfzc1HeoOnNM4JKUhL2Cfa7iAx3xiupmUHT7WdXKT1LdtLCClJSthzhg6eFnWEWu4D6VWMfvbV0eX6MaV3fYOSvrViaLITUpCUsPvMoPQtW68nzJWiGsdloBtxkmuhvdVdPmdfidooHMsnW1PtPpYTlJq9qENZJ6QkLWHPj7Y+Y42QeLUbbbhjQCcs21ic1/3X9p9Xeewra2exd/jYGz78vuGOhJCStIR9ohOUhogOxGhBP4Eopo/d3Gbvsr002QkpSErYc7x87Ep5vxhiWezLFPiubyf2jpjSim6OWmm3119/28euccUUwdMyn+3qjoTMlSSF3Zam+ASPbgmsYazg6TJWd6wvOqbP3EytLhXA4Ckh/iQl7O6rO1bFpyJArkLXY732ly/7sCxU55Vdw7pvFjEt9nrw1PyZDrW8MSFTJS1hd1fk2rlucSmravzeB5XyYaNq7NrQn+uujXVU8jv12RhgNQdPmzNPa+PYK0sP5NBeJ6QkLWEP1MWi/MAqMYT9uaxx+jG/ZXRtZs2Zp4TYk5awZ0ffCUq+w+h8XThl+fpxKWhcIy4rNDbzl3UY8tbaM/jYm98cDOdAc+aprjYqOyE5QcIuIv9ZRL4tIveKyJdE5NJYHRuCpnti9OBpYPvW7bSGxbi1XN1ow5jHxg/fuK752FvBU33duvuEzJ1Qi/0WAC9TSr0CwHcA3BjeJX+Cgqcwj97or8et3djlXdpYtFMPTFo3W8mv0F3YalZv40ZtVExX8JSxU0I6CRJ2pdSfKaXOZpd/C2BHeJcC+uNo8zZnOgbPHPUrPnzwtHn0Cp7maQY3i+HclGdRb9tiL8ex9+ygZGiDkDkS08f+LwF8NWJ9zoRacmueCj2F4Xe6Provc6yrw7tLVnWVM0+7+0FXDCElG/syiMifA3iB5tZHlFJfzvJ8BMBZAJ/uqGcPgD0AcNVVV3l11ha/4GmE4Y6BM1eX+X7QuUaaQdTeOirH/uCp0p6b+tSsq/ltor7nqUbYabMTUtAr7EqpN3XdF5H3AXg7gDeqDmVTSu0FsBcAVldX14WJ2/zqX1x7akR48HUYfDfaKPPbBE/15139WOt4GfSu7khdJ6SgV9i7EJHrAfwGgH+ilDoRp0v+NIOCvflr5wE+9qJ93/J5v5foY9ddOHqelKYSG2vcZnYqAKytqda9IqVnSQFCSEmoj/0TAC4GcIuI3CMi/yNCn7xx/oNvmJXe66kHB0/Dytu1UR8BVDsGBU97MnfkagdPNXk0E5S0rhha7IQUBFnsSqmXxOpIDEKF0dfHHtryEBZol2ja17Hk4GlHWu+yvfSxE1KQ1MzTHPvgad2KDRapwAqGcjG0g5aOAm/htrKaoNTsR8cEpb5RMdR1QkqSEnb3IGD93H9JgezoVbriY/csH9yejyump89Wqzu2gqeV7foaAYCqRa4Nnhp7Tcj8SEvYO2YravPXxGf8jTaWGjzVBDO9Z55WfOymh66FL8zKXmNNqUKgWxOgGj52+tQJMZOWsLta7I2ywaNafIOvmv7ERreZtetonOpz9vXZZjNrXfA03zCj+d7onXlKpSekIC1hDywfOkHJm9GCp8PX0Vk/VM2Xvkhb0PSxN2Wcsk5ISVLCnmOrNc0hgKNvbTdY8FQ1rl3L95exccXo+lFY7I17zVExTQudBjshJWkJezFByda1UC2qSovd06UzlivHro32uauPvZq/r89WwdMuH3sjT33Z3raFTl0npCQpYXfV5WZAMX8huI6OiTZBaYkWu+pQWvsJSjVl7yzru5l1MSqmGTyt5Rvoqw0hEyUtYXcWSL1LwlU2QpcECLX4XdpYnNf767qZdS14ajnipau+KqXLRdXytF0x9XIMnhJSkpawB7oy1gqL3bnhIJY5zLFsQ5PmWodlWggrDYFWhvTmTFPKOiElSQl7jrWPvemKcSwfm6HWdW/P+HQv3/cZWQVPNc/bdMXkNEfL0MlOiJmkhN05GFg7Lzez9h0P7x88DSvv1oq/T78ePM3TbIKn/XlymsHTPFPL9dIqR2UnJCctYW+d9ORvWOxlbNA1eBo2qiU0+OrSRrUd99hAmT/GWjE6yuGO1RYBmuSE2JOWsAcqY+FjX3NsN6zZQRwwWv+4p8UeUkcfrQlKSp/eDp7G7QchUyYpYc+xtz8rViXKoGnoYmC+DOXab00Mci1vUUa3Nk1fP4AySNr89sPgKSH2JCXszmuf1MRHebtEgl0p5YpavjXYN1FpxdW3X83f535ShnNTnpyWK8bkY6fFToiRpIQdjgLbFJ8yeOroY8+P63ocuzl4aj+OPRfz/j77bGYN6CYo1dOLfP3dJWS2JCXszhaoaoudS/lmPb66PHTwtGmzu1rs1bGhRmtc8w2hXV/7jjTulUsKdEs5R8UQUpKWsAeavOUEJT+L3Zchxq/HCHzq64jbd+sJSlwEjBAjSQl7jp9QKv+Zp2UVQQw1Mao1Qcm1PBw/Y6O7pp22YjlBqTWOncJOSEFSwu468aa9CFh27tywZ7lGP5bqimmMAKodvYKn9bRW3porpj/AmtO0xI0vu5aQU9kJyUlL2BvH/vz64Y7uwVO/oGtR3vGFFNJGvT23T6w6EqavqM9m1oB5SQEObyTEnrSE3VHZ22LnFkxs1hPqwRnGEVN5EeXXHs8bYzNr3Z3mZtZl8LS7T3TFEFKSlrAHSmM+49R3PXZfRlvdcRLB00X9bZ86LXhCTCQl7Dmu64vnZZpWonO7ExD4RTvd1zbl+9eK6a9fHzwV/T1OUCLEmqSE3Tl42ijru6TAEFvbhaINnjouXlZ1v/S5j+r7yRqCpzofe6uPeXq3hc5x7ISUJCXsOdbB08YEJe9lewODn8MHT/187NWga1+frSx23U+qCJ52+/AJIWaSEvZCDGzXiqmdq8qwRVeLPW/fqVi9bY923dowJ9q/CNuFY25mDVQXAavn6dsKj64YQkqSEvZQgicokWBaOyVlcAMlQuxJStidhw02hjs2fc/W1QQHTePU091Ge1y5syumElzu73NrLGlXjoJm8LQYFdNjoXMza0JKogi7iPy6iCgRuTxGfb64B0/rGcvgqXPLWbt+yhzqynFpY3Fe76/rKCJVKWEz4sXFFdMsZR7HTiEnxESwsIvITgBvBvBIeHfCcN/Srn7uG7ALnqDkKLB+bbTPiyTHUUTVz6ovb7Ptep72jeZ67IQQd2JY7B8DcAPWwQAG3ciPzvy18+pIj6GDp2Hl3Vppf7NxdV0pVPtsETy1GTqTUSwC1ijbt+gXPTGElAQJu4i8A8BjSqn9kfozKr7L9pJ4NGee5vRttMFx7ISUbOzLICJ/DuAFmlsfAfCbAN5i05CI7AGwBwCuuuoqhy7a0/z632fFtV0x7XSrdkNdKYGuHKsmND5v560EK/n7rH3dhKh2njbNZXvLrvUFTw2NEDJDeoVdKfUmXbqIvBzA1QD2Z37RHQDuEpHXKqW+r6lnL4C9ALC6uroUDbNbeKqapy4+vsMdY7lilqns9ZdePTBp22w1v1Pw1JhHc6PwsdfjHW2LnWvFEGKiV9hNKKXuA/D8/FpEDgJYVUo9EaFfnlT9yAp9f+5NP3Cf37ivnskET5tHx+ApVH+frUbFaNJMSwoQQuxJaxx7oAg0LVnrcmHNDhJ11r6sPF1O9TTPDhloTlAyr+7YfU3InPG22JsopXbFqisGdq6Y+rnvImBFHcEvlrDy9g11XroWN+RpfnvS5NGOitEPd+xbppcTlAgpSdZitxLJxmxM/3HsTeeGa/mQ0pZt1M7rbhTXtXWsgqcWVeqDp/laMd3fnrgeOyFm0hL2WjC0X1lU42osiz10az2rNjQvvbDgqb3byi14mt906xshpCQtYXe02FvDHYsLv3b9g6dh5a3a0Aw/dH3eakBT9RSuB08NeTRpzeBpXlGvRU6TnZCCpIQ9lNyC5ASl8dBNUNK5z/uGPxIyZ5ISdn/feOZe8LScQ10pQ6zuiMa3k8XRcZhlJX9fn+vB0/4+5aysNPoIvTHOUTGEmElL2F1dMY2yvksKxHKlDB88za59XDGNOlp5a64YU33tO7nlXfXh60a8cIISIWbSEnbX4GntRaBKYV9zbDfQ4lYDmOydwVM3g72x/EK//9xmdmpOruHly5JuMUJcSUrYde4Gy+w1V4x7s4GuGE1/YtO1dovrZtbVOswWe/9LVi/s7a3x6IohxI20hD2QXGgYPB0P3dZ42uBp65rKTkhOUsLetMB78zcnKDmMza7XY99mZ/klvk9qdSvVenaXOlRlbKg5eGpou5anfWNFswiYTrS5mTUhZtIS9ppYOQZAAyYouY4H17VdPS6DLreTbav6CUr2bhZKlp+MAAAGhUlEQVSbPNpFwKwsdkJITlrCbjg35m8U8F1SIHSW5DAWe/PbSbv93jqq5Xufuf8lq0uV1sxTusUIcSUtYXcOntaDgWuFwLpb+z7lqm0vynsVd2pj0Y5q9NUyeFodx95T1OZn0R08Ld8cWmu8b7lHQmZMUsIeSjnzdOSOzBj/4CkhJCfasr1D8HtffxA37T9svP/9Z04W5+/6b3+FDT1W3NMnThfnN3z+Xjxx/BQA4NiJ03jzR//Cul+PPn0CAHDizDmncjlP/WDRj4eOHvcqb8MPTp0tzj9x28P41O2PFNd/8/CTVu1+78nFcz5x/DSOnTgDADh++qy27A/PnCvO/+ivD2p/boeP/bCVlgdPf+tL9+PC876No8dPQSDYsNIMljJ4SoiJSQn7tovPw+7tFxnv795+EV54yQU48twpnLWcZbT1ws04fXYNxzPh27l1Cw499UMn3+7u7Rdh5/O24NCxH3q5Y/7B9ouxY+sFePSpE85lXXj9eRuxacNK8UL7hy+4GFdccj4e0wisjt3bL6p9Pju3bsGhp83P/LoXX4YtmzcUL0xdfS+67EKsCHDk2VM4eXYNv/SPr8aPnL8Jz506U+R56QsvwUtf+CP412/cjVNnz+HRp07g9S/Zhi2bN+Dho8dx8MkTeMs15ba8n3zfKs6c49cuMl9kmUvFmlhdXVX79u0bvF1CCJkyInKnUmq1Lx997IQQkhgUdkIISQwKOyGEJAaFnRBCEoPCTgghiUFhJ4SQxKCwE0JIYlDYCSEkMUaZoCQiRwF8z7P45QCeiNidKcBnngd85nkQ8swvUkpt68s0irCHICL7bGZepQSfeR7wmefBEM9MVwwhhCQGhZ0QQhJjisK+d+wOjACfeR7wmefB0p95cj52Qggh3UzRYieEENLBpIRdRK4XkQdE5CER+fDY/YmFiPyhiBwRkfsraVtF5BYReTA7Pi9LFxH53ewzuFdEXjNez/0QkZ0icpuIHBCRb4nIB7L0ZJ8ZAETkfBG5Q0T2Z8/9b7P0q0Xk9uy5Pysim7P087Lrh7L7u8bsvy8iskFE7haRm7PrpJ8XAETkoIjcJyL3iMi+LG2w3+/JCLuIbADw+wB+FsA1AN4rIteM26to/BGA6xtpHwbwdaXUbgBfz66BxfPvzv7tAfDfB+pjTM4C+JBS6scBvA7Ar2Q/y5SfGQBOAbhOKfVKAK8CcL2IvA7AfwLwsey5nwbw/iz/+wE8rZR6CYCPZfmmyAcAHKhcp/68OT+jlHpVZWjjcL/f+Y716/0fgJ8G8LXK9Y0Abhy7XxGfbxeA+yvXDwC4Iju/AsAD2fkfAHivLt9U/wH4MoA3z+yZtwC4C8BPYTFZZWOWXvyeA/gagJ/Ozjdm+WTsvjs+545MxK4DcDMW+44n+7yV5z4I4PJG2mC/35Ox2AFcCeDRyvWhLC1VtiulHgeA7Pj8LD2pzyH7uv1qALdjBs+cuSXuAXAEwC0AHgZwTCmV7zZefbbiubP7zwC4bNgeB/NxADcAyDchvgxpP2+OAvBnInKniOzJ0gb7/Z7SZta6fejnOKQnmc9BRC4C8AUAH1RKPSuie7RFVk3aJJ9ZKXUOwKtE5FIAXwLw47ps2XHSzy0ibwdwRCl1p4i8IU/WZE3ieRtcq5Q6LCLPB3CLiHy7I2/0556SxX4IwM7K9Q4Ah0fqyxD8PxG5AgCy45EsPYnPQUQ2YSHqn1ZKfTFLTvqZqyiljgH4BhYxhktFJDeyqs9WPHd2/xIATw3b0yCuBfAOETkI4DNYuGM+jnSft0ApdTg7HsHiBf5aDPj7PSVh/yaA3VlEfTOA9wC4aeQ+LZObALwvO38fFn7oPP0Xs0j66wA8k3+9mwqyMM0/CeCAUuqjlVvJPjMAiMi2zFKHiFwA4E1YBBVvA/DuLFvzufPP490AblWZE3YKKKVuVErtUErtwuLv9Val1D9Hos+bIyIXisjF+TmAtwC4H0P+fo8dZHAMSLwVwHew8Et+ZOz+RHyuPwHwOIAzWLy934+Fb/HrAB7MjluzvILF6KCHAdwHYHXs/ns87+ux+Kp5L4B7sn9vTfmZs+d4BYC7s+e+H8BvZ+kvBnAHgIcAfA7AeVn6+dn1Q9n9F4/9DAHP/gYAN8/hebPn25/9+1auVUP+fnPmKSGEJMaUXDGEEEIsoLATQkhiUNgJISQxKOyEEJIYFHZCCEkMCjshhCQGhZ0QQhKDwk4IIYnx/wEdYbM3TApMWgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step: 1500, with epsilon: -15.369999999999761\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAADTZJREFUeJzt23uMpfVdx/H3R9YWpVAsuyB22a6NJIIJ0GYCNDThYiVAETT2D0hvGuoG0yhN2mCJBi2JaUyMJU1s6QabpiqtNbIRiVIIl/CHvc3KbSmXUtxaXHQXS0Fj0pTy9Y95hozDzM6ZM2fnON99v5LJeS6/OfP7DYf3PPucmVQVkqRefmzaE5AkTZ5xl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLU0KZpfeHNmzfX9u3bp/XlJWlD2r1793NVtWWlcVOL+/bt25mdnZ3Wl5ekDSnJd0YZ520ZSWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDW0aZVCSvcB/AT8CXqqqmUXn3w387rD738BvVdVDE5ynJGkVRor74Pyqem6Zc/8CnFtVzye5GNgJnLXm2UmSxrKauC+rqv5pwe5Xga2TeF5J0nhGvedewJ1JdifZscLYq4B/XNu0JElrMeqV+zlVtS/J8cBdSR6vqvsXD0pyPnNxf/tSTzL8YNgBsG3btjGnLElayUhX7lW1b3jcD+wCzlw8JslpwM3A5VX1n8s8z86qmqmqmS1btow/a0nSQa0Y9yRHJTl6fhu4ENizaMw24FbgvVX15KGYqCRpdKPcljkB2JVkfvwtVXVHkqsBquom4HrgOOBTw7hX/bqkJGn9rBj3qnoaOH2J4zct2P4A8IHJTk2SNC7/QlWSGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqaKS4J9mb5JEkDyaZXeL8zyf5SpIfJPnI5KcpSVqNTasYe35VPbfMue8BvwP8ytqnJElaq4nclqmq/VX1DeCHk3g+SdLajBr3Au5MsjvJjkM5IUnS2o16W+acqtqX5HjgriSPV9X9q/1iww+GHQDbtm1b7adLkkY00pV7Ve0bHvcDu4Azx/liVbWzqmaqambLli3jPIUkaQQrxj3JUUmOnt8GLgT2HOqJSZLGN8ptmROAXUnmx99SVXckuRqgqm5K8tPALHAM8HKSDwGnVtWLh2jekqSDWDHuVfU0cPoSx29asP3vwNbJTk2SNC7/QlWSGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ2NFPcke5M8kuTBJLNLnE+STyZ5KsnDSd46+alKkka1aRVjz6+q55Y5dzFw8vBxFvDp4VGSNAWTui1zOfD5mvNV4NgkJ07ouSVJqzTqlXsBdyYp4DNVtXPR+TcC312w/8xw7Nm1T/H/+tjfP8o397046aeVpHVz6s8cwx/88i8c0q8xatzPqap9SY4H7kryeFXdv+B8lvicWnwgyQ5gB8C2bdtWPVlJ0mhGintV7Rse9yfZBZwJLIz7M8BJC/a3AvuWeJ6dwE6AmZmZV8V/FIf6p50kdbDiPfckRyU5en4buBDYs2jYbcD7ht+aORt4oaomfktGkjSaUa7cTwB2JZkff0tV3ZHkaoCqugn4B+AS4Cngf4DfODTTlSSNYsW4V9XTwOlLHL9pwXYBH5zs1CRJ4/IvVCWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGRo57kiOSPJDk9iXOvSnJ3UkeTnJfkq2TnaYkaTVWc+V+DfDYMuf+BPh8VZ0G3AB8fK0TkySNb6S4D1fi7wRuXmbIqcDdw/a9wOVrn5okaVyjXrnfCFwLvLzM+YeAXxu2fxU4Oslxa5ybJGlMK8Y9yaXA/qrafZBhHwHOTfIAcC7wb8BLSzzXjiSzSWYPHDgw7pwlSStIVR18QPJx4L3MxfpI4Bjg1qp6zzLjXwc8XlUHfVN1ZmamZmdnx5q0JB2ukuyuqpmVxq145V5V11XV1qraDlwB3LM47Ek2J5l/ruuAz44xZ0nShIz9e+5Jbkhy2bB7HvBEkieBE4A/msDcJEljWvG2zKHibRlJWr2J3ZaRJG08xl2SGjLuktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktSQcZekhoy7JDVk3CWpoZHjnuSIJA8kuX2Jc9uS3DucfzjJJZOdpiRpNVZz5X4N8Ngy534f+FJVvQW4AvjUWicmSRrfSHFPshV4J3DzMkMKOGbYfj2wb+1TkySNa9OI424ErgWOXub8HwJ3Jvlt4CjgHWufmiRpXCteuSe5FNhfVbsPMuxK4HNVtRW4BPiLJK967iQ7kswmmT1w4MDYk5YkHdwot2XOAS5Lshf4InBBkr9cNOYq4EsAVfUV4Ehg8+InqqqdVTVTVTNbtmxZ08QlSctbMe5VdV1Vba2q7cy9WXpPVb1n0bB/BX4RIMkpzMXdS3NJmpKxf889yQ1JLht2Pwz8ZpKHgC8Av15VNYkJSpJWb9Q3VAGoqvuA+4bt6xcc/yZzt28kSf8P+BeqktSQcZekhoy7JDVk3CWpIeMuSQ0Zd0lqyLhLUkPGXZIaMu6S1JBxl6SGjLskNWTcJakh4y5JDRl3SWrIuEtSQ8Zdkhoy7pLUkHGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGjLuktRQqmo6Xzg5AHxnzE/fDDw3welsBK758OCaDw9rWfObqmrLSoOmFve1SDJbVTPTnsd6cs2HB9d8eFiPNXtbRpIaMu6S1NBGjfvOaU9gClzz4cE1Hx4O+Zo35D13SdLBbdQrd0nSQWy4uCe5KMkTSZ5K8tFpz2dSknw2yf4kexYce0OSu5J8a3j8qeF4knxy+B48nOSt05v5+JKclOTeJI8leTTJNcPxtutOcmSSryd5aFjzx4bjP5vka8Oa/zrJa4bjrx32nxrOb5/m/MeV5IgkDyS5fdhvvV6AJHuTPJLkwSSzw7F1e21vqLgnOQL4M+Bi4FTgyiSnTndWE/M54KJFxz4K3F1VJwN3D/swt/6Th48dwKfXaY6T9hLw4ao6BTgb+ODw37Pzun8AXFBVpwNnABclORv4Y+ATw5qfB64axl8FPF9VPwd8Yhi3EV0DPLZgv/t6551fVWcs+LXH9XttV9WG+QDeBnx5wf51wHXTntcE17cd2LNg/wngxGH7ROCJYfszwJVLjdvIH8DfAb90uKwb+Engn4GzmPuDlk3D8Vde58CXgbcN25uGcZn23Fe5zq1DyC4AbgfSeb0L1r0X2Lzo2Lq9tjfUlTvwRuC7C/afGY51dUJVPQswPB4/HG/3fRj++f0W4Gs0X/dwi+JBYD9wF/Bt4PtV9dIwZOG6XlnzcP4F4Lj1nfGa3QhcC7w87B9H7/XOK+DOJLuT7BiOrdtre9NaPnkKssSxw/HXfVp9H5K8Dvhb4ENV9WKy1PLmhi5xbMOtu6p+BJyR5FhgF3DKUsOGxw295iSXAvuraneS8+YPLzG0xXoXOaeq9iU5HrgryeMHGTvxdW+0K/dngJMW7G8F9k1pLuvhP5KcCDA87h+Ot/k+JPlx5sL+V1V163C4/boBqur7wH3Mvd9wbJL5i62F63plzcP51wPfW9+Zrsk5wGVJ9gJfZO7WzI30Xe8rqmrf8LifuR/iZ7KOr+2NFvdvACcP77S/BrgCuG3KczqUbgPeP2y/n7l70vPH3ze8w3428ML8P/U2ksxdov858FhV/emCU23XnWTLcMVOkp8A3sHcG433Au8ahi1e8/z34l3APTXclN0Iquq6qtpaVduZ+//1nqp6N03XOy/JUUmOnt8GLgT2sJ6v7Wm/6TDGmxSXAE8yd5/y96Y9nwmu6wvAs8APmfspfhVz9xrvBr41PL5hGBvmfmvo28AjwMy05z/mmt/O3D89HwYeHD4u6bxu4DTggWHNe4Drh+NvBr4OPAX8DfDa4fiRw/5Tw/k3T3sNa1j7ecDth8N6h/U9NHw8Ot+q9Xxt+xeqktTQRrstI0kagXGXpIaMuyQ1ZNwlqSHjLkkNGXdJasi4S1JDxl2SGvpf9p6hucFMOIMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Percent of succesful episodes: 0.6525%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFFpJREFUeJzt3X2MHPV9x/HP1z5sML5gE58x9dmcoQbqpgg7V0pFAyqhPJeHElWmD0FJJStSI4FalEAtVfSvQmlTREihbkPzRAuKCI2FeA4PSdpCORsbG/xsTMH44cyDbbCN787f/rFz9u7e3u3uzezM7G/eL+m0u7OzM9/57d1n536/mR1zdwEAwjEh6wIAAMki2AEgMAQ7AASGYAeAwBDsABAYgh0AAkOwA0BgCHYACAzBDgCB6chipTNmzPCenp4sVg0AbWvFihV73L2r3nyZBHtPT4/6+vqyWDUAtC0ze7uR+eiKAYDAEOwAEBiCHQACQ7ADQGAIdgAIDMEOAIEh2AEgMJkcx94KT63doe7pU3Tf85t1740LNakj/mfW2u17NTB0RAvnTk+gwsYcGhjS46/v0A2LZsvMas7zwobdOvOUTs2edkLN5weGjuix17brS4u6NWGCyd316MrtuvqcU3X8cRP1n69t1yULTtHUyaW3/8339unQ4JAWRdv5+rsfSZLO6Z5WsdwDhwf1+OodkklfWtQtl/TAS1t0RteJuvDMLj21dqeuO3e2lvxwhWadNFlnzfqM7n9hsz48MKDenumaPe0EzeycnFBLAe3p+kXdmjfjxJauI4hg33doQF/70cqjj//pxc265ZIzYy/36m//UpK07c6rYi+rUXc+uV7f++9t6uqcrIvOrH2C2Vf+7VV1Ht+hNXdcVvP5f35pi/7+mY2aaKYbPt+tlzb269Yfr9ba7Xt1/cLZuuWRVbru3F/RPYsXSpKuvPcXko5t5zX3/VfF42F/s/xNPdL3jiRpcscE7Ts0qLuf3iBJumFRtx5d+a5e3vq+nlu3a0RNv9i0R5I0ymcVUBiLTptOsDdiaKjygtzvf3w4o0ri273/kCTp40ODY863f4zn90Tbv/fgQMW8/R9/qk8Ol+7v3Heo6drKX7P34IA+OjBQ9txBSdL2jw6OuYy3/ja9D0mgqOhjB4DAEOwAEBiCHQACQ7ADQGAIdgAIDMEOAIEh2AEgMAQ7AASGYAeAwBDsABAYgh0AAkOwA0BgCHYACAzBDgCBSSzYzWyimb1mZo8ntUwAQPOS3GO/WdK6BJcHABiHRILdzLolXSXpX5NYHgBg/JLaY79H0jckHUloeQCAcYod7GZ2taTd7r6iznxLzKzPzPr6+/vjrnZMLq8/ExLlNDmQG0nssV8g6Roz2ybpYUkXm9mPqmdy92Xu3uvuvV1dtS/SDACIL3awu/vt7t7t7j2SFkt63t3/JHZlAIBx4Th2AAhMR5ILc/cXJb2Y5DIBAM0Jco+dgbz00eZAfgQZ7ABQZAQ7AASGYAeAwBDsABCYIIOdcbz0cbYvkB9BBjsAFBnBDgCBIdgBIDAEOwAEJshg5yzI9NHmQH4EGewAUGQEOwAEhmAHgMAQ7AAQmECDnZG8tNHiQH4EGuwAUFwEOwAEhmAHgMAQ7AAQmCCDnbMgM0CbA7kRZLADQJER7AAQGIIdAAJDsANAYIII9upxOwZP08c1T4H8CCLYUcks6woAZIlgDxD/sQDFRrADQGAIdgAITBDB7lV9DyEP5FVva17ktCygkGIHu5nNMbMXzGydmb1hZjcnURjGj8FToNg6EljGoKS/dPeVZtYpaYWZPevubyawbIwDe89AscXeY3f3He6+Mrq/X9I6SbPjLhcAMD6J9rGbWY+khZJeSXK5AIDGJRbsZjZV0qOSbnH3fTWeX2JmfWbW19/fn9RqJYV15ulw7aMNADezbV51K1esr9ctf6l7ZS1eNh1AthIJdjM7TqVQf8jdf1JrHndf5u697t7b1dWVxGqRI3k9WgcooiSOijFJ35W0zt2/Fb+kYhs+osUU/9AWq7qVlT8Y//KkUp3lR99Y9IAjcoDsJbHHfoGkP5V0sZmtin6uTGC5AIBxiH24o7v/UrH2AwEASQrkzNOqx9mUkYi6g6fNLKv6Na0cPI0e0NUOZC+IYEf2yHMgPwj2nGnbwdOy6QCyRbADQGAIdgAITBDBXj3Q2M4DePXPPG184zjzFCimIIId2SPQgfwg2HOmbQdPrfIWQHYIdgAIDMEOAIEJI9hHnHnavh2+7XvmaeUtgOyEEezIHHkO5AfBnjNtO3haNh1Atgh2AAgMwQ4AgSHYASAwQQT7iIG7gEfy8nrUSU7LAgopiGBHJQYwgWIj2AOU1716AOkg2AEgMAQ7AAQmiGAP6WLW9eT26xLo/wFyI4hgRyUGT4FiI9gDxM4zUGwEOwAEhmAHgMAEEewjL2Ydbl9EXjctp2UBhRREsKMSg6dAsRHsAcrrXj2AdBDsABCYRILdzC43sw1mttnMbktimQCA8Ykd7GY2UdJ3JF0haYGkG81sQdzlNqNIZ57mFd0/QH4kscd+nqTN7r7V3Q9LeljStQksF+PE4ClQbEkE+2xJ75Q9fjeaFoy7nlqvoSP52yV96JW3a05vZO/55a0faP+hgYQrApAHSQR7rf3DEdFiZkvMrM/M+vr7+xNYbXruf3GLfr4xfzUvfWxtrNff89ymhCoBkCdJBPu7kuaUPe6W9F71TO6+zN173b23q6srgdWmK4977HENDh3JugQALZBEsL8qab6ZzTOzSZIWS1qewHIbVh25IQ/k5XXbcvt1wkABdcRdgLsPmtnXJT0taaKkB939jdiVYdwYPAWKLXawS5K7PyHpiSSWhfjyulcPIB2ceQoAgSHYASAwQQR79df0htwTkddBSrp/gPwIIthRicFToNgIdgAIDMEeILpFgGIj2AEgMEEE+4iv7Q14lzWvmzZcV17rA4okiGBHJQZPgWIj2AEgMAR7gOgOAYqNYAeAwAQZ7CHvsOZ124br4r8FIHtBBnvRMXgKFBvBDgCBIdgDRHcIUGwEOwAEJohgH7GHGvAea17Pqh2uK69fKwwUSRDBjkoMngLFRrADQGAI9gDltLcGQEoIdgAITBDBXj1gF/IAXt63jP8WgOwFEeyoxOApUGwEOwAEhmAPEN0hQLER7AAQmCCCfeQ1T1uwjuQXWXs9w9cOHWWNzWybV93KVbEhzW5T+fzulbUcqxtA1oIIdmTv6AcRyQ5kjmBvUFoHmgwf0WIx1nhsGZW3MlVsSLNrKJ/frPLom6P1ckQOkDmCPUAMngLFFivYzexuM1tvZq+b2WNmNi2pwgAA4xN3j/1ZSZ9z93MkbZR0e/ySmjfiW3sDHjxtppBUB0/pYwdyI1awu/sz7j4YPXxZUnf8ktCO6n4gAUhNkn3sX5X0ZILLyxUGT+sMniZQN4BkdNSbwcyekzSrxlNL3f2n0TxLJQ1KemiM5SyRtESS5s6dO65iAQD11Q12d79krOfN7CZJV0v6oo9x3TZ3XyZpmST19vby/3oLcVQMUGx1g30sZna5pG9KusjdDyRTUvOqP09a0c+bl8HTZrYtmzNP+VQBsha3j/0+SZ2SnjWzVWb2QAI1oQ0d/RAh14HMxdpjd/dfTaqQvGPwtN6Zp5XrBpAdzjwFgMAQ7AGiOwQotiCCvUhnnub2a3vHUR+A1ggi2JG94SOTyHUgewR7gxg8rXfmqY1rmQCSR7ADQGAI9gDRzw0UWxDBPuKap61YRwuWWXM9dc88bWJZ1a9p6Zmn9LEDeRFEsCN7x46KIdqBrBHsDWLwtNEzTxk+BbJGsANAYAj2ANEbAhRbIMFe9bW9QZ95mtOv7R1HfQBaI5BgR+a4ljWQGwR7gxg8HXvwVDZyHgDZINjbHF0fAKoR7AEi64FiCyLYRwZZyNc8rT3/WPOmMXhKHzuQH0EEO7LH97ED+UGwN4jBU655CrQLgr3NsYMMoBrBDgCBCSLYi3zN01qHO3rVQCbXPAWKJYhgR/b4PnYgPwj2BgUzeFo+X7PLrVoH1zwF8olgb1Be90Tj1JXXbQIQD8EOAIEJItjTuOZpWl0M9c88rb91Yw2elr+62W3imqdAewgi2IssL0ehHPvwyElBQIER7DmT5uBp08utWketM0859RTIXiLBbma3mpmb2YwklpdHed0PbaRrZvTXAghR7GA3szmSfk/S/8UvBwAQVxJ77P8o6RvKcAeweq+1FRefyMvgaSOtPObgadnrEx08rV45gMzECnYzu0bSdndfnVA9aFJecrT6wwRAdjrqzWBmz0maVeOppZL+StKljazIzJZIWiJJc+fObaLEYmnbwVOueQrkRt1gd/dLak03s9+QNE/S6uh08m5JK83sPHffWWM5yyQtk6Te3t6227Fru4IbEOI2AWgg2Efj7mskzRx+bGbbJPW6+54E6gIAjBPHsTeonboYGu13b6dtAtC4ce+xV3P3nqSW1fy6qx5nU0Yq4n73fJzj3sdcLoOnQG6wxx4gTv4Eio1gb1Be90Q58xRANYIdAAJDsDeonXo3GDwFii2IYB95geds6khD3G1tVdsMdwmF3PZAuwgi2FGJwVOg2Aj2BuV1R5RrngKoRrADQGAI9ga1U+8Gg6dAsQUR7CO+jz2jOtIQ97vnW9U2db9HHkBqggh2VMpy8JSjYoDsEewNymteMXgKoBrBDgCBIdgbFOJAY4jbBECyVlz4uZ7e3l7v6+tr+nXf/tkmLV/93ojphwaH9M4HByumzZ85ddz1Ddu0++Oj90896XhNnZzYtxzXXWdX52RNO+G4Ec8PHnG9teeTo49P7zpRE6s61YeXcfxxEzRn+hR9dHBA/fs/lSTNnnaCtn90rK3mz5x6dP7hNqt+XL1cSZoyaaIODgw13ae+7c6rmnsBgKPMbIW799abr/VJlaCuzsmaf0rtwH7ng4M6o+tEben/RJf/+ixNSOB/kY8/HdTegwM6cHhIC+dOi7/ABpz22Sl6bt1u/WbP9FHneWvPJ+qc3KGPDw/q7FmdI54/vetEPf3GLv3uWTOPDqQ+sWanLvm1mZrUMeFosF+64BR1TDR98umgDgwMHW3bDw8MyEwj2nrOyVP0/PrdkqSLzuySu/TUG6WrIF589kw9v363rvjcLD25tjTt7FmdWr9zvyTpj35rrm5YNDtGywBoVFsF++Lz5mrxeVwIGwDGQh87AASGYAeAwBDsABAYgh0AAkOwA0BgCHYACAzBDgCBIdgBIDCZfKWAmfVLenucL58haU+C5SSFuppDXc3Ja11SfmsLsa7T3L2r3kyZBHscZtbXyHclpI26mkNdzclrXVJ+aytyXXTFAEBgCHYACEw7BvuyrAsYBXU1h7qak9e6pPzWVti62q6PHQAwtnbcYwcAjKGtgt3MLjezDWa22cxuS3G9c8zsBTNbZ2ZvmNnN0fQ7zGy7ma2Kfq4se83tUZ0bzOyyFte3zczWRDX0RdNONrNnzWxTdDs9mm5mdm9U2+tmtqhFNZ1V1i6rzGyfmd2SRZuZ2YNmttvM1pZNa7p9zOymaP5NZnZTi+q628zWR+t+zMymRdN7zOxgWbs9UPaaz0fv/+ao9lhXPRylrqbft6T/Xkep65GymraZ2apoeprtNVo+ZPc75u5t8SNpoqQtkk6XNEnSakkLUlr3qZIWRfc7JW2UtEDSHZJurTH/gqi+yZLmRXVPbGF92yTNqJr2d5Jui+7fJumu6P6Vkp5U6ZKn50t6JaX3bqek07JoM0kXSlokae1420fSyZK2RrfTo/vTW1DXpZI6ovt3ldXVUz5f1XL+V9JvRzU/KemKFtTV1PvWir/XWnVVPf8Pkv46g/YaLR8y+x1rpz328yRtdvet7n5Y0sOSrk1jxe6+w91XRvf3S1onaazrvF0r6WF3/9Td35K0WaX603StpO9H978v6bqy6T/wkpclTTOzU1tcyxclbXH3sU5Ka1mbufvPJX1QY33NtM9lkp519w/c/UNJz0q6POm63P0Zdx+MHr4sqXusZUS1fcbd/8dL6fCDsm1JrK4xjPa+Jf73OlZd0V73H0r6j7GW0aL2Gi0fMvsda6dgny3pnbLH72rscG0JM+uRtFDSK9Gkr0f/Tj04/K+W0q/VJT1jZivMbEk07RR33yGVfvEkzcyoNklarMo/uDy0WbPtk0W7fVWlPbth88zsNTN7ycy+EE2bHdWSRl3NvG9pt9cXJO1y901l01Jvr6p8yOx3rJ2CvVY/WKqH9JjZVEmPSrrF3fdJul/SGZLOlbRDpX8FpfRrvcDdF0m6QtKfm9mFY8ybam1mNknSNZJ+HE3KS5uNZrQ60m63pZIGJT0UTdohaa67L5T0F5L+3cw+k2Jdzb5vab+fN6py5yH19qqRD6POOkoNidXWTsH+rqQ5ZY+7Jb2X1srN7DiV3rSH3P0nkuTuu9x9yN2PSPoXHes6SLVWd38vut0t6bGojl3DXSzR7e4salPpw2alu++KasxFm6n59kmtvmjQ7GpJfxx1Fyjq6ng/ur9Cpf7rM6O6yrtrWlLXON63NNurQ9IfSHqkrN5U26tWPijD37F2CvZXJc03s3nRXuBiScvTWHHUf/ddSevc/Vtl08v7pq+XNDxav1zSYjObbGbzJM1XacCmFbWdaGadw/dVGnxbG9UwPKp+k6SfltX25Whk/nxJe4f/XWyRij2pPLRZ2fqaaZ+nJV1qZtOjbohLo2mJMrPLJX1T0jXufqBsepeZTYzun65S+2yNattvZudHv6dfLtuWJOtq9n1L8+/1Eknr3f1oF0ua7TVaPijL37E4o8Fp/6g0mrxRpU/fpSmu93dU+pfodUmrop8rJf1Q0ppo+nJJp5a9ZmlU5wbFHHWvU9vpKh1xsFrSG8PtIumzkn4maVN0e3I03SR9J6ptjaTeFtY2RdL7kk4qm5Z6m6n0wbJD0oBKe0V/Np72UanPe3P085UW1bVZpX7W4d+zB6J5b4je39WSVkr6/bLl9KoUtFsk3afoxMOE62r6fUv677VWXdH070n6WtW8abbXaPmQ2e8YZ54CQGDaqSsGANAAgh0AAkOwA0BgCHYACAzBDgCBIdgBIDAEOwAEhmAHgMD8P5SOX0iGtdYHAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Begin training Deep Q-Network!!\")\n",
    "for i in range(num_episodes):\n",
    "#for i in range(1):  #use if you only want to see one episode!\n",
    "    player_position = int_to_grid(0)   #we start the player at the topleft corner of our map_grid\n",
    "    current_state = np.stack([np.stack([player_position, map_grid], axis=-1)])  #stack the player_position array and map_grid array channel/depth-wise\n",
    "    assert(current_state.shape == (1,4,4,2))                                    #check that our input current_state has shape (batch_size, rows, cols, channels)\n",
    "    rewardTally = 0  #start reward Score at 0\n",
    "    j = 0            #counter in case our agent does too many stupid moves\n",
    "    while j < 32:    #limit the number of moves our agent can take to 32\n",
    "        j += 1\n",
    "        \n",
    "        #########################################################################\n",
    "        #Our Deep Q-Network takes an input state (x) of the environment and\n",
    "        #predicts an output list of Q-values (y) representing the 'q'uality of\n",
    "        #the actions we can take the move the player around the map grid        \n",
    "        \n",
    "        #########################################################################\n",
    "        ## Feed current state into Deep Q-Network\n",
    "        current_state_Q_values = model.predict(x=current_state, batch_size=1)    #Use our neural network to get the Q values for each possible move given the current player position and map_grid\n",
    "        #print(current_state_Q_values)                                           #outputs the Q values for our possible moves left, down, right, up as an array e.g. [[0.23, 0.22, 0.24, 0.26]]\n",
    "        \n",
    "        ## Decide what action to perform based on current_state_Q_values, and randomize with epsilon\n",
    "        action = np.argmax(current_state_Q_values)                               #find the maximum/greatest Q value to decide what action to take\n",
    "        if np.random.rand(1) < epsilon:                                          #if some random number is less than our epsilon value\n",
    "            action = int(np.random.uniform(low=0, high=4))                       #we get a new action randomly from a uniform distribution!\n",
    "        #print(actionKeys[action])\n",
    "        \n",
    "        ## Take a step towards somewhere\n",
    "        player_position = player_step(old_position=player_position, action=action)\n",
    "        #print(player_position)  #uncomment to see player position!\n",
    "        \n",
    "        ## Get new future state (stack of player_postion and map_grid representing our observation of the environment) and reward associated with it\n",
    "        future_state = np.stack([np.stack([player_position, map_grid], axis=-1)])\n",
    "        reward = -map_grid.ravel()[np.argmax(player_position)]                    #note that we do a negative here, because dangerous holes are mapped +ve in our map\n",
    "        rewardTally += reward\n",
    "        #print(rewardTally)\n",
    "        \n",
    "        ## Check if we fell in the hole, else continue\n",
    "        if rewardTally <= -5.0:\n",
    "            break\n",
    "        \n",
    "        #########################################################################\n",
    "        ## Feed the future state into Deep Q-Network and get max Q value from it\n",
    "        future_state_Q_values = model.predict(x=future_state, batch_size=1)\n",
    "        future_state_max_Q = np.max(future_state_Q_values)\n",
    "        #print(future_state_max_Q)\n",
    "        \n",
    "        ## Implement Bellman equation here!! Q(s,a) = r + γ(max(Q(s’,a’))\n",
    "        #Basically, the Q-value for a given state (s) and action (a) should represent the current reward (r) plus the maximum discounted (γ) future reward expected\n",
    "        target_Q = current_state_Q_values                       #set our target to the current_state first\n",
    "        target_Q[0,action] = reward + gamma*future_state_max_Q  #replace the single Q-value at the index position of the action taken\n",
    "        \n",
    "        ## Retrain our Keras Deep Q-Network to learn this new target Q value\n",
    "        model.fit(x=current_state, y=target_Q, verbose=0, batch_size=1)\n",
    "        \n",
    "        ## Check if we are done (reached goal), otherwise continue\n",
    "        if rewardTally > 0:\n",
    "            epsilon = epsilon - (0.02)  #Reduce (by hardcoded amount) epsilon value, i.e. the chance of random action as we train the model.\n",
    "            break\n",
    "        \n",
    "        ## Set the next current_state as our current future_state\n",
    "        current_state = future_state\n",
    "    \n",
    "    ## After finishing an episode... \n",
    "    rewardList.append(rewardTally) #we append the rewardTally for the episode to the rewardList\n",
    "    \n",
    "    ## Some print information\n",
    "    if i%500==0 and i!=0:\n",
    "        print(f\"Step: {i-500} to {i}, with epsilon: {epsilon}\")\n",
    "        plt.plot(rewardList[-500:])  #plot the 500 last rewardTally-s\n",
    "        plt.show()\n",
    "\n",
    "#Show final score\n",
    "success = len([r for r in rewardList if r == 5])   #when we reached the goal\n",
    "#failure = len([r for r in rewardList if r == 0])  #when we reached nowhere\n",
    "epicfail = len([r for r in rewardList if r == -5]) #when we dropped into a hole\n",
    "\n",
    "print(\"Percent of succesful episodes: \" + str(success/num_episodes) + \"%\")\n",
    "plt.plot(rewardList[-num_episodes:])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### See how our player does"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_player_and_map(player_position, map_grid):\n",
    "    \"\"\"\n",
    "    Matplotlib function to plot the player position on the left\n",
    "    and the map_grid position on the right\n",
    "    \"\"\"\n",
    "    assert(player_position.shape == map_grid.shape)\n",
    "    \n",
    "    fig, axarr = plt.subplots(nrows=1, ncols=2, squeeze=False, figsize=(5,5))\n",
    "    axarr[0, 0].imshow(X=player_position, cmap='winter')\n",
    "    axarr[0, 1].imshow(X=-map_grid, cmap='BrBG')\n",
    "    \n",
    "    return plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAChCAYAAABNqJPmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACDRJREFUeJzt3c+LnGUWxfFzpu34gxYiVC9ikp44IELGhUoTBpRZBBcZN251IQhCVoLCbFzqP+BuNgFDz0IUGV2IOEgWERNwop2kI8l0HDKi2CgkhRENA0rkuuhaJE6w3zbv87zV9/l+oKCqU6l7q3NyeLuqq8oRIQDI5HdDLwAAfaPYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0rmlxI16dEdoz/YSN93NybuHm920zxUxdukpo9EoFhYWSo/BFDp9+vQ4IuY3ul6RYtOe7dLywSI33YlfHG520xarTFlYWNCxY8eqzMJ0mZub+6LL9fhRFEA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOl0KjbbB2x/avuC7RdKL4W2kC/0bcNisz0j6W+S/iJpr6Qnbe8tvRjaQL5QQpcjtn2SLkTEZxHxo6TXJT1edi00hHyhd12KbaekL6+5vDb52nVsH7S9bHtZl/7X137Ib9P5Go/H1ZbD1tSl2G70xoHxf1+IOBQRixGxqPk7bn4ztGLT+RqNRhXWwlbWpdjWJO2+5vIuSV+VWQcNIl/oXZdi+1jSvbbvsb1N0hOS3i67FhpCvtC7Dd8aPCKu2n5W0nuSZiQdjohzxTdDE8gXSuj0mQcR8a6kdwvvgkaRL/SNVx4ASIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5BOp5dUbdrJuyW/WOSmgaE9/chdg81eOn55sNnSsPd9MzhiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDobFpvtw7Yv2j5bYyG0h4yhb12O2JYkHSi8B9q2JDKGHm1YbBHxgaRvKuyCRpEx9I3H2ACk01ux2T5oe9n2snSpr5sFJF2fr/F4PPQ6mHK9FVtEHIqIxYhYlOb7ullA0vX5Go1GQ6+DKcePogDS6fLrHq9J+lDSfbbXbD9Tfi20hIyhbxt+5kFEPFljEbSLjKFv/CgKIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6G76kCsD1lo5fHnoFbIAjNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIp8sHJu+2fdT2qu1ztp+rsRjaQL5QQpd397gq6a8Rccr2nZJO2j4SEf8uvBvaQL7Quw2P2CLi64g4NTn/vaRVSTtLL4Y2kC+UsKnH2GzvkfSgpBM3+LODtpdtL0uX+tkOTemar/F4XHs1bDGdi832nKQ3JT0fEd/98s8j4lBELEbEojTf545owGbyNRqN6i+ILaVTsdme1XroXo2It8quhNaQL/Sty7OilvSKpNWIeLn8SmgJ+UIJXY7YHpb0lKT9tlcmp8cK74V2kC/0bsNf94iI45JcYRc0iHyhBF55ACAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOl3eQReYKp+fX9HTj9w12Pyl45cHmz20f8z9ceANVjpdiyM2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUinyyfB32b7I9tnbJ+z/VKNxdAG8oUSury7xw+S9kfEFduzko7b/mdE/KvwbmgD+ULvunwSfEi6Mrk4OzlFyaXQDvKFEjo9xmZ7xvaKpIuSjkTEiRtc56DtZdvL0qW+90Rim83XD1fpPfy6TsUWET9FxAOSdknaZ/v+G1znUEQsRsSiNN/3nkhss/m69RbXXxJbyqaeFY2IbyW9L+lAkW3QNPKFvnR5VnTe9vbJ+dslPSrpfOnF0AbyhRK6PCu6Q9Lfbc9ovQjfiIh3yq6FhpAv9K7Ls6KfSHqwwi5oEPlCCbzyAEA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdLz+Pn8936h9SdIXv/GvjySNe1yH+fXm/z4iir9n1U3mSxr2e7yV/32nYX6njBUptpthe3n9Pd2Y3+L8Goa8j0N/f1uZz4+iANKh2ACkM43Fdoj5Tc+vYcj7OPT3t4n5U/cYGwDcrGk8YgOAm0KxAUhnqorN9gHbn9q+YPuFyrMP275o+2zNuZPZu20ftb1q+5zt5yrPv832R7bPTOa/VHN+La3mazJ/sIwNkq+ImIqTpBlJ/5X0B0nbJJ2RtLfi/D9LekjS2QHu+w5JD03O3ynpP5XvuyXNTc7PSjoh6U9DZ6Ln+9hsvibzB8vYEPmapiO2fZIuRMRnEfGjpNclPV5reER8IOmbWvN+MfvriDg1Of+9pFVJOyvOj4i4Mrk4Ozlle1ap2XxN5g+WsSHyNU3FtlPSl9dcXlPF/9zTwvYerX9q04nKc2dsr0i6KOlIRFSdXwH5mhgiY7XzNU3F5ht8LdtRw6+yPSfpTUnPR8R3NWdHxE8R8YCkXZL22b6/5vwKms+XNFzGaudrmoptTdLuay7vkvTVQLtUZ3tW64F7NSLeGmqPiPhW0vuSDgy1QyFN50uajozVytc0FdvHku61fY/tbZKekPT2wDtVYduSXpG0GhEvDzB/3vb2yfnbJT0q6XztPQprNl/SsBkbIl9TU2wRcVXSs5Le0/oDm29ExLla822/JulDSffZXrP9TK3Zkh6W9JSk/bZXJqfHKs7fIemo7U+0XgBHIuKdivOLazxf0rAZq54vXlIFIJ2pOWIDgL5QbADSodgApEOxAUiHYgOQDsUGIB2KDUA6PwPNAxyb1Q+V3gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "player_position = int_to_grid(0)   #we start the player at the topleft corner of our map_grid\n",
    "plot_player_and_map(player_position=player_position, map_grid=map_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Down v\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAChCAYAAABNqJPmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACDRJREFUeJzt3cGLnPUdx/HPp+umWlZQmD2kSbaxIELqQe0SCkoPwUPqxaseBEHISVDoxaP+A956CRjSgyhSPYhYJIeIWbDRMdlI0o0lFcUlQjIY0VBQIt8edgqJXZxnzfP7Pc9+n/cLBmY2k/l+Z/PJh2dndmYcEQKATH7R9QIA0DaKDUA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIJ1bStyoPQppb4mbRq99poiJS08ZjUaxtLRUegx66PTp05OIWJx1vSLFtlFq4zI3jR5brjJlaWlJJ06cqDIL/bKwsPB5k+vxoyiAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSaVRstg/a/sT2BdvPlV4Kw0K+0LaZxWZ7TtJfJP1J0j5Jj9veV3oxDAP5QglNjtj2S7oQEZ9GxPeSXpX0aNm1MCDkC61rUmy7JH1x3eX16dduYPuQ7bHtsXS5rf2Q35bzNZlMqi2H7alJsW32xoHxf1+IOBwRyxGxLM18Hzjgf7acr9FoVGEtbGdNim1d0p7rLu+WdLHMOhgg8oXWNSm2DyXdbfsu2zskPSbpzbJrYUDIF1o3863BI+Ka7aclvSNpTtKRiDhXfDMMAvlCCY0+8yAi3pb0duFdMFDkC23jlQcA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKTT6CVVW/b7i9L4+SI33Yg7nI30nnzozs5mH1250tlsqdv7vhUcsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiCdmcVm+4jtS7bP1lgIw0PG0LYmR2xHJR0svAeG7ajIGFo0s9gi4j1JX1XYBQNFxtA2HmMDkE5rxWb7kO2x7bEu/6etmwUk3ZivyWTS9TroudaKLSIOR8RyRCxr8Vdt3Swg6cZ8jUajrtdBz/GjKIB0mvy6xyuS3pd0j+1120+VXwtDQsbQtpmfeRARj9dYBMNFxtA2fhQFkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiCdmS+p+lk++rXk54vcNNC1oytXul4BM3DEBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpNPnA5D22j9tes33O9jM1FsMwkC+U0OTdPa5J+nNEnLJ9u6SPbB+LiH8W3g3DQL7QuplHbBHxZUScmp7/VtKapF2lF8MwkC+UsKXH2GzvlXS/pJOb/Nkh22PbY+lyO9thUJrmazKZ1F4N20zjYrO9IOl1Sc9GxDc//vOIOBwRyxGxLC22uSMGYCv5Go1G9RfEttKo2GzPayN0L0fEG2VXwtCQL7StybOilvSSpLWIeLH8ShgS8oUSmhyxPSjpCUkHbK9OT48U3gvDQb7Qupm/7hERK5JcYRcMEPlCCbzyAEA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdJq8gy7QK5+dX9WTD93Z2fyjK1c6m921vy38ruMNVhtdiyM2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUinySfB32r7A9tnbJ+z/UKNxTAM5AslNHl3j+8kHYiIq7bnJa3Y/ntE/KPwbhgG8oXWNfkk+JB0dXpxfnqKkkthOMgXSmj0GJvtOdurki5JOhYRJze5ziHbY9tj6XLbeyKxrebru2v0Hn5ao2KLiB8i4j5JuyXtt33vJtc5HBHLEbEsLba9JxLbar5+eYvrL4ltZUvPikbE15LelXSwyDYYNPKFtjR5VnTR9h3T87dJeljS+dKLYRjIF0po8qzoTkl/tT2njSJ8LSLeKrsWBoR8oXVNnhX9WNL9FXbBAJEvlMArDwCkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSMcb7/PX8o3alyV9/jP/+kjSpMV1mF9v/m8iovh7Vt1kvqRuv8fb+d+3D/MbZaxIsd0M2+ON93Rj/hDn19Dlfez6+zuU+fwoCiAdig1AOn0stsPMH/T8Grq8j11/fwcxv3ePsQHAzerjERsA3BSKDUA6vSo22wdtf2L7gu3nKs8+YvuS7bM1505n77F93Paa7XO2n6k8/1bbH9g+M53/Qs35tQw1X9P5nWWsk3xFRC9OkuYk/VvSbyXtkHRG0r6K8/8o6QFJZzu47zslPTA9f7ukf1W+75a0MD0/L+mkpD90nYmW7+Ng8zWd31nGushXn47Y9ku6EBGfRsT3kl6V9Git4RHxnqSvas370ewvI+LU9Py3ktYk7ao4PyLi6vTi/PSU7VmlweZrOr+zjHWRrz4V2y5JX1x3eV0V/3P3he292vjUppOV587ZXpV0SdKxiKg6vwLyNdVFxmrnq0/F5k2+lu2o4SfZXpD0uqRnI+KbmrMj4oeIuE/Sbkn7bd9bc34Fg8+X1F3GauerT8W2LmnPdZd3S7rY0S7V2Z7XRuBejog3utojIr6W9K6kg13tUMig8yX1I2O18tWnYvtQ0t2277K9Q9Jjkt7seKcqbFvSS5LWIuLFDuYv2r5jev42SQ9LOl97j8IGmy+p24x1ka/eFFtEXJP0tKR3tPHA5msRca7WfNuvSHpf0j22120/VWu2pAclPSHpgO3V6emRivN3Sjpu+2NtFMCxiHir4vziBp4vqduMVc8XL6kCkE5vjtgAoC0UG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5DOfwHhXR2K0MlrcQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Down v\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAChCAYAAABNqJPmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACDJJREFUeJzt3cGLnPUdx/HPp+umWlZQmD2kSbaxIELqQe0SCoqH4CH14lUPgiDkJCj04lH/AW+9BAzpQRSpHkQskkPEBDQ6JhtJurGkorhESAYjGgpK5OthpzSxi/OseX6/59nv837BwMxmMt/vbD758OzMzowjQgCQya+6XgAA2kaxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApHNTiRu1RyHtLnHT6LXPFDFx6Smj0SiWlpZKj0EPnTp1ahIRi7OuV6TY1kttXOam0WPLVaYsLS3p2LFjVWahXxYWFj5vcj1+FAWQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6jYrN9n7bn9g+b/vZ0kthWMgX2jaz2GzPSfqrpD9L2iPpMdt7Si+GYSBfKKHJEdteSecj4tOI+F7SK5IeKbsWBoR8oXVNim2HpC+uubw2/dp1bB+wPbY9li61tR/y23S+JpNJteWwNTUpto3eODD+7wsRByNiOSKWpZnvAwf816bzNRqNKqyFraxJsa1J2nXN5Z2SLpRZBwNEvtC6JsX2oaQ7bd9he5ukRyW9UXYtDAj5QutmvjV4RFy1/ZSktyXNSToUEWeLb4ZBIF8oodFnHkTEW5LeKrwLBop8oW288gBAOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQavaQKwP888cDtnc0+fPxyZ7Olbu/7ZnDEBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHRmFpvtQ7Yv2j5TYyEMDxlD25ocsR2WtL/wHhi2wyJjaNHMYouIdyV9VWEXDBQZQ9t4jA1AOq0Vm+0Dtse2x9Kltm4WkHR9viaTSdfroOdaK7aIOBgRyxGxLC22dbOApOvzNRqNul4HPcePogDSafLrHi9Lek/SXbbXbD9Zfi0MCRlD22Z+5kFEPFZjEQwXGUPb+FEUQDoUG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0Zr6k6hf54wVp/FyRm27EHc5GeoePX+56BczAERuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApNPkA5N32T5qe9X2WdtP11gMw0C+UEKTd/e4KukvEXHS9q2SPrJ9JCL+WXg3DAP5QutmHrFFxJcRcXJ6/ltJq5J2lF4Mw0C+UMKmHmOzvVvSvZJObPBnB2yPbY916T/tbIdBaZqvyWRSezVsMY2LzfaCpNckPRMR3/z0zyPiYEQsR8SyFn/T5o4YgM3kazQa1V8QW0qjYrM9r/XQvRQRr5ddCUNDvtC2Js+KWtKLklYj4oXyK2FIyBdKaHLEdr+kxyXts70yPT1ceC8MB/lC62b+ukdEHJfkCrtggMgXSuCVBwDSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApNPkHXQ376PfSn6uyE0Dn51b0RMP3N7Z/MPHL3c2u2t/X/hDxxusNLoWR2wA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkE6TT4K/2fYHtk/bPmv7+RqLYRjIF0po8u4e30naFxFXbM9LOm77HxHxfuHdMAzkC61r8knwIenK9OL89BQll8JwkC+U0OgxNttztlckXZR0JCJObHCdA7bHtsfSpbb3RGKbzdd3V+k9/LxGxRYRP0TEPZJ2Stpr++4NrnMwIpYjYllabHtPJLbZfP36JtdfElvKpp4VjYivJb0jaX+RbTBo5AttafKs6KLt26bnb5H0kKRzpRfDMJAvlNDkWdHtkv5me07rRfhqRLxZdi0MCPlC65o8K/qxpHsr7IIBIl8ogVceAEiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQjtff56/lG7UvSfr8F/71kaRJi+swv97830VE8fesusF8Sd1+j7fyv28f5jfKWJFiuxG2x+vv6cb8Ic6vocv72PX3dyjz+VEUQDoUG4B0+lhsB5k/6Pk1dHkfu/7+DmJ+7x5jA4Ab1ccjNgC4IRQbgHR6VWy299v+xPZ5289Wnn3I9kXbZ2rOnc7eZfuo7VXbZ20/XXn+zbY/sH16Ov/5mvNrGWq+pvM7y1gn+YqIXpwkzUn6t6TfS9om6bSkPRXnPyjpPklnOrjv2yXdNz1/q6R/Vb7vlrQwPT8v6YSkP3WdiZbv42DzNZ3fWca6yFefjtj2SjofEZ9GxPeSXpH0SK3hEfGupK9qzfvJ7C8j4uT0/LeSViXtqDg/IuLK9OL89JTtWaXB5ms6v7OMdZGvPhXbDklfXHN5TRX/c/eF7d1a/9SmE5XnztlekXRR0pGIqDq/AvI11UXGauerT8XmDb6W7ajhZ9lekPSapGci4puasyPih4i4R9JOSXtt311zfgWDz5fUXcZq56tPxbYmadc1l3dKutDRLtXZntd64F6KiNe72iMivpb0jqT9Xe1QyKDzJfUjY7Xy1adi+1DSnbbvsL1N0qOS3uh4pypsW9KLklYj4oUO5i/avm16/hZJD0k6V3uPwgabL6nbjHWRr94UW0RclfSUpLe1/sDmqxFxttZ82y9Lek/SXbbXbD9Za7ak+yU9Lmmf7ZXp6eGK87dLOmr7Y60XwJGIeLPi/OIGni+p24xVzxcvqQKQTm+O2ACgLRQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkM6PtHcdirMwi00AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Right ->\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAChCAYAAABNqJPmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACChJREFUeJzt3cGLnPUdx/HPp+tGhRUUZg8hyTYWREg9qCyhoPQQPKRevOpBEIScBIVePOo/4K2XgGF7EEXUg4hFcoiYgI1Ok40k3VhSUVwUsoMRzUWJfHvYkSZ2cZ5tnt/vefb7vF8wMLOZzPc7m08+PDuzM+OIEABk8puuFwCAtlFsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6dxS4kbtUUj7S9w0eu1zRUxcespoNIqlpaXSY9BDZ8+enUTE4qzrFSm2zVIbl7lp9NhylSlLS0s6efJklVnol4WFhS+aXI8fRQGkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5BOo2Kzfdj2p7Yv2X6+9FIYFvKFts0sNttzkv4i6U+SDkh6wvaB0othGMgXSmhyxHZQ0qWI+CwifpT0mqTHyq6FASFfaF2TYtsj6cvrLq9Pv3YD20dsj22PpY229kN+287XZDKpthx2pibFttUbB8b/fCHiaEQsR8SyNPN94ICfbTtfo9GowlrYyZoU27qkfddd3ivpqzLrYIDIF1rXpNg+lnSP7btt75L0uKS3y66FASFfaN3MtwaPiGu2n5H0nqQ5Scci4kLxzTAI5AslNPrMg4h4V9K7hXfBQJEvtI1XHgBIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkE6jl1QB+K+nHr6rs9krp650Nlvq9r5vB0dsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSGdmsdk+Zvuy7fM1FsLwkDG0rckR24qkw4X3wLCtiIyhRTOLLSI+kPRNhV0wUGQMbeMxNgDptFZsto/YHtseSxtt3Swg6cZ8TSaTrtdBz7VWbBFxNCKWI2JZWmzrZgFJN+ZrNBp1vQ56jh9FAaTT5Nc9XpX0oaR7ba/bfrr8WhgSMoa2zfzMg4h4osYiGC4yhrbxoyiAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOnMfEnVjhQvdDfbHc5GFSunrnS9AmbgiA1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0mnygcn7bJ+wvWb7gu1nayyGYSBfKKHJu3tck/TniDhj+w5J/7B9PCL+WXg3DAP5QutmHrFFxNcRcWZ6/ntJa5L2lF4Mw0C+UMK2HmOzvV/SA5JOb/FnR2yPbY+ljXa2w6A0zddkMqm9GnaYxsVme0HSm5Kei4jvfvnnEXE0IpYjYllabHNHDMB28jUajeoviB2lUbHZntdm6F6JiLfKroShIV9oW5NnRS3pZUlrEfFS+ZUwJOQLJTQ5YntI0pOSDtlenZ4eLbwXhoN8oXUzf90jIk5JcoVdMEDkCyXwygMA6VBsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANJp8g66O49f6HoDFPT5xVU99fBdnc1fOXWls9lde2Ph9x1vsNroWhyxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6TT4J/jbbH9k+Z/uC7RdrLIZhIF8oocm7e/wg6VBEXLU9L+mU7b9FxN8L74ZhIF9oXZNPgg9JV6cX56enKLkUhoN8oYRGj7HZnrO9KumypOMRcXqL6xyxPbY9ljba3hOJbTdfP1yj9/DrGhVbRPwUEfdL2ivpoO37trjO0YhYjohlabHtPZHYdvN16y2uvyR2lG09KxoR30p6X9LhIttg0MgX2tLkWdFF23dOz98u6RFJF0svhmEgXyihybOiuyX91facNovw9Yh4p+xaGBDyhdY1eVb0E0kPVNgFA0S+UAKvPACQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB1vvs9fyzdqb0j64v/86yNJkxbXYX69+b+NiOLvWXWT+ZK6/R7v5H/fPsxvlLEixXYzbI8339ON+UOcX0OX97Hr7+9Q5vOjKIB0KDYA6fSx2I4yf9Dza+jyPnb9/R3E/N49xgYAN6uPR2wAcFMoNgDp9KrYbB+2/antS7afrzz7mO3Lts/XnDudvc/2Cdtrti/Yfrby/Ntsf2T73HT+izXn1zLUfE3nd5axTvIVEb04SZqT9G9Jv5O0S9I5SQcqzv+jpAclne/gvu+W9OD0/B2S/lX5vlvSwvT8vKTTkv7QdSZavo+Dzdd0fmcZ6yJffTpiOyjpUkR8FhE/SnpN0mO1hkfEB5K+qTXvF7O/jogz0/PfS1qTtKfi/IiIq9OL89NTtmeVBpuv6fzOMtZFvvpUbHskfXnd5XVV/M/dF7b3a/NTm05Xnjtne1XSZUnHI6Lq/ArI11QXGaudrz4Vm7f4Wrajhl9le0HSm5Kei4jvas6OiJ8i4n5JeyUdtH1fzfkVDD5fUncZq52vPhXbuqR9113eK+mrjnapzva8NgP3SkS81dUeEfGtpPclHe5qh0IGnS+pHxmrla8+FdvHku6xfbftXZIel/R2xztVYduSXpa0FhEvdTB/0fad0/O3S3pE0sXaexQ22HxJ3Wasi3z1ptgi4pqkZyS9p80HNl+PiAu15tt+VdKHku61vW776VqzJT0k6UlJh2yvTk+PVpy/W9IJ259oswCOR8Q7FecXN/B8Sd1mrHq+eEkVgHR6c8QGAG2h2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0/gMkcBq9qQuj8AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Right ->\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAChCAYAAABNqJPmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACChJREFUeJzt3cGLnPUdx/HPp+tGhRUUZg8hyTYWREg9qCyhoPQQPKRevOpBEIScBIVePOo/4K2XgGF7EEXUg4hFcoiYgI1Ok40k3VhSUVwUsoMRzUWJfHvYkSZ2cZ5tnt/vefb7vF8wMLOZzPc7m08+PDuzM+OIEABk8puuFwCAtlFsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6dxS4kbtUUj7S9w0eu1zRUxcespoNIqlpaXSY9BDZ8+enUTE4qzrFSm2zVIbl7lp9NhylSlLS0s6efJklVnol4WFhS+aXI8fRQGkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5BOo2Kzfdj2p7Yv2X6+9FIYFvKFts0sNttzkv4i6U+SDkh6wvaB0othGMgXSmhyxHZQ0qWI+CwifpT0mqTHyq6FASFfaF2TYtsj6cvrLq9Pv3YD20dsj22PpY229kN+287XZDKpthx2pibFttUbB8b/fCHiaEQsR8SyNPN94ICfbTtfo9GowlrYyZoU27qkfddd3ivpqzLrYIDIF1rXpNg+lnSP7btt75L0uKS3y66FASFfaN3MtwaPiGu2n5H0nqQ5Scci4kLxzTAI5AslNPrMg4h4V9K7hXfBQJEvtI1XHgBIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkE6jl1QB+K+nHr6rs9krp650Nlvq9r5vB0dsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSGdmsdk+Zvuy7fM1FsLwkDG0rckR24qkw4X3wLCtiIyhRTOLLSI+kPRNhV0wUGQMbeMxNgDptFZsto/YHtseSxtt3Swg6cZ8TSaTrtdBz7VWbBFxNCKWI2JZWmzrZgFJN+ZrNBp1vQ56jh9FAaTT5Nc9XpX0oaR7ba/bfrr8WhgSMoa2zfzMg4h4osYiGC4yhrbxoyiAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOnMfEkVtile6Ha+O54/ACunrnS9AmbgiA1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0mnygcn7bJ+wvWb7gu1nayyGYSBfKKHJu3tck/TniDhj+w5J/7B9PCL+WXg3DAP5QutmHrFFxNcRcWZ6/ntJa5L2lF4Mw0C+UMK2HmOzvV/SA5JOb/FnR2yPbY+ljXa2w6A0zddkMqm9GnaYxsVme0HSm5Kei4jvfvnnEXE0IpYjYllabHNHDMB28jUajeoviB2lUbHZntdm6F6JiLfKroShIV9oW5NnRS3pZUlrEfFS+ZUwJOQLJTQ5YntI0pOSDtlenZ4eLbwXhoN8oXUzf90jIk5JcoVdMEDkCyXwygMA6VBsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANJp8g662A6/0PUG6X1+cVVPPXxXZ/NXTl3pbHbX3lj4fccbrDa6FkdsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5BOk0+Cv832R7bP2b5g+8Uai2EYyBdKaPLuHj9IOhQRV23PSzpl+28R8ffCu2EYyBda1+ST4EPS1enF+ekpSi6F4SBfKKHRY2y252yvSros6XhEnN7iOkdsj22PpY2290Ri283XD9foPfy6RsUWET9FxP2S9ko6aPu+La5zNCKWI2JZWmx7TyS23XzdeovrL4kdZVvPikbEt5Lel3S4yDYYNPKFtjR5VnTR9p3T87dLekTSxdKLYRjIF0po8qzobkl/tT2nzSJ8PSLeKbsWBoR8oXVNnhX9RNIDFXbBAJEvlMArDwCkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSMeb7/PX8o3aG5K++D//+kjSpMV1mF9v/m8jovh7Vt1kvqRuv8c7+d+3D/MbZaxIsd0M2+PN93Rj/hDn19Dlfez6+zuU+fwoCiAdig1AOn0stqPMH/T8Grq8j11/fwcxv3ePsQHAzerjERsA3BSKDUA6vSo224dtf2r7ku3nK88+Zvuy7fM1505n77N9wvaa7Qu2n608/zbbH9k+N53/Ys35tQw1X9P5nWWsk3xFRC9OkuYk/VvS7yTtknRO0oGK8/8o6UFJ5zu477slPTg9f4ekf1W+75a0MD0/L+m0pD90nYmW7+Ng8zWd31nGushXn47YDkq6FBGfRcSPkl6T9Fit4RHxgaRvas37xeyvI+LM9Pz3ktYk7ak4PyLi6vTi/PSU7VmlweZrOr+zjHWRrz4V2x5JX153eV0V/3P3he392vzUptOV587ZXpV0WdLxiKg6vwLyNdVFxmrnq0/F5i2+lu2o4VfZXpD0pqTnIuK7mrMj4qeIuF/SXkkHbd9Xc34Fg8+X1F3GauerT8W2LmnfdZf3Svqqo12qsz2vzcC9EhFvdbVHRHwr6X1Jh7vaoZBB50vqR8Zq5atPxfaxpHts3217l6THJb3d8U5V2LaklyWtRcRLHcxftH3n9Pztkh6RdLH2HoUNNl9StxnrIl+9KbaIuCbpGUnvafOBzdcj4kKt+bZflfShpHttr9t+utZsSQ9JelLSIdur09OjFefvlnTC9ifaLIDjEfFOxfnFDTxfUrcZq54vXlIFIJ3eHLEBQFsoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiCd/wDr0hq9Ove26wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Down v\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAChCAYAAABNqJPmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACCNJREFUeJzt3cGLnPUdx/HPp+umCisozB5Ckm0siJB6UFlCQekheEi9eNWDIAg5CQq9eKz/gLdeFgzbgyhSPYhYJIeIWbDRbbKRbDeWVBQXhexgRHNRIt8edkoTuzjPNs/v9zz7fd4vGJjZTOb7nc0nH56d2ZlxRAgAMvlF1wsAQNsoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHRuK3Gj9iikwyVuGr32mSLGLj1lNBrFwsJC6THoofPnz48jYn7a9YoU23aprZa5afTYYpUpCwsLOnPmTJVZ6Je5ubnPm1yPH0UBpEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQTqNis33c9ie2L9t+ofRSGBbyhbZNLTbbM5L+JOn3ko5IetL2kdKLYRjIF0pocsR2VNLliPg0In6Q9Jqkx8uuhQEhX2hdk2I7IOmLGy5vTr52E9snbK/aXpW22toP+e06X+PxuNpy2JuaFNtObxwY//OFiKWIWIyIRWnq+8AB/7HrfI1GowprYS9rUmybkg7dcPmgpC/LrIMBIl9oXZNi+0jSvbbvsb1P0hOS3iq7FgaEfKF1U98aPCKu235W0ruSZiSdjIj14pthEMgXSmj0mQcR8Y6kdwrvgoEiX2gbrzwAkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiCdRi+pAvBfTz9yd2ezl1eudjZb6va+7wZHbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUhnarHZPmn7iu2LNRbC8JAxtK3JEduypOOF98CwLYuMoUVTiy0i3pf0dYVdMFBkDG3jMTYA6bRWbLZP2F61vSpttXWzgKSb8zUej7teBz3XWrFFxFJELEbEojTf1s0Ckm7O12g06nod9Bw/igJIp8mve7wq6QNJ99netP1M+bUwJGQMbZv6mQcR8WSNRTBcZAxt40dRAOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSmfqSKgA3W1652vUKmIIjNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIp8kHJh+yfdr2hu1128/VWAzDQL5QQpN397gu6Q8Rcc72nZL+bvtURPyj8G4YBvKF1k09YouIryLi3OT8d5I2JB0ovRiGgXyhhF09xmb7sKQHJZ3d4c9O2F61vSpttbMdBqVpvsbjce3VsMc0Ljbbc5LekPR8RHz70z+PiKWIWIyIRWm+zR0xALvJ12g0qr8g9pRGxWZ7VtuheyUi3iy7EoaGfKFtTZ4VtaSXJW1ExEvlV8KQkC+U0OSI7WFJT0k6Znttcnqs8F4YDvKF1k39dY+IWJHkCrtggMgXSuCVBwDSodgApEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApNPkHXSxG/HHbue74/kVfHZpTU8/cndn85dXrnY2u2t/mftNxxusNboWR2wA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkE6TT4K/3faHti/YXrf9Yo3FMAzkCyU0eXeP7yUdi4hrtmclrdj+a0T8rfBuGAbyhdY1+ST4kHRtcnF2coqSS2E4yBdKaPQYm+0Z22uSrkg6FRFnd7jOCdurtlelrbb3RGK7zdf31+k9/LxGxRYRP0bEA5IOSjpq+/4drrMUEYsRsSjNt70nEtttvn55m+sviT1lV8+KRsQ3kt6TdLzINhg08oW2NHlWdN72XZPzd0h6VNKl0othGMgXSmjyrOh+SX+2PaPtInw9It4uuxYGhHyhdU2eFf1Y0oMVdsEAkS+UwCsPAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIx9vv89fyjdpbkj7/P//6SNK4xXWYX2/+ryKi+HtW3WK+pG6/x3v537cP8xtlrEix3Qrbq9vv6cb8Ic6vocv72PX3dyjz+VEUQDoUG4B0+lhsS8wf9PwauryPXX9/BzG/d4+xAcCt6uMRGwDcEooNQDq9Kjbbx21/Yvuy7Rcqzz5p+4rtizXnTmYfsn3a9obtddvPVZ5/u+0PbV+YzH+x5vxahpqvyfzOMtZJviKiFydJM5L+JenXkvZJuiDpSMX5v5P0kKSLHdz3/ZIempy/U9I/K993S5qbnJ+VdFbSb7vORMv3cbD5mszvLGNd5KtPR2xHJV2OiE8j4gdJr0l6vNbwiHhf0te15v1k9lcRcW5y/jtJG5IOVJwfEXFtcnF2csr2rNJg8zWZ31nGushXn4rtgKQvbri8qYr/ufvC9mFtf2rT2cpzZ2yvSboi6VREVJ1fAfma6CJjtfPVp2LzDl/LdtTws2zPSXpD0vMR8W3N2RHxY0Q8IOmgpKO27685v4LB50vqLmO189WnYtuUdOiGywclfdnRLtXZntV24F6JiDe72iMivpH0nqTjXe1QyKDzJfUjY7Xy1adi+0jSvbbvsb1P0hOS3up4pypsW9LLkjYi4qUO5s/bvmty/g5Jj0q6VHuPwgabL6nbjHWRr94UW0Rcl/SspHe1/cDm6xGxXmu+7VclfSDpPtubtp+pNVvSw5KeknTM9trk9FjF+fslnbb9sbYL4FREvF1xfnEDz5fUbcaq54uXVAFIpzdHbADQFooNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSOffcQoaPAvFYe4AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 360x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Right ->\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAATYAAAChCAYAAABNqJPmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAACCVJREFUeJzt3cGLnPUdx/HPp+tGhRUUZg8hyTYWREg9qCxBUHoIHlIvXvUgCEJOgkIvHus/4K2XgGF7EEWqBxGL5BAxARudJhtJurGkorhEyA5GNBcl8u1hpzSxi/Ns8/x+z7Pf5/2CgZnNZL7f2Xzy4dmZnRlHhAAgk191vQAAtI1iA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSOe2Ejdqj0LaX+Km0WtfKGLi0lNGo1EsLS2VHoMeOnv27CQiFmddr0ixbZbauMxNo8eWq0xZWlrSyZMnq8xCvywsLHzZ5Hr8KAogHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0GhWb7cO2P7N9yfZLpZfCsJAvtG1msdmek/QnSb+XdEDS07YPlF4Mw0C+UEKTI7aDki5FxOcR8aOkNyQ9WXYtDAj5QuuaFNseSV/dcHl9+rWb2D5ie2x7LG20tR/y23a+JpNJteWwMzUptq3eODD+5wsRRyNiOSKWpZnvAwf8x7bzNRqNKqyFnaxJsa1L2nfD5b2SLpdZBwNEvtC6JsX2iaT7bN9re5ekpyS9U3YtDAj5QutmvjV4RFy3/byk9yXNSToWEReKb4ZBIF8oodFnHkTEe5LeK7wLBop8oW288gBAOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQavaQKwH89+9g9nc1eOXW1s9lSt/d9OzhiA5AOxQYgHYoNQDoUG4B0KDYA6VBsANKh2ACkQ7EBSIdiA5AOxQYgHYoNQDozi832MdtXbJ+vsRCGh4yhbU2O2FYkHS68B4ZtRWQMLZpZbBHxoaRvKuyCgSJjaBuPsQFIp7Vis33E9tj2WNpo62YBSTfnazKZdL0Oeq61YouIoxGxHBHL0mJbNwtIujlfo9Go63XQc/woCiCdJr/u8bqkjyTdb3vd9nPl18KQkDG0beZnHkTE0zUWwXCRMbSNH0UBpEOxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUhn5kuqANxs5dTVrlfADByxAUiHYgOQDsUGIB2KDUA6FBuAdCg2AOlQbADSodgApEOxAUiHYgOQDsUGIB2KDUA6TT4weZ/tE7bXbF+w/UKNxTAM5AslNHl3j+uS/hARZ2zfJenvto9HxD8K74ZhIF9o3cwjtoj4OiLOTM9/L2lN0p7Si2EYyBdK2NZjbLb3S3pI0ukt/uyI7bHtsbTRznYYlKb5mkwmtVfDDtO42GwvSHpL0osR8d3P/zwijkbEckQsS4tt7ogB2E6+RqNR/QWxozQqNtvz2gzdaxHxdtmVMDTkC21r8qyoJb0qaS0iXim/EoaEfKGEJkdsj0p6RtIh26vT0xOF98JwkC+0buave0TEKUmusAsGiHyhBF55ACAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOk3eQRc7Sfyxu9nLl6uM+eLiqp597J4qs7aycupqZ7O79peF33a8wWqja3HEBiAdig1AOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHQoNgDpNPkk+Dtsf2z7nO0Ltl+usRiGgXyhhCbv7vGDpEMRcc32vKRTtv8aEX8rvBuGgXyhdU0+CT4kXZtenJ+eouRSGA7yhRIaPcZme872qqQrko5HxOktrnPE9tj2WNpoe08ktt18/XCd3sMva1RsEfFTRDwoaa+kg7Yf2OI6RyNiOSKWpcW290Ri283X7be5/pLYUbb1rGhEfCvpA0mHi2yDQSNfaEuTZ0UXbd89PX+npMclXSy9GIaBfKGEJs+K7pb0Z9tz2izCNyPi3bJrYUDIF1rX5FnRTyU9VGEXDBD5Qgm88gBAOhQbgHQoNgDpUGwA0qHYAKRDsQFIh2IDkA7FBiAdig1AOhQbgHS8+T5/Ld+ovSHpy//zr48kTVpch/n15v86Ioq/Z9Ut5kvq9nu8k/99+zC/UcaKFNutsD3efE835g9xfg1d3seuv79Dmc+PogDSodgApNPHYjvK/EHPr6HL+9j193cQ83v3GBsA3Ko+HrEBwC2h2ACk06tis33Y9me2L9l+qfLsY7av2D5fc+509j7bJ2yv2b5g+4XK8++w/bHtc9P5L9ecX8tQ8zWd31nGOslXRPTiJGlO0r8k/UbSLknnJB2oOP93kh6WdL6D+75b0sPT83dJ+mfl+25JC9Pz85JOS3qk60y0fB8Hm6/p/M4y1kW++nTEdlDSpYj4PCJ+lPSGpCdrDY+IDyV9U2vez2Z/HRFnpue/l7QmaU/F+RER16YX56enbM8qDTZf0/mdZayLfPWp2PZI+uqGy+uq+J+7L2zv1+anNp2uPHfO9qqkK5KOR0TV+RWQr6kuMlY7X30qNm/xtWxHDb/I9oKktyS9GBHf1ZwdET9FxIOS9ko6aPuBmvMrGHy+pO4yVjtffSq2dUn7bri8V9Lljnapzva8NgP3WkS83dUeEfGtpA8kHe5qh0IGnS+pHxmrla8+Fdsnku6zfa/tXZKekvROxztVYduSXpW0FhGvdDB/0fbd0/N3Snpc0sXaexQ22HxJ3Wasi3z1ptgi4rqk5yW9r80HNt+MiAu15tt+XdJHku63vW77uVqzJT0q6RlJh2yvTk9PVJy/W9IJ259qswCOR8S7FecXN/B8Sd1mrHq+eEkVgHR6c8QGAG2h2ACkQ7EBSIdiA5AOxQYgHYoNQDoUG4B0/g1aIhrVgBuMWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 360x360 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(6):  #take 6 steps and see if our player reaches the goal!\n",
    "    #plot_player_and_map(player_position=player_position, map_grid=map_grid)\n",
    "    current_state = np.stack([np.stack([player_position, map_grid], axis=-1)])\n",
    "    current_state_Q_values = model.predict(x=current_state, batch_size=1)\n",
    "    action = np.argmax(current_state_Q_values)\n",
    "    print(actionKeys[action])\n",
    "    player_position = player_step(old_position=player_position, action=action)\n",
    "    plot_player_and_map(player_position=player_position, map_grid=map_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nz_space_challenge",
   "language": "python",
   "name": "nz_space_challenge"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}

Created for a 'real' life application for crevasse avoidance. See this repository here