Sutton and Barto Racetrack: Sarsa

racetrack_sarsa.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sutton and Barto Racetrack: Sarsa\n",
    "Exercise 5.8 from *Reinforcement Learning: An Introduction* by Sutton and Barto.\n",
    "\n",
    "This notebook applies the **Sarsa** algorithm from Chapter 6 to the Racetrack problem from Chapter 5. \n",
    "\n",
    "Python Notebook by Patrick Coady: [Learning Artificial Intelligence](https://learningai.io/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class RaceTrack(object):\n",
    "    \"\"\"\n",
    "    RaceTrack object maintains and updates the race track \n",
    "    state. Interaction with the class is through\n",
    "    the take_action() method. The take_action() method returns\n",
    "    a successor state and reward (i.e. s' and r)\n",
    "    \n",
    "    The class constructor is given a race course as a list of \n",
    "    strings. The constructor loads the course and initializes \n",
    "    the environment state.\n",
    "    \"\"\"\n",
    "    def __init__(self, course):\n",
    "        \"\"\"\n",
    "        Load race course, set any min or max limits in the \n",
    "        environment (e.g. max speed), and set initial state.\n",
    "        Initial state is random position on start line with \n",
    "        velocity = (0, 0).\n",
    "        \n",
    "        Args:\n",
    "            course: List of text strings used to construct\n",
    "                race-track.\n",
    "                    '+': start line\n",
    "                    '-': finish line\n",
    "                    'o': track\n",
    "                    'X': wall\n",
    "        \n",
    "        Returns:\n",
    "            self\n",
    "        \"\"\"\n",
    "        self.NOISE = 0.0\n",
    "        self.EPS = 0.1  # epsilon-greedy coefficient\n",
    "        self.MAX_VELOCITY = 4\n",
    "        self.start_positions = []\n",
    "        self.course = None\n",
    "        self._load_course(course)\n",
    "        self._random_start_position()\n",
    "        self.velocity = np.array([0, 0], dtype=np.int16)\n",
    "\n",
    "\n",
    "    def take_action(self, action):\n",
    "        \"\"\"\n",
    "        Take action, return state' and reward\n",
    "        \n",
    "        Args:\n",
    "            action: 2-tuple of requested change in velocity in x- and\n",
    "                y-direction. valid action is -1, 0, +1 in each axis.\n",
    "                \n",
    "        Returns:\n",
    "            reward: integer\n",
    "        \"\"\"\n",
    "\n",
    "        if self.is_terminal_state(): return 0.0\n",
    "        self._update_velocity(action)\n",
    "        self._update_position()\n",
    "        if self.is_terminal_state():\n",
    "            return 100.0\n",
    "\n",
    "        return -1.0\n",
    "    \n",
    "\n",
    "    def get_state(self):\n",
    "        \"\"\"Return 2-tuple: (position, velocity). Each is a 2D numpy array.\"\"\"\n",
    "        return self.position.copy(), self.velocity.copy()\n",
    "    \n",
    "\n",
    "    def _update_velocity(self, action):\n",
    "        \"\"\"\n",
    "        Update x- and y-velocity. Clip at 0 and self.MAX_VELOCITY\n",
    "        \n",
    "        Args:\n",
    "            action: 2-tuple of requested change in velocity in x- and\n",
    "                y-direction. valid action is -1, 0, +1 in each axis.        \n",
    "        \"\"\"\n",
    "        if np.random.rand() > self.NOISE:\n",
    "            self.velocity += np.array(action, dtype=np.int16)\n",
    "            self.velocity = np.minimum(self.velocity, self.MAX_VELOCITY)\n",
    "            self.velocity = np.maximum(self.velocity, 0)\n",
    "            \n",
    "    def reset(self):\n",
    "        self._random_start_position()\n",
    "        self.velocity = np.array([0, 0], dtype=np.int16)\n",
    "\n",
    "    def _update_position(self):\n",
    "        \"\"\"\n",
    "        Update position based on present velocity. Check at fine time \n",
    "        scale for wall or finish. If wall is hit, set position to random\n",
    "        position at start line. If finish is reached, set position to \n",
    "        first crossed point on finish line.\n",
    "        \"\"\"\n",
    "        for tstep in range(0, self.MAX_VELOCITY+1):\n",
    "            t = tstep / self.MAX_VELOCITY\n",
    "            pos = self.position + np.round(self.velocity * t).astype(np.int16)\n",
    "            if self._is_wall(pos):\n",
    "                self._random_start_position()\n",
    "                self.velocity = np.array([0, 0], dtype=np.int16)\n",
    "                return\n",
    "            if self._is_finish(pos):\n",
    "                self.position = pos\n",
    "                self.velocity = np.array([0, 0], dtype=np.int16)\n",
    "                return\n",
    "        self.position = pos\n",
    "    \n",
    "\n",
    "    def _random_start_position(self):\n",
    "        \"\"\"Set car to random position on start line\"\"\"\n",
    "        self.position = np.array(random.choice(self.start_positions),\n",
    "                                dtype=np.int16)\n",
    "    \n",
    "\n",
    "    def _load_course(self, course):\n",
    "        \"\"\"Load course. Internally represented as numpy array\"\"\"\n",
    "        y_size, x_size = len(course), len(course[0])\n",
    "        self.course = np.zeros((x_size, y_size), dtype=np.int16)\n",
    "        for y in range(y_size):\n",
    "            for x in range(x_size):\n",
    "                point = course[y][x]\n",
    "                if point == 'o':\n",
    "                    self.course[x, y] = 1\n",
    "                elif point == '-':\n",
    "                    self.course[x, y] = 0\n",
    "                elif point == '+':\n",
    "                    self.course[x, y] = 2\n",
    "                elif point == 'W':\n",
    "                    self.course[x, y] = -1\n",
    "        # flip left/right so (0,0) is in bottom-left corner\n",
    "        self.course = np.fliplr(self.course)\n",
    "        for y in range(y_size):\n",
    "            for x in range(x_size):\n",
    "                if self.course[x, y] == 0:\n",
    "                    self.start_positions.append((x, y))\n",
    "                    \n",
    "\n",
    "    def _is_wall(self, pos):\n",
    "        \"\"\"Return True is position is wall\"\"\"\n",
    "        return self.course[pos[0], pos[1]] == -1\n",
    "        \n",
    "\n",
    "    def _is_finish(self, pos):\n",
    "        \"\"\"Return True if position is finish line\"\"\"\n",
    "        return self.course[pos[0], pos[1]] == 2\n",
    "    \n",
    "\n",
    "    def is_terminal_state(self):\n",
    "        \"\"\"Return True at episode terminal state\"\"\"\n",
    "        return (self.course[self.position[0], \n",
    "                           self.position[1]] == 2)\n",
    "    \n",
    "    \n",
    "    def action_to_tuple(self, a):\n",
    "        \"\"\"Convert integer action to 2-tuple: (ax, ay)\"\"\"\n",
    "        ax = a // 3 - 1\n",
    "        ay = a % 3 - 1\n",
    "        \n",
    "        return ax, ay\n",
    "    \n",
    "    \n",
    "    def tuple_to_action(self, a):\n",
    "        \"\"\"Convert 2-tuple to integer action: {0-8}\"\"\"\n",
    "        return int((a[0] + 1) * 3 + a[1] + 1)\n",
    "    \n",
    "    \n",
    "    def greedy_eps(self, Q):\n",
    "        \"\"\"Based on state and Q values, return epsilon-greedy action\"\"\"\n",
    "        s = self.get_state()\n",
    "        s_x, s_y = s[0][0], s[0][1]\n",
    "        s_vx, s_vy = s[1][0], s[1][1]\n",
    "        if np.random.rand() > self.EPS:\n",
    "            if (np.max(Q[s_x, s_y, s_vx, s_vy, :, :]) ==\n",
    "                np.min(Q[s_x, s_y, s_vx, s_vy, :, :])):\n",
    "                a = (0, 0)\n",
    "            else:\n",
    "                a = np.argmax(Q[s_x, s_y, s_vx, s_vy, :, :])\n",
    "                a = np.unravel_index(a, (3, 3)) - np.array([1, 1])\n",
    "                a = (a[0], a[1])\n",
    "        else:\n",
    "            a = self.action_to_tuple(random.randrange(9))\n",
    "            \n",
    "        return a\n",
    "    \n",
    "    \n",
    "    def state_action(self, s, a):\n",
    "        \"\"\"Build state-action tuple for indexing Q NumPy array\"\"\"\n",
    "        s_x, s_y = s[0][0], s[0][1]\n",
    "        s_vx, s_vy = s[1][0], s[1][1]\n",
    "        a_x, a_y = a[0]+1, a[1]+1\n",
    "        s_a = (s_x, s_y, s_vx, s_vy, a_x, a_y)\n",
    "        \n",
    "        return s_a        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Race Track from Sutton and Barto Figure 5.6\n",
    "\n",
    "big_course = ['WWWWWWWWWWWWWWWWWW',\n",
    "              'WWWWooooooooooooo+',\n",
    "              'WWWoooooooooooooo+',\n",
    "              'WWWoooooooooooooo+',\n",
    "              'WWooooooooooooooo+',\n",
    "              'Woooooooooooooooo+',\n",
    "              'Woooooooooooooooo+',\n",
    "              'WooooooooooWWWWWWW',\n",
    "              'WoooooooooWWWWWWWW',\n",
    "              'WoooooooooWWWWWWWW',\n",
    "              'WoooooooooWWWWWWWW',\n",
    "              'WoooooooooWWWWWWWW',\n",
    "              'WoooooooooWWWWWWWW',\n",
    "              'WoooooooooWWWWWWWW',\n",
    "              'WoooooooooWWWWWWWW',\n",
    "              'WWooooooooWWWWWWWW',\n",
    "              'WWooooooooWWWWWWWW',\n",
    "              'WWooooooooWWWWWWWW',\n",
    "              'WWooooooooWWWWWWWW',\n",
    "              'WWooooooooWWWWWWWW',\n",
    "              'WWooooooooWWWWWWWW',\n",
    "              'WWooooooooWWWWWWWW',\n",
    "              'WWooooooooWWWWWWWW',\n",
    "              'WWWoooooooWWWWWWWW',\n",
    "              'WWWoooooooWWWWWWWW',\n",
    "              'WWWoooooooWWWWWWWW',\n",
    "              'WWWoooooooWWWWWWWW',\n",
    "              'WWWoooooooWWWWWWWW',\n",
    "              'WWWoooooooWWWWWWWW',\n",
    "              'WWWoooooooWWWWWWWW',\n",
    "              'WWWWooooooWWWWWWWW',\n",
    "              'WWWWooooooWWWWWWWW',\n",
    "              'WWWW------WWWWWWWW']\n",
    "\n",
    "# Tiny course for debug\n",
    "\n",
    "tiny_course = ['WWWWWW',\n",
    "               'Woooo+',\n",
    "               'Woooo+',\n",
    "               'WooWWW',\n",
    "               'WooWWW',\n",
    "               'WooWWW',\n",
    "               'WooWWW',\n",
    "               'W--WWW',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 200\n",
      "Episode 400\n",
      "Episode 600\n",
      "Episode 800\n",
      "Episode 1000\n",
      "Episode 1200\n",
      "Episode 1400\n",
      "Episode 1600\n",
      "Episode 1800\n",
      "Episode 2000\n"
     ]
    }
   ],
   "source": [
    "# Problem Initialization\n",
    "\n",
    "course = big_course\n",
    "x_size, y_size = len(course[0]), len(course)\n",
    "# Q[x_pos, y_pos, x_velocity, y-velocity, x-acceleration, y-acceleration]\n",
    "Q = np.zeros((x_size, y_size, 5, 5, 3, 3), dtype=np.float64)\n",
    "position_map = np.zeros((x_size, y_size), dtype=np.float64)  # track explored positions\n",
    "\n",
    "N = 2000   # num episodes\n",
    "gamma = 1.0\n",
    "alpha = 0.1\n",
    "track = RaceTrack(course)\n",
    "\n",
    "# Sarsa\n",
    "\n",
    "epochs = []\n",
    "counts = []\n",
    "count = 0\n",
    "for e in range(N):\n",
    "    if (e+1) % 200 == 0: print('Episode {}'.format(e+1))\n",
    "    track.reset()\n",
    "    s = track.get_state()\n",
    "    a = track.greedy_eps(Q)\n",
    "    while not track.is_terminal_state():\n",
    "        position_map[s[0][0], s[0][1]] += 1\n",
    "        count += 1\n",
    "        r = track.take_action(a)\n",
    "        s_prime = track.get_state()\n",
    "        a_prime = track.greedy_eps(Q)\n",
    "        s_a = track.state_action(s, a)\n",
    "        s_a_prime = track.state_action(s_prime, a_prime)\n",
    "        Q[s_a] = Q[s_a] + alpha * (r + gamma * Q[s_a_prime] - Q[s_a])\n",
    "        s, a = s_prime, a_prime\n",
    "    epochs.append(e)\n",
    "    counts.append(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkUAAAGHCAYAAAC+tibjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XecVPX1//HXAUEFBWyAHbAgiAisDXvEEmKNiWXVaH52\njQ1jSzRKNPnGEsUejRpNc20xVpQoFowiSFNEQKXYaYKgiLQ9vz/OnezdcXbZHWZ3tryfj8d9zMy9\nn3vvubOre/hUc3dEREREmrsWxQ5AREREpCFQUiQiIiKCkiIRERERQEmRiIiICKCkSERERARQUiQi\nIiICKCkSERERAZQUiYiIiABKikREREQAJUUiBWFmM83sL0W692AzKy/wNfcxs3Iz27uQ15XGK/l9\nuLKe7/mAmc2oz3tK86akSKQaZraDmT2WJD1LzOxTM/uPmZ2TVbQcKNaaOZ7vvc3sLDM7qZrr1jsL\nJ5rZm2b2pZktMrOpZvZXM9s1Va6HmV1lZlsUI86GLPl9La9iG5rnZfP+PVsNxbinNGNrFDsAkYbK\nzHYHXgI+Av4MzAI2B3YDzgNuTxXvTiRGjc3ZwFzgr+md7v6qma3t7suKENNtSVxPAP8AVhDf70Bg\nGjAqKdcTuAp4Gfi4/sNs0BwYD/wRsKxjn+d5zbWJn4VIk6WkSKRqlwNfATu5+9fpA2a2Yfqzuy+v\nz8DqQzESIjPrCJwF3O3uZ2UdHpT1vRuqRajOZ+5eVqiLFSlBFqlXaj4TqVo3YFJ2QgTg7vPSn7P7\nFJnZSUlTxR5mdquZzTGzBWZ2l5mtYWbtzexvZjY/2a7Lul7OPj1mtmWy/8TqAjez/2dmw81stpl9\nZ2aTzOzMrDIzgO2BfVNNKy+t4v5HmdkYM/vWzOaa2d/NbJOsMg+Y2ddmtomZPZG8n2NmN5hZdq1F\ntq5EsvNGroOZ7z1p8nsk2f1KEuvKdLxmNtDMRpjZN0kT3DNm1rOKWLua2bCk7Gdm9psc3+mxybMv\nMrOFZvaOmZ1X1YMkP+cvzey+HMfWTZpjr0/tO9fM3jWzxcnvxFtmdmz1X9fqqeXzV+pTZGbrmNnN\nZjYj+R2bbdG03CfrvFX+ziTljkief0ny3R5RRcxmZhekys5K/rvqkFVup+SZ5ib3np7rZyGSpqRI\npGofASVmtn0NylZVY3EbsBVwJfAkcBrwO+Bp4o//r4DXgIvM7IQaXrMmzgRmAr8HLiSal+40s3Tt\ny/nAp8Bk4HjghKR8zvub2c+Bh4HlwGVEk+KRwGtm1i7rvBbAMKJp7pfAK0kcp68i7o+S16PMbO1q\nyr0K3Jq8/10S+8+SZ8HMfgY8A3wNXAJcDfRIYk33QcrE+jzwBXAxMAb4rZkNTj37AcCDwJfJ9S4l\nmu12rypAd18B/Bs4wsyya+V/DLQGypLrnwbcArxL/FyuJJq/diV/rcxsgxzbWukwqcHzV+Fu4Azg\nUaJ27wbgW+J7Jnmun1OD3xkzOxB4jGieu4xoOr0f2CnHff8MXEf8d3Me8Bfi9/d5M2uZXG8j4vdv\nC+APwDlEU+zqfJ/SHLi7Nm3acmzA/sAy4n/orwPXAgcAa+QoOwP4S+rzSUQfo2ezyr0OrARuT+1r\nQSQtL6X27ZOU2zvr/C2T656Y2ncVsDKr3Jo5YnwO+CBr38T0fau6P9HUPguYALROlftREs9VqX33\nJ+f+OuuaY4HRNfjeH0jO/xL4F5FMdc9R7idVfEdtgfnAn7L2bwQsAO7KEeuQrLJPA0uA9ZPPQ4AF\nefwOHZB8Pz/K2v9s+mdBJE/vFPB3d0Zy3+xtJXBJbZ8/2VcOXJn6vAC4tZoYavM7M55I0NdJ7RuQ\nlJue2rdnsu+YKr7nY5PPhyfP1bdQ36m25rGppkikCu7+ItCfqOHpTfwrehjwmZkdWpNLEP+KTct0\nEv7ffncvJ/513m11Y05dc2nmvZm1M7MNgBFANzNbN49L7gR0BO70VN8Sdx8KTAEOznHO3VmfX6MG\nz+juPyf+ZT8dOIKogZhsZi/manbJ4QCgPfBQuoaE+HmMAn6Q45w7sj7fDqxJJMYQfcvamtlBNbh/\n2kvAPOCYzI6kmWd/4KFUua+AzcwsV81Ivt4kEov9U9sBJLVTWXI9f2sqnj+Xr4BdzWzjKo7X6HfG\nzDoDOwIPuPs3qXLDgfeyrvnT5L7Ds36244FvqPjZfkXUxB6Wo5ZOpEpKikSq4e5j3f2nwHrALsD/\nAesAj5rZdjW4RPaoqIXJ6yc59q+3OrGmWfRletHMviH+QMylommsfR6X3JJIKt7PcWxKcjztO3f/\nMmvfAmr4jO7+J3ffGdiQ+Ff/UGA/cv9Bz7YN8QfxZeK5M9scIinomFW+nEjA0jLP2SV5vTPZN9TM\nPjGz+2qSILn7SqK263Aza5Xs/glRi/JIquh1xB/10Wb2vpndbjH6cXXMc/eX3f2lrC37d6+q5zcq\nnj+XS4BewCdmNspieoSuqeM1/Z3JvH6Yo9zUrM/bAB2In2X2z7Ytyc/W3V8lmuOuBOZZ9G37uZm1\nruZ5RJQUidSEu69IEqQriOHirYGjanDqylrsT3dCrqo/UctV3dDMugEvAusDg4jmiv2JJiCon//u\nq3ruWnH3Be7+jLsfQvQj2tPMNl/FaS2I7+94KteSZGpKDs8jjrlAH+AwouZwX+A5M7u/Bqc/BLQj\nphQAOBqY4u4TU9efQkw7cAxRo3Yk8F8zu6q2sdYXd3+UqPk7B/gMuAiYlEdtWm20AGbz/RqwzM/2\nfx3B3f1ooqb3NmATonZ2jJm1qcP4pJFTtaJI7Y1JXqtqNiiEBUSS1CFrf5canHsokbQd6u6fZXaa\n2YAcZWvamfujJJ7uRKfptO5UdJCuS2OAvYnv/ROqjn0aEetcd3+pBtdtQfxxT9dUdE9eZ2Z2eHSc\nfjbZMLM/Aaeb2TXunl3TkjaC6MR8jJm9TjTxXJNdyN2XEJ2WH02afP4NXG5mf/C6HQ5fo+fPxd1n\nA3cBd1lMlzCemMpiGDX/ncm8bpPjFt2zPk8jEqI30k3E1cQ3GhgN/MbMSoF/Asfy/WZtEUA1RSJV\nMrN9qziU6T+TXbVfSB+RdCLO2n82q05kMrU0//vv28zaAz/PUXYx30+8chlDNFGcmWoGwswGEqON\nnqnBNVbJzDqZWY8c+1sRtQHlVPzxXkzuxHEYsAj4da7+JJY1x1Qie4byc4hO9sOTc9bPcU6mpmfN\nnA+TcHcnmnIOJUbItaRy09n3rp8kYJOJ52uVlFnbzLonfWgKrdrnz2ZmLbJGHOIxXcLnVHwfNfqd\ncfdMZ+yT0v3dkhF/laZQIL63NUjVCKXKt0x+zzP9trK9nbxW+/OS5k01RSJVuy2pav830QeiNbAH\n0fwxnRi5U51VzclTJXdfZGaPAudZTO0zDTiEGEG1Kv8hRsw9Y2Z3A+sCpxLNDp2zyo4l/mhdTiQb\nc9z95ez43X2FmV1K/At7hJmVJdc6j/gubs7rQb9vM6JfzUvEH+RZRD+RUqKz+xB3n5+UnUAkgJcm\nfwSXAsPdfV4y9cDfgHFm9hDR72QLIqH9bxJ3xlLgh2b2ANER+0dEU9fvU/2i7k0Sl5eIUVJdiMRh\nvLtPrsFzPQycC/wWmOju2Qn1f8xsFjE6cTaRDPwCeMbdFydldiH6SQ0mphhYlU3N7Pgc+79x9ydT\nn2vy/NnWBT41s8eIZOMbovlqJ2K0YG1/Z35FJEmvW8z3tQHx/b5L9OEjueaI5Hf6Mov5kDK/69sS\nnbDPAx4nEqyzif92pyXxnkb03ct3mRNpDoo9/E2btoa6AQcC9wCTiP+ZLiFqh4YAG2aVnQ7cl/p8\nEvEHu19WuauS/etn7b8fWJi1bwPiX8ZfEyOY7iD+hb2S7w/JX5F17sFEU8Zi4o/CL4maopXAFqly\nHYGniM7YK0mG51P1lAA/JWoAvqVieZCNV/UsVcWZo8w6xB/DoURt2XdJbP8F/l+O8icDHxC1GpXi\nJWrZhhLD8xcTHX7vIzVMO4l1EZHkPJ98158Dv8m6z4+JKQ2+SH4PZiQ/j461+H3K1P5dluPYqUTC\nMyf5bt8n5tdJD1HP/Ex+U4N7zUjK5tqm1/b5k7L/uzdRe3UtMC75+SxK3p+e47xV/s4k5Y4gkqBv\niVq4w5P4puUoewrRLJYZSDCBGATRKTneh5iXaEZyvS+IuY80RF9btZu5a5Z8EWmeko7SP3H3dqss\n3AQ19+cXydYg+hSZ2V5m9pTF9PLlZnZYjjI9zOxJM/vKYir6UWa2Wer4mmZ2h5nNs5i2/jGLdZTS\n19jczJ61mEZ/lpldb2Ytssrsa2ZjLaatf99yrCBuZr+wmNp+icVK3jsX8vsQERGR+tcgkiJifokJ\nVNGJ1My2IoapvkdUie9AjN74LlXsZqLJ4CdJmU2I+UEy12hBVKWvQaxyfhLRnHB1qkwXol17ODGZ\n2C1EX4IDUmWOAW4kmgL6Eu3pw6rovCkiIiKNRINrPjOzcuAId38qta8MWObu36u1SY63I9qqj3X3\nfyf7uhOjN3Zz99HJiIeniLbszKKSZxDt4ht5dAq8Dhjo7r2z7t3e3X+UfH4TGOXu5yefjRgefKu7\n/29xRxFp+JLmoyPdPZ8JLRu95v78ItkaSk1RlZKk42DgAzN73mIl5jfNLD0BWwlRA/S/4aMeozs+\nJibvgqgdmuiVVzcfRszuu32qzItZIQzLXCMZVlqSdR9PzumPiDQq7v7/mnNC0NyfXyRbg0+KiNEx\n6xCrUg8lhn3+G3jczPZKynQmapIWZZ2bHoLcOfmcfZwalGlnZmsSSw60rKJM9lBnERERaUQawzxF\nmcTtCXe/NXn/jsW6QGcSfY0arGSitYOImWG/q760iIiIpKxFTBkxzKueN6tgGkNSNA/IzO6aNpmY\nSA9igrfWZtYuq7aoU3IsUyZ7lFin5PWLVJlOOcoscvelZjaPmKsjV5lZ5HYQMbW8iIiI5Od44MG6\nvkmDT4rcfbmZvcX318DZloo1c8YSidMAomkt09F6C+CNpMxIYtr/DVP9ig4kJuWbnCqTWbSRVJmR\nqVjGJvd5KrmPJZ9vJbeZAP/4xz/o0eN7qxdIIzVo0CCGDBmy6oLSKOjn2bTo59l0TJ48mRNOOAFW\nsQ5foTSIpMjM2gJbU7GsQDcz2xGY7+6fADcAD5nZa8SsrwOJJQ/2gf8tiXAfcJOZLSBmZb0VeN3d\n30qu+R9iSP/fk6nnNyaG9d/u7suTMncBv0hGof2FSHZ+Skx7n3ET8ECSHI0mViFvAzxQxeN9B9Cj\nRw/69euXz9cjDVD79u3182xC9PNsWvTzbJLqpftJg0iKiPVyXibmKHJiHiCI6eBPdvcnzOxM4NfE\n3EFTiWGkI1PXGEQ0bT1GLPj3PLF2EADuXm5mhwB/ImqPFhOJzFWpMjPN7GBiGYfziDWOTnH3F1Nl\nHknmJLqaaDabABzk7nML81WIiIhIMTSIpMjdX2UVI+Hc/QGqro3B3ZcSCy6eW02ZT4gapuruM4IY\ndl9dmTuBO6srIyIiIo1LYxiSLyIiIlLnlBSJ5KG0tLTYIUgB6efZtOjnKflSUiSSB/1Pt2nRz7Np\n0c9T8qWkSERERAQlRSIiIiKAkiIRERERQEmRiIiICKCkSERERARQUiQiIiICKCkSERERAZQUiYiI\niABKikREREQAJUUiIiIigJIiEREREUBJkYiIiAigpEhEREQEUFIkIiIiAigpEhEREQGUFImIiIgA\nSopEREREACVFIiIiIoCSIhERERFASZGIiIgIoKRIREREBFBSJCIiIgIoKRIREREBlBSJiIiIAEqK\nRERERIAGkhSZ2V5m9pSZfWZm5WZ2WDVl70rKnJe1fz0z+6eZLTSzBWZ2r5m1zSrT28xGmNkSM/vI\nzC7Ocf2jzGxyUuZtMxuYo8zVZva5mX1rZi+Y2dar8/wiIiJSfA0iKQLaAhOAswGvqpCZ/RjYFfgs\nx+EHgR7AAOBgYG/g7tS56wLDgBlAP+BiYLCZnZoqs3tynXuAPsCTwBNm1jNV5lLgHOB0YBdgMTDM\nzFrX9qFFRESkspUrYcYMeOYZuPfe+r33GvV7u9zc/XngeQAzs1xlzGxT4BbgIGBo1rHtkv0l7j4+\n2Xcu8KyZXeTus4ATgFbAKe6+AphsZn2BC4HM134e8Jy735R8vtLMDiCSoLOTfecD17j7M8l9TgRm\nA0cAj6zWFyEiItKMzJoFb70Fb7wBEyfChx/C9OmwfHkcX3fd+o2nQSRFq5IkSn8Drnf3yTnypv7A\ngkxClHiRqHXalajx2Q0YkSREGcOAS8ysvbsvTK5zY9a1hwGHJ3F0AzoDwzMH3X2RmY1KzlVSJCIi\nkmXhQpgwAcaPh3ffhQ8+iO2LL+J4x46wyy4wcCBsvXVsPXrA3Lmw0071F2ejSIqAy4Bl7n57Fcc7\nA3PSO9x9pZnNT45lykzPOm926tjC5HV2jjKZa3QiEq3qyoiIiDRLc+bA5MkwdSq8/35skyZF7Q/A\nWmtBz56w7bawzz6www6w666w+eaQq51o3rz6jb/BJ0VmVkI0a/Wtq1vU0XUrGTRoEO3bt6+0r7S0\nlNLS0vq4vYiISEGUl8Onn1ZOfCZPjuavWbOiTIsW0LVrJD+HHw59+8a23XawRhWZR1lZGWVlZZX2\nLVy4sI6fprIGnxQBewIbAZ+kms1aAjeZ2QXu3g2YBXRMn2RmLYH1gaRyjllETU9apuZn1irKpI9b\nsm92VpnxVGPIkCH069evuiIiIiINxvLlMG1a9POZOhWmTInE5513YMmSKNOqFWy1FXTvDqeeCjvu\nCNtvD926wZpr1u5+uSoKxo0bR0lJSYGeaNUaQ1L0N+CFrH3/Sfbfn3weCXQws76pfkUDiARmdKrM\n78yspbuvTPYdCExN+hNlygwAbk3d64BkP+4+w8xmJWXeATCzdkS/pTtW90FFRESKYe7cSH7efx/G\njYsk6K23YP78ON62bSQ+PXvCUUdFf5/u3WHLLauu+WmMGsSjJPMJbU1FU1Y3M9sRmO/unwALssov\nB2a5+wcA7j7FzIYB95jZWUBr4DagLBl5BjHU/krgL2Z2HbAD0Sx3furStwCvmNmFwLNAKVACnJYq\nczNwhZl9CMwErgE+JTpzi4iINFjffhs1PpMnx/beezBmDHzySUWZbbaJpOess2DAgPi86aa5+/w0\nNQ0iKQJ2Al4mmrKcihFgfwVOzlE+11xGxwG3E6POyoHHSCU8ySixA4kanTHAPGCwu9+XKjPSzI4D\nfp9sHwCHu/t7qTLXm1kbYg6kDsBrwEB3X5bHc4uIiBTcggWR8GSSn8z20UfgyV/QjTeO5OfYY2Hn\nnaPmp1s3WGed4sZeTOZe5VyJUgBm1g8YO3bsWPUpEhGRglq+PGp+xo+Pvj6TJkW/n8+SKY7NItHp\n0aPytt120KFDcWOviVSfohJ3H1fX92soNUUiIiJSjW++icRn/PjKc/4sXRrHu3aFXr3gxBPjdfvt\nY/TX2msXN+7GREmRiIhIA7JyZXR4zjR/vftuJEAffBBNX61aRcLTp08kQH37Qu/ekDXri+RBSZGI\niEiRLFwYTV7vvhuv77wTtUBffRXH118/EqCDDoLLLotEqGfP2g93l5pRUiQiIlJHystjKYtp0yq2\n6dMr3n/5ZZRr0SJGefXqBb/8Jey+e7zfaKPmMeqroVBSJCIisprKy+Hjj6Om5+23Y3vvvVjt/bvv\nKsptumlMdtizJxx6aLzPdHxea63ixS9BSZGIiEgNuUfNz8SJFVum78/ixVFmvfViZucBAyLpyWxd\nu6rTc0OnpEhERCSHuXOjr8/kyVED9O67kQAtSKYTbtMmmrh69YKjj654v9lmavJqrJQUiYhIs7Zi\nBcycWbG+18iRMdors7L7GmtEE9cOO8DAgdHxeYcdouanRYuihi4FpqRIRESahfLyWN9rzJho9po6\nNbYPPohJECFqf/r1gyOOgJ12iuHuW20Vw+Cl6VNSJCIiTc6KFbGe16RJMHw4jB0bnZ8XLYrjm20W\nnZv32y/W+Npuu1jmYtNNVfvTnCkpEhGRRm3FiqgBmjQJ/vtfGDo0mr5WrIjjW2wBu+0WTV8lJbFt\nsEFxY5aGSUmRiIg0GpkmsLfeqtjGj4clS+L4JpvA4YfDoEGx5tc220CXLur4LDWjpEhERBqcxYtj\ncsMPPoglLzKrvE+ZEmuAQfT12XlnOPLI6PvTqxd07FjcuKVxU1IkIiJFk5n3Z8KEGO7+zjvwxhuR\nEGW0bx+jv3r3hmOOiTmASkpiCQyRQlJSJCIidc49Oj5PnhxNXlOmRA3QBx9UrPPVtm0Mdx84MEZ+\nbbNNbBtuqOYvqR9KikREpKC++SaSngkTYNSoisVOv/46jnfoEMnP9tvDj38co7769YsO0Rr5JcWk\npEhERFbLV1/Bc8/Bo49GLdCnn8Z+s2jy6t07kp/tt481v7bcUsmPNExKikREpEaWLYvmrsyyF5mF\nTz/6KI7vthuccEIkPtttF69t2xY3ZpHaUFIkIiKVLFkSMz1PmhTD3SdPjmRo+nRYuTLKdOoUHZ6P\nOipe99wzhr6LNGZKikREmqkvv4zOzpk1v957L7bp06NjNMT6Xr16wcEHR+1Pjx6xbbRRcWMXqQtK\nikREmrivv47Ozm+/Het+TZoUydD8+RVlttgi+vwccUQ0e/XsGclP+/bFi1ukvikpEhFpIlaujPl9\nMgudjh4didAHH0TNzxprRK1P795w6KGw7baxbb11LIQq0twpKRIRaWTKy+GzzyIBmjYt+vy8+WYM\ngV+8OMq0aROzPA8cCJddBn36RM3PWmsVN3aRhkxJkYhIA7ViRfTxmTgx1vv64IPYpkypWO3dDDbf\nHHbfPdb86ts3+v5suqkmPBSpLSVFIiINwLJlkQCNHRvNXqNHRw3Q0qVxvGPHmN25R49Ifnr3jmav\nLl1U+yNSKEqKRETq2YIF0ddnwoSK7b33YPnymNSwR4+Y8+fkk2O4e58+0K5dsaMWafqUFImI1KH5\n82Oun/Hjo9/PmDEVkx2utRbssAPssgucfnokQDvuCOusU9yYRZqrBpEUmdlewMVACbAxcIS7P5Uc\nWwP4PTAQ6AYsBF4ELnP3L1LXWA+4HTgEKAf+BZzv7otTZXonZXYG5gC3u/sNWbEcBVwNdAHeT+7z\nXFaZq4FTgQ7A68BZ7v5hIb4LEWmc3GHWrBju/s478PrrMQz+/ffjeNu2sb7X0UdHzU+fPjHya40G\n8X9hEYEGkhQBbYEJwH3A41nH2gB9gN8C7wDrAbcCTwK7pMo9CHQCBgCtgQeAu4ETAMxsXWAY8B/g\nDGAH4H4zW+Du9yZldk+ucynwLHA88ISZ9XX395IylwLnACcCM4HfAcPMrIe7LyvItyEiDdry5dHZ\nedSoaAabODFeM6u9r7027Lwz/PCHcPnlsOuu0f+nZcvixi0i1TPPTFvaQJhZOamaoirK7ASMArZ0\n90/NrAcwCShx9/FJmYOIxGYzd59lZmcB1wCd3X1FUuYPwOHu3jP5/BDQxt0PS91rJDDe3c9OPn8O\n3ODuQ5LP7YDZwEnu/kiOWPsBY8eOHUu/fv1W78sRkXrlDvPmVaz4Pnp0JECTJ0fH6BYtYoX37beP\nmp/MgqdbbaUESKQQxo0bR0lJCcTf93F1fb+GUlNUWx0AB5J/l7EbsCCTECVeTMrsStQq7QaMyCRE\niWHAJWbW3t0XAv2BG7PuNQw4HMDMugGdgeGZg+6+yMxGJed+LykSkYZv5cro55MZ8v7++5EIjRsX\nS2FANHOVlEQN0CmnxOivkhL1/xFpShpdUmRmawLXAg+6+zfJ7s5EH6H/cfeVZjY/OZYpMz3rcrNT\nxxYmr7NzlMlcoxORaFVXRkQasBUrKmp+xo2D116Lvj/ffRfHW7eOmp5tt4Vzz40ZoLfbLpq/1lyz\nuLGLSN1qVElR0un6USIxObtQly3QdUSkgVm8uGLI+/jx8fruuxVz/3TpAnvtBccfH81e22wTa4Cp\n6UukeWo0SVEqIdoc2C9VSwQwC+iYVb4lsD7wRapMp6zLZmp+Zq2iTPq4JftmZ5UZTzUGDRpE+6yV\nFUtLSyktLa3uNBGpgfJy+OSTilXex4yJjs9Tp8axVq2ixqdPHzjxxHjt3Rs6dCh25CKSUVZWRllZ\nWaV9CxcurNcYat3R2sz2A5a4+8jk86nE8PT3gAvcfdFqBZSjo3UqIeoG/MDd52edsx3R0XqnVEfr\nA4GhVHS0PpMYKdbJ3VcmZf4vuVe6o/Xa7n546tqvA2/XoKP1ie7+aI7nUUdrkQKaOzeGvI8fH6+T\nJkVz2LffxvE2bSLh2WmnSH5KSqIWqHXr4sYtIrXXGDpaDwEuBzCznsS8P3cCewM3EQlSrZhZW2Br\nKpqyupnZjsB8oqbnX8Sw/EOAVmaWqc2Z7+7L3X2KmQ0D7klGmbUGbgPK3D1Ty/MgcCXwFzO7jhiS\nfx5wfiqUW4BXzOxCYuRaKTF30mmpMjcDV5jZh8SQ/GuAT4nO3CJSQEuWwMsvx5w/Y8ZEEjQr+S+6\nbduY+LBfv2j+2m67GP21+eYxKkxEpLbySYq2ImplAH4KPO/uFybD5J/OM46dgJeJpiynYgTYX4n5\niQ5N9k9I9lvy+QfAiGTfcUSC9iIxeeNjpBKeZJTYgcAdwBhgHjDY3e9LlRlpZscRk0X+HviAGLL/\nXqrM9WbWhpgDqQPwGjBQcxSJrB53mDkzhrxPmgT//W8kREuWQKdOMevzqadGLdAOO0T/H/X9EZFC\nyicpWg5klh/cn6iBgUgy2uc8YxXc/VWgun/brfLffe7+FclEjdWUeRfYZxVl/kXUTFVXZjAweFUx\niUhuS5dGf5+JE6P2J9McNjvpqdeuXUx4OHgwHHpo1AJpxXcRqWv5JEVvANeZ2WvE3D/HJ/u3Bj4v\nVGAi0jR89VXF6K9x42KbOjXmBoIY7dW7d6z9tfPO0Ry2ySZKgkSk/uWTFJ0L/JnoO3S+u3+S7D+c\naLoSkWZqzpzKyc/48TBtWhxbe+1IfvbZp2L+n169NAJMRBqOWidF7j6daDbL3n9uQSISkQbPHT79\ntHLyM26rDKOLAAAgAElEQVQcfPZZHG/XDvr2hcMOi5qfvn1jOQwtfioiDVle/4syMwMOBnokuyYD\nQ929vFCBiUjD4B7NXWPGwNixFX2A5s2L4xttFInPiSdG8tOvH3TtqhFgItL41DopMrPuxPDzLlQs\nm9ENmGlmR7j7lMKFJyL1KbMExsSJMfPzxInRH+iTpJF8q61i7p9zz61IgNT/R0Sainxqiu4DZgB7\nu/scADPrSAyfvwfYq3DhiUhdWbkSZsyIFd/HjIkh8KNGxdIYAJttFvP+lJbC3nvDHnuo/4+ING35\nJEUlwM6ZhAjA3eeY2SXA6IJFJiIF4R59fd59N5a+mDAhlsKYOrViDbANNoA994Srror5gHbcUQmQ\niDQ/+SRFHwIb5Ni/PlGDJCJFNHMmvPJK1P5MmRKJUKb/z7rrRsLTvz+ccgr06BHbppuqCUxEJJ+k\n6CLgFjO7Angz2bcbcDUwyMz+t8KQZnkWqXtLlsAbb8Tsz8OGRTJkFhMe9uwJv/hF9P3p1StWhVcH\naBGR3PJJip5LXp8iltqAijXLhmaV1ST8IgWyYkU0eY0bF01gkybB++9HzZB7jALbd1+4+GLYf39Y\nf/1iRywi0rjkkxQNLHgUIlJJeTl89FHM//P22/DaazByJHz3XRzv1i3W/zrqKNh221gSY/vt1QQm\nIrI68pm8cVhdBCLSXLnHkPcJE+Cpp2IOoMmT4Ztv4ninTlBSAr//fbz26QPt81plUEREqpPv5I27\nAKcDWwHHu/vnZnYsMNPd36z+bJHmyx0+/jgmQUxvmY7QXbrAD34QNUC9ekWn6E02KWrIIiLNRj6T\nNx4GPAw8BvQH1koOdSRWqT+kYNGJNGLuMH16xVIYY8fG65dfxvHOnaPm5+yzYaedogZo003VEVpE\npFjyqSm6CjjH3e8zsyNS+/8L/KowYYk0PgsXxgzQw4fDm29GR+jMTNCbbRYjwM49N15LSlQDJCLS\n0OSTFG0HDM+x/ytgvdULR6Rx+OILeP316AidWQvs44/jWOvWMfrr6KNjNNjOO0e/IBERadjySYrm\nAF2BmVn7+6PJG6UJWrQoEqBRo6Im6M034fPP49jGG0ez17HHQu/esXXvHomRiIg0LvkkRfcDN5vZ\nicQ8RRuYWV/gj8D1hQxOpBi+/hqefrqiL1BmKPyGG8Yw+BNOgN12i2HwagITEWk68kmKfge0AkYS\nnazfBFYAtwI3Fy40kfrhDqNHwxNPRG3Qm2/GLNFdu8ZK8NdcA4ceGvMBaR4gEZGmK595isqB35jZ\ntUB3YB1gorsvKHRwInXhu++iD9BLL0Ut0Jgx0RzWsWOsCXb11TEkfsstix2piIjUp3yG5N8JXOLu\n3wDjUvvbAH9097MLGJ/IavvyS3j88agNGjkyFklduRLatoXdd4fjj4+O0T/4AbRqVexoRUSkWPJp\nPjsDGAx8k7W/DTGho5IiKbr//AeeeQZGjIhaIfdYHHXvveG882JSxL591SFaREQq1DgpMrPWxMKv\nBrROPme0BPYD5hU2PJGaWbEi1ggbOxb+8Y9YK6xr1xgSf8EFcMABMTGiiIhIVWpTU/QdMdrMgY+q\nKPP71Y5IpAZWrozZoseMgRdegOeeg1mzoiP0fvvBY4/BT35S7ChFRKQxqU1SNJCoJRoKHAekO1Yv\nI9Y90zxFUieWLYPnn48EaPTomC9oyZI41qtX9As67LBoElt33eLGKiIijVONkyJ3HwZgZj2A993d\n6ywqEWJE2NNPR7+gp5+O+YO23jpGiB1zTMVkiR07FjtSERFpCmrTp6gDsLa7T03t2wa4EGgLPOHu\njxc+RGlO5s6NDtIPPhhriLVsGRMmXnQR/PjH8V5ERKQu1Kb57A5gLnABgJltCLxBNKPNAB42sxPd\nvazgUUqTNm8ePPlkDJt/8UVYvjyGyt9zDxx5JKynFfVERKQetKhF2f7Ak6nPPwO+Bnq5+0HAlcC5\n+QRhZnuZ2VNm9pmZlZvZYTnKXG1mn5vZt2b2gpltnXV8PTP7p5ktNLMFZnavmbXNKtPbzEaY2RIz\n+8jMLs5xn6PMbHJS5m0zG1jbWGTVFi2Cu++OTtGdOsHpp8PixXDDDTB7Nvz3v3DKKUqIRESk/tQm\nKeoMTE99HgA87u7Lks+PA9vmGUdbYAIxx9H3+iqZ2aXAOcQ8SLsAi4FhWdMCPAj0SOI6GNgbuDt1\njXWBYUStVj/gYmCwmZ2aKrN7cp17gD5EEviEmfWsZSxShRkz4KyzIhE6++yYLPGuu2LV+VdeiTmE\nNtqo2FGKiEhzVJvms2+AdqnPuwJ/TX1eSayFVmvu/jzwPIBZztWlzgeucfdnkjInArOBI4BHks7f\nBwEl7j4+KXMu8KyZXeTus4ATiDXbTnH3FcDkZCHbC4F7k/ucBzzn7jcln680swOIJCgzKWW1seTz\n/E2dOwwdCvffH81krVvDpZfCqafCZpsVOzoREZFQm5qi0cAvAMzsUKAD8FLq+DbAp4ULLZhZV6KW\nanhmn7svAkYRTXoAuwELMglR4kWi1mnXVJkRSUKUMQzobmbtk8/9k/PIKtM/iaVbDWKRlHffhT33\nhEMOgbfeghtvhDlzYPBgJUQiItKw1CYpugo42sy+Bp4Ahrj7l6njxwAjChlcojOR3MzO2j87OZYp\nMyd90N1XAvOzyuS6BjUokzneqQaxCDB+fPQJ6tMnkqCyMpg5M5rH2rZd5ekiIiL1rjbzFI01s+2J\nvjqz3P3VrCJPAe8UMrh6kqu5TvKweDEMGwa33BJzC228Mfzxj3DmmbBWXg2rIiIi9adWC8K6+xfA\nw1Ucq6s5imYRiUsnKtfQdALGp8pUmsLPzFoC6wNfpMp0yrp2puZn1irKpI+vKpacBg0aRPv27Svt\nKy0tpbS0tLrTGoVp06JZ7N57Yzh9//6xzMbhh8Ma+Sw5LCIizU5ZWRllZZVn9Vm4cGG9xtDg/2S5\n+wwzm0WMKnsHwMzaEX2F7kiKjQQ6mFnfVL+iAUQCMzpV5ndm1jJpWgM4EJjq7gtTZQYAt6ZCOCDZ\nX9NYchoyZAj9+vWr7eM3aNOnw2WXwb//HZ9POy0WX9023zGIIiLSbOWqKBg3bhwlJSX1FkODSIqS\n+YS2pqIpq5uZ7QjMd/dPgJuBK8zsQ2AmcA3RqftJAHefYmbDgHvM7CygNXAbUJaMPIMYan8l8Bcz\nuw7YgRhtdn4qlFuAV8zsQuBZoBQoAU5Llak2luZgxQr45z/hnHOif9ANN8DJJ0O7dqs+V0REpKFq\nEEkRsBPwMtGU5cCNyf6/Aie7+/Vm1oaYd6gD8BowMDVHEsQitbcTo8fKgcdIJTzuvsjMDiRqdMYA\n84DB7n5fqsxIMzsO+H2yfQAc7u7vpcrUJJYma9KkGEr/5ptwxBExAaPWHhMRkabAtK5r3TKzfsDY\nsWPHNurms7feggcegDvvjKH0Dz0Ee+xR7KhERKQpSzWflbj7uLq+X941RWa2BdG5udKwfncfnfsM\naYzmz4cLL4S//jVqhK68En75SzWViYhI01PrpMjMSoB/EpM1Zg9nd6BlAeKSIps7N0aTXXstLFsW\nQ+svuCBWrRcREWmK8qkpugeYQnRC/oIca5VJ4+Uecw2ddx58+GH0G/rTn2KtMhERkaYsn6SoO3C0\nu39Y6GCkuObMgSuugHvuge22iyU6evZc9XkiIiJNQW2W+cgYC3QpcBxSRO4xmmzvvaMz9RlnwHvv\nKSESEZHmJZ+aohuAG83sD8BEYHn6oLu/X4jApH5MnBjzDY0YAd26xZpl229f7KhERETqXz5JUWaS\nwgep3J/IUEfrRmPlSrjjDvjVr6BrV3jyyVjJvkU+dYciIiJNQD5JUY+CRyH16rPP4Cc/gVGjopbo\nD3+AddYpdlQiIiLFVeukyN2n1kUgUj9GjoQjj4yFWl9/HXbfvdgRiYiINAx5Td6YTNx4LhW1Ru8B\ntyXrlEkD9eSTcPTRsMsusYq9htmLiIhUqHUPEjPbD5hKrDD/cbIdCEw1s30LGp0UzJgxcOyxcPDB\nMHy4EiIREZFs+dQUXQ/8yd0vTO80s5uIkWk7FyIwKZwVK6KGqHNn+Mc/oHXrYkckIiLS8OQz1qgX\ncFeO/Xclx6QBmTcvRpXNnBmLuLZpU+yIREREGqZ8kqIvgVwz2fRKjkkDUV4OP/sZvPoqPP447Lpr\nsSMSERFpuPJpPvsLcE/S2fqNZN8ewBXAnYUKTFbfbbfFOmaPPBJrmImIiEjV8kmKrgK+BX4DrJ/s\n+5LoT3RDgeKSAhg9GvbYA37602JHIiIi0vDVuvnM3cvd/Q/uviHQCejk7hu5+3XuXl74ECUf5eUx\nyqyXenmJiIjUSF7zFGW4+9xCBSKFNX06zJ4Nhx9e7EhEREQahxolRWb2BvAjd//KzEZSec2zStxd\ncyQ3AEOGwFprqXO1iIhITdW0puhVYFnqfZVJkRTfihVQVhbrmq23XrGjERERaRxqlBS5+69S7y+r\nu3CkEEaOhAUL1MFaRESkNvJZ5uM9M1s/x/72ZvZeYcKSfC1eDL/+NWy0EeysucVFRERqLJ+O1ttV\ncd5awFarF46srssvh3Hj4P77oUU+U3OKiIg0UzVOiszswNTHfc3sq9TnlsD+xOKwUiRffQV33QXn\nnx9rnYmIiEjN1aam6Pnk1YGHso458ClwQSGCkvw89hgsXQrnnlvsSERERBqf2iRFawMGzAB2BtJz\nFK1w95WFDExqZ+RIOO88OPhg2GyzYkcjIiLS+NQ4KXL3pcnbjesoFlkNf/wjbLEFPPxwsSMRERFp\nnPKa0drM1iQWgd0CaJ0+5u5/LkBcUkszZsA++0DbtsWOREREpHHKZ0j+DsCHwDPAvcQisHcBNwOD\nCxlc6p4tzOwaM5tuZt+a2YdmdkWOcleb2edJmRfMbOus4+uZ2T/NbKGZLTCze82sbVaZ3mY2wsyW\nmNlHZnZxjvscZWaTkzJvm9nAwj91zc2dC2+/DTvuWMwoREREGrd8Bm0PAYYD7YElQF9gW2AccEbh\nQqvksuTaZxNTAlwCXGJm52QKmNmlwDnA6cAuwGJgmJmla7IeBHoAA4CDgb2Bu1PXWBcYRvSb6gdc\nDAw2s1NTZXZPrnMP0Ad4EnjCzHoW9pFr7qyzoFUrOOKIYkUgIiLS+OWTFJUA17n7cqAcWNPdPyQS\nlesKGVxKf+BJd3/e3T9298eB/xDJT8b5wDXu/oy7vwucCGwCHAFgZj2Ag4BT3H2Mu78BnAsca2ad\nk2ucALRKykx290eAW4ELU/c5D3jO3W9y96nufiWREJ5DEUyeDM88A1ddBZtsUowIREREmoZ8kqKV\nwPLk/Rxg8+T9PKBLAWLK5Q1ggJltA2BmOxJ9moYmn7sCnYkaLADcfREwikioAHYDFrj7+NR1XySm\nE9g1VWaEu69IlRkGdDez9snn/sl5ZJXpTxFceCF07Rq1RSIiIpK/fDpaTyBqiz4EXgOuSpqdTgIm\nFTC2tGuBdsAUM1tJJHOXu3tmvqTORHIzO+u82cmxTJk56YPuvtLM5meVmZ7jGpljC5PX6u5Tr6ZO\njYkaO3Qoxt1FRESajnxqin4DzE/eX07UGj1ILPFxZoHiynYMcBxwLNGH6STgYjP7WR3dL5vV031q\n5emnYeZM2GGHYkciIiLS+NW6psjdR6befwHsW8iAqnA98H/u/mjyeZKZdQF+BfwdmEUkLp2oXIvT\nCcg0l80COqYvamYtgfWBL1JlOmXduxNRCzVrFWVmUY1BgwbRvn37SvtKS0spLS2t7rQqvfACHHYY\n7L8/HHNMXpcQERFpMMrKyigrK6u0b+HChfUaQ17zFBVBGyIxSSsnqely9xlmNosYVfYOgJm1I/oK\n3ZGUHwl0MLO+qX5FA4hkanSqzO/MrGVqhu4DganuvjBVZgDRATvjgGR/lYYMGUK/fv1q+Lir9tvf\nwrbbwnPPwRqN5acoIiJShVwVBePGjaOkpKTeYqjRn1MzG8n3k5Kc3H331Yoot6eBy83sE6LfUj9g\nEDFPUsbNwBVm9iEwE7iGWI/tySSuKWY2DLjHzM4iJp28DShz90wtz4PAlcBfzOw6YAditNn5qfvc\nArxiZhcCzwKlRB+r0wr90FV59FF4/XX45z+VEImIiBRKTf+kvlKXQdTAOUSScwfRBPY58KdkHwDu\nfr2ZtSHmHepAdAIf6O7LUtc5DridGD1WDjxGKuFx90VmdmBynzHEiLrB7n5fqsxIMzsO+H2yfQAc\n7u7vFfqhc/noIzj5ZNh7b8iz5U1ERERyMPcaVQBJnsysHzB27Nixq9189tlnsNdesGQJjB4Nm2++\n6nNEREQaq1TzWYm7j6vr+6nxpRF58slY42zKFCVEIiIihVbrpMjMllBN/yJ3b7NaEUmVvvwSOnWC\n7t2LHYmIiEjTk09N0c+zPrci5g4qBa5e3YCkavPmwYYbFjsKERGRpimfeYoezrH7H2b2NnA4cNdq\nRyXfU14OL78cw/BFRESk8PKZ0boqrxMLrkodeOEFmDgRTqu3gf8iIiLNS0GSIjNrBZxBxczQUmB/\n/COsvz7ss0+xIxEREWma8ulo/QWVO1obsVTGCuDEAsUlKRMnRtPZTTdBG3VjFxERqRP5dLQenPW5\nHJgLvOHuc75fXFbXQw9Bu3ZwyinFjkRERKTpyqej9d11EYhU7bPPYLvtoG3bYkciIiLSdOU1eWPS\nh6gHseRGpX5J7v6fAsQlKVOnQteuxY5CRESkacunT9F+wN+BjXMcdqDl6gYlFb7+Gt56C046qdiR\niIiING35jD77EzAU6Aq0AdZObeoGXGCPPw4rV8KAAcWOREREpGnLp/lsY+Bad/+o0MFIZWPHwnnn\nRUK09dbFjkZERKRpy6em6Elgz0IHIt93443QuTP8619gVuxoREREmrZ8aorOAh4ys/7ARGB5+qC7\n/7kQgQmMGAHHHQft2xc7EhERkaYvn6Tox8AByfsFVJ7I0QElRQUwdWoMxd9tt2JHIiIi0jzkkxRd\nB1wLXOPuKwocjyROOinmJdprr2JHIiIi0jzk06eoDfA3JUR159tvYdQouOgi2GijYkcjIiLSPOST\nFP0dOKLQgUiFd96JVy3+KiIiUn/yaT5bClxhZgcC7/D9jta/LkRgzdmf/wybbgo771zsSERERJqP\nfJKiPYApQDu+PzTfv19casMdRo+GgQNhnXWKHY2IiEjzkc+CsP3rIhAJzz4LkybBlVcWOxIREZHm\nJZ8+RVKHXn45ms6OPrrYkYiIiDQvNaopMrMHgTPc/evkfZXc/biCRNYMucPDD8OhhxY7EhERkean\nps1nVsV7KaBp02LCRiVFIiIi9a9GSZG7l+Z6L4VVVgYtWsCeWllORESk3q12nyIz29XM9jMzjZVa\nDcOHR+fqk0+Gdu2KHY2IiEjzU+OkyMwuNLOrsvY9CYwEXgSmmFn3AsfXbNx9N2y/fcxRJCIiIvWv\nNjVFxwPvZz6Y2ZHAgcTisJsBk4Crcp+6+sxsEzP7u5nNM7NvzextM+uXVeZqM/s8Of6CmW2ddXw9\nM/unmS00swVmdq+Ztc0q09vMRpjZEjP7yMwuzhHLUWY2OSnztpkNXJ1nmzYNHn0UTjkFTD22RERE\niqI2SVE34O3U54OBf7n7cHf/HPgt35/MsSDMrAPwOjGb9kFAD+CXwIJUmUuBc4DTgV2AxcAwM2ud\nutSDybkDkvj3Bu5OXWNdYBgwA+gHXAwMNrNTU2V2T65zD9AHeBJ4wsx65vt8zz8fr6eeWn05ERER\nqTu1mbyxFbAk9bk/cHvq86dAXS1fehnwsbun04aPssqcD1zj7s8AmNmJwGxinbZHzKwHkVCVuPv4\npMy5wLNmdpG7zwJOIJ7zlGTB28lm1he4ELg3uc95wHPuflPy+UozO4BIyM7O5+EWLIiFX9ddN5+z\nRUREpBBqU1M0jVjiAzPbDNgWGJE6vikwv3ChVXIoMMbMHjGz2WY2Lqv2pivQGRie2efui4BRRPIG\nsBuwIJMQJV4klibZNVVmRJIQZQwDuptZ++Rz/+Q8ssrkPdP3nDmw3nr5ni0iIiKFUJuk6G7gdjO7\nA3gGGOvu76aO7wtMKGBsad2As4CpRD+mPwG3mtnPkuOdieRmdtZ5s5NjmTJz0gfdfSWRyKXL5LoG\nNSjTmTysXBlD8TvndbaIiIgUSo2bz9z9TotewIcC44HfZBXpBjxQsMgqawGMdvfMPd82s17AmcDf\n6+ieaavd/XnQoEG0b9++0r7S0lJ2262UefPgggtW9w4iIiKNV1lZGWVlZZX2LVy4sF5jqNWCsO5+\nJ3BnFcdOK0hEuX0BTM7aNxk4Mnk/i0hcOlG5FqcTkcBlynRMX8DMWgLrJ9fPlOmUdZ9ORC3UrFWU\nmUU1hgwZQr9+/b63/9VX47VHj+rOFhERadpKS0spLa08P/S4ceMoKSmptxgay4KwrwPZcyB1J+ls\n7e4ziKRkQOagmbUj+gq9kewaCXRIOk5nDCCSqdGpMnsnyVLGgcBUd1+YKjOAyg5I9tfaCy9A+/bQ\npUs+Z4uIiEihNJakaAiwm5n9ysy2MrPjgFOpPPrtZuAKMzvUzHYA/kaMiHsSwN2nEB2i7zGznc1s\nD+A2oCwZeQYx1H4Z8Bcz62lmxxCjzW5M3ecW4IfJZJbdzWwwUJIVS409/zz84Aew1lr5nC0iIiKF\n0iiSIncfA/wYKAUmApcD57v7Q6ky1xNJzt3EqLO1gYHuvix1qeOAKcTosWeI0XNnpK6xiKgZ6gKM\nAW4ABrv7fakyI5PrnE50LD8SONzd36vtcy1ZAmPHwkEH1fZMERERKbRa9SkqJncfCgxdRZnBwOBq\njn9FzEVU3TXeBfZZRZl/Af+qrkxNLFoUr5tssrpXEhERkdXVKGqKmqpMp/qsQWkiIiJSBDWqKTKz\nB2t6QXc/Lv9wmhclRSIiIg1HTZvPtExpHfj883jt2LH6ciIiIlL3apQUuXvpqktJbT3/PHToABtv\nXOxIRERERH2KimT2bLjvPjj5ZDDVw4mIiBRdXqPPzOwQ4GhgC6B1+pi7716AuJq8oUNh+XI4//xi\nRyIiIiKQR02RmZ0FPAwsJVaGnwqsBHoSM09LDdx1Fxx4IGyxRbEjEREREciv+ew84MxkrbNlwDXu\nvhdwF9CqkME1ZVOmwAEHFDsKERERycgnKdqSmAka4Dtg3eT9fcDxhQiqqfv225i4UR2sRUREGo58\nkqI5wHrJ+4+BnZL3m9OIZsgupi++iNfOnYsbh4iIiFTIJyl6GTgkef934BYzexp4BHi6UIE1ZbOS\n5WdVUyQiItJw5FOzc0bmPHe/2cy+AnYHriMWZJVVmD07XjVpo4iISMORT1LUwd3nZD64+wPAAwBm\n1pHoZyTV+PbbeF1nneLGISIiIhXyaT77Ikl+KjGzDYAvVj+kpm/p0nht3br6ciIiIlJ/8kmKqpp/\nuQ2qJaqR776DVq2gheYTFxERaTBq3HxmZv+XvHXgcjNbnDrckpjIcWIBY2uyli6FtdYqdhQiIiKS\nVps+RT9IXg3YA1ieOrYMmAFcW6C4mrTvvoM11yx2FCIiIpJW46TI3fsDmFkZcIa7L6qzqJo41RSJ\niIg0PLUefebupZn3ZrZhsm9eIYNq6pYuVU2RiIhIQ5PPgrBmZpeY2RxgNjDbzOaY2cVmVlUnbElZ\nskRJkYiISEOTzzxFvwV+AfwOeD3ZtydwOdAWGFyQyJqw+fNhgw2KHYWIiIik5ZMUnQKc6u7/Tu0b\nbWYfAbegpGiVZs+GTp2KHYWIiIik5TNTzgbApBz7JybHZBU+/1yLwYqIiDQ0+SRF7wKn59h/RnJM\nqlFeDtOmwdZbFzsSERERScun+ewy4GkzGwC8kezbHegOHFKowJqqzz+PeYqUFImIiDQsta4pcvcX\nge2A4UCXZBsO9HD3lwoZXFM0eXK8KikSERFpWGqzzMeVwB/d/Vt3/wi4qO7Carreegs6dIBttil2\nJCIiIpJWm5qiq4B16iqQ2jCzy8ys3MxuSu1b08zuMLN5Zva1mT1mZh2zztvczJ41s8VmNsvMrjez\nFlll9jWzsWb2nZm9b2Yn5bj/L8xshpktMbM3zWznmsa+eDG0b6/FYEVERBqa2vxpbhATMyYJyOnA\n21mHbgYOBn4C7A1sAvwrdV4LYChRO7YbcBLwc+DqVJkuwDNEc+COxBQD95rZAakyxwA3Ekli3ySO\nYZnZvVdl+XJo3bqGDysiIiL1prb1FV4nUdSQma0D/AM4Ffgqtb8dcDIwyN1fdffxwP8D9jCzXZJi\nBxF9oY5394nuPgz4DfALM8s0I54FTHf3S9x9qrvfATwGDEqFMQi4293/5u5TgDOBb5P7r9KyZUqK\nREREGqLaJkXvm9n86rY6ibLCHcDTOTp070TUAA3P7HD3qcDHQP9k127AxKx12oYB7YHtU2VezLr2\nsMw1zKwVUJJ1H0/O6U8NLF8OrVrVpKSIiIjUp9oOyb8KWFgXgayKmR0L9CESoGydgGXuvihr/2wg\nM01i5+Rz9vHMsberKdPOzNYE1gdaVlGme02eQzVFIiIiDVNtk6KH3H1OnURSDTPbjOgztL+7L6/v\n+xeSaopEREQaptokRcXsT1QCbASMM7NMh++WwN5mdg7wQ2BNM2uXVVvUCZiVvJ8FZI8Sy6xA9kWq\nTPaqZJ2ARe6+1MzmASurKDOLagwaNIj27dszblxM3njYYVBaWkppaWl1p4mIiDQLZWVllJWVVdq3\ncGH9Nk5ZdImpQUGzcqBzkWqK2gJbZu1+AJgMXAt8BswFjs0sVGtm3ZPju7r7W2b2Q+BpYONMvyIz\nOx24Dujo7svN7FpgoLvvmLr3g0AHd/9R8vlNYJS7n598NqLv0q3ufkOO2PsBY8eOHUu/fv046ij4\n+mt4/vnCfDf/v737D7ajrO84/v4IGAw2SRUkOII4tYKKUk1RtKgoCgqKdaBqaiuK1mJRp0wrjp1p\noTaMo/MAABEcSURBVNix/hh/IQad+gNRjEPxR6lGQWgrotBoUilKBKsgKgYIMElIIAHy7R+7VzfH\ne29uQu6ecy/v18zOObv77Nnv3mfO7vc++zxnJUmarVauXMmiRYsAFlXVyune35RbiqpqaL+sU1Ub\ngGu6y5JsAG6rqlXt/CeA9ye5A1gPnAl8u6q+225ycfsZn0nyNmAf4B3AWZ1bch+lGY32buCTwBHA\n8cDRnV2/HzgnyQpgOc1otLk0Sdo2bd7s7TNJkkbRjjz7bFQMNnGdQnNr6wJgDvB14ORfF67akuTF\nwNk0z2zbQJPInNYpc0OSY4APAG8BfgG8rn20yViZ89vfJDqD5rbZ94GjqurWqQS9eTPMnbt9BypJ\nkqbfjE2Kqup5A/ObgDe300Tb/JxtPLS2qi6j6cM0WZklwJIpB9uxcSM8/OE7sqUkSZpOPmyiZ7fd\nZlIkSdIoMinq2e23mxRJkjSKTIp6tG4d3HIL7LPPsCORJEmDTIp6dN11cN99sGjSHkuSJGkYTIp6\ntHFj8/rQhw43DkmS9NtMino0lhQ5JF+SpNFjUtSju+5qXk2KJEkaPSZFPbKlSJKk0WVS1KONGyGB\nOXOGHYkkSRpkUtSjn/+8+Y2iZNiRSJKkQSZFPbr0Ujj88GFHIUmSxmNS1KOrr4ZDDhl2FJIkaTwm\nRT1Zv76Z9ttv2JFIkqTxmBT1ZO3a5nWvvYYbhyRJGp9JUU/uuad53X334cYhSZLGZ1LUk02bmleT\nIkmSRpNJUU/GWor8jSJJkkaTSVFPbCmSJGm0mRT1ZPPm5tWWIkmSRpNJUU/saC1J0mgzKerJ2O0z\nW4okSRpNJkU9uf56mD8f9thj2JFIkqTxmBT15Kab4KCDYJddhh2JJEkaj0lRT9avb1qKJEnSaDIp\n6smdd8KCBcOOQpIkTcSkqCd33gnz5g07CkmSNBGTop7ce68jzyRJGmUmRT3ZsgUe5F9bkqSRNSMu\n00nenmR5knVJbk7ypSSPGygzJ8lHkqxJsj7JBUkeMVBm3yRfTbIhyeok70nyoIEyhydZkeTuJNcl\nOWGceE5Ocn2Su5JcmeSQbR1DFSQ7+heQJEnTbUYkRcCzgA8DTweeD+wGXJzkIZ0yHwSOAY4Dng08\nEvjC2Mo2+VkG7AocCpwAvAY4o1Nmf+ArwKXAwcCHgI8neUGnzCuA9wGnAU8BrgIuSrLnZAdgS5Ek\nSaNt12EHMBVVdXR3PslrgFuARcDlSeYBJwKvrKpvtmVeC6xK8rSqWg4cBRwIPLeq1gBXJ/l74F1J\nTq+qe4E3Aj+tqlPbXV2b5DDgFOAb7bJTgI9V1bntfk6iScZOBN4z0TGYFEmSNNpm6mV6AVDA7e38\nIpoE79KxAlV1LXAj8Ix20aHA1W1CNOYiYD7wxE6ZSwb2ddHYZyTZrd1Xdz/VbvMMtsGkSJKk0TXj\nLtNJQnOr7PKquqZdvBDYXFXrBorf3K4bK3PzOOuZQpl5SeYAewK7TFBmIZOwpUiSpNE2I26fDVgC\nPAE4bNiBbA+TIkmSRtuMSoqSnAUcDTyrqm7qrFoNPDjJvIHWor3bdWNlBkeJ7d2+/qpTZu9xyqyr\nqk1J1gD3TVBmNZO4/fZTOP/8+Vx11W+WLV68mMWLF0+2mSRJDwhLly5l6dKlWy1bu3ZtrzHMmKSo\nTYheCjynqm4cWL0CuBc4AvhSW/4AYD/gO22ZK4C/S7Jnp1/RkcBaYFWnzIsGPvvIdjlVdU+SFe1+\nLmz3k3b+zMninz//Ayxe/FROP32qRyxJ0gPHeA0FK1euZNGiRb3FMCOSoiRLgMXAscCGJGMtNWur\n6u6qWpfkE8D7k9wBrKdJUr5dVd9ty14MXAN8JsnbgH2AdwBnVdU9bZmPAicneTfwSZpk53ia1qkx\n7wfOaZOj5TSj0eYC50x2DFXePpMkaZTNiKQIOIlmtNl/DSx/LXBu+/4UmltbFwBzgK8DJ48VrKot\nSV4MnE3TerSBJpE5rVPmhiTHAB8A3gL8AnhdVV3SKXN++5tEZ9DcNvs+cFRV3TrZAdinSJKk0TYj\nkqKq2mY6UVWbgDe300Rlfg68eBufcxnNsPvJyiyh6fA9ZSZFkiSNNi/TPfH2mSRJo83LdE9MiiRJ\nGm1epnvi7TNJkkabl+meVEEy7CgkSdJETIp6YkuRJEmjzct0T+xTJEnSaPMy3RNbiiRJGm1epnti\nUiRJ0mjzMt0Tb59JkjTavEz3xJYiSZJGm5fpnjgkX5Kk0WZS1CNbiiRJGl1epntkUiRJ0ujyMt0j\nkyJJkkaXl+kemRRJkjS6vEz3yKRIkqTR5WW6RyZFkiSNLi/TPXJIviRJo8ukqEe77jrsCCRJ0kRM\ninr0xCcOOwJJkjQRk6IePfrRw45AkiRNxKSoJ7vtBnPnDjsKSZI0EZOinsyfb0drSZJGmUlRTxYs\nGHYEkiRpMiZFPTEpkiRptJkU9WT+/GFHIEmSJmNS1JO99hp2BJIkaTImRT1ZuHDYEUiSpMmYFO2g\nJCcnuT7JXUmuTHLIZOX32aevyNSHpUuXDjsE7UTW5+xifWpHmRTtgCSvAN4HnAY8BbgKuCjJnhNt\nY0vR7OJJd3axPmcX61M7yqRox5wCfKyqzq2qHwEnARuBEyfa4MAD+wpNkiTtCJOi7ZRkN2ARcOnY\nsqoq4BLgGRNt58NgJUkabSZF229PYBfg5oHlNwPeJJMkaYay/WL67Q6watWqYcehnWjt2rWsXLly\n2GFoJ7E+Zxfrc/boXDt372N/ae78aKra22cbgeOq6sLO8nOA+VX1soHyfwqc12uQkiTNLq+qqs9N\n905sKdpOVXVPkhXAEcCFAEnSzp85ziYXAa8CbgDu7ilMSZJmg92B/WmupdPOlqIdkOTlwDk0o86W\n04xGOx44sKpuHWJokiRpB9lStAOq6vz2N4nOAPYGvg8cZUIkSdLMZUuRJEkSDsmXJEkCTIokSZIA\nk6Jpt70PjlX/kpyWZMvAdE1n/ZwkH0myJsn6JBckecTAZ+yb5KtJNiRZneQ9Sfx+9STJs5JcmOSX\nbf0dO06ZM5LclGRjkm8keezA+t9Ncl6StUnuSPLxJHsMlHlyksva7/PPkrx1uo/tgWhb9ZnkU+N8\nZ5cNlLE+R0SStydZnmRdkpuTfCnJ4wbK7JTzbJLDk6xIcneS65KcsD2xetKeRjvy4FgNzQ9oOs0v\nbKfDOus+CBwDHAc8G3gk8IWxle2XchnNwIVDgROA19B0xFc/9qAZ8PBXwG91lEzyNuBNwBuApwEb\naL6LD+4U+xzweJqf1ziGpq4/1vmM36EZFnw98FTgrcDpSV4/DcfzQDdpfba+xtbf2cUD663P0fEs\n4MPA04HnA7sBFyd5SKfM/T7PJtkf+ArNY7gOBj4EfDzJC6YcaVU5TdMEXAl8qDMf4BfAqcOOzWmr\nejoNWDnBunnAJuBlnWUHAFuAp7XzLwLuAfbslPlL4A5g12Ef3wNtauvm2IFlNwGnDNTrXcDL2/nH\nt9s9pVPmKOBeYGE7/0ZgTbdOgX8Grhn2Mc/maYL6/BTwxUm2OdD6HN2J5nFZW4DD2vmdcp4F3g38\n78C+lgLLphqbLUXTZEcfHKuh+f22qf4nST6bZN92+SKa/0y69XgtcCO/qcdDgaurak3n8y4C5gNP\nnP7QNZkkj6FpSejW4Trgv9m6Du+oqv/pbHoJTSvF0ztlLquqeztlLgIOSDJ/msLXxA5vb8X8KMmS\nJA/rrHsG1ucoW0BTF7e38zvrPHsoTT0zUGbK11yTounjg2NnjitpmmGPovlBzscAl7X9DxYCm9uL\naFe3Hhcyfj2DdT0KFtKcgCf7Li4EbumurKr7aE7a1vPo+RrwauB5wKnAc4Bl7dMFwPocWW0dfRC4\nvKrG+m7urPPsRGXmJZkzlfj88UY94FVV9+fjf5BkOfAz4OX4aBZp5FTV+Z3ZHya5GvgJcDjwn0MJ\nSlO1BHgCW/fbHBm2FE2fNcB9NB0Bu/YGVvcfjqaqqtYC1wGPpamrByeZN1CsW4+rGb+ewboeBatp\n+vNN9l1cDQyOdNkFeBjwq04Z63kEVdX1NOfcsRGF1ucISnIWcDRweFXd1Fl1f8+z26rTdVW1aSox\nmhRNk6q6Bxh7cCyw1YNjvzOsuLRtSR4K/B5N59wVNJ0zu/V4ALAfv6nHK4AnDYwqPBJYC1yDhqq9\nYK5m6zqcR9O3pFuHC5I8pbPpETTJ1PJOmWe3F9cxRwLXtom0hiTJo4CH85uLo/U5YtqE6KXAc6vq\nxoHV9/c8u6pT5gi2dmS7fGqG3Qt9Nk80t1820tz7PpBmOOhtwF7Djs1pq3p6L80Q0EcDzwS+QXMf\n+uHt+iU0w3YPp+kQ+G3gW53tH0TzcwtfA55M0zfpZuAdwz62B8pEM4T7YOAPaEas/HU7v2+7/tT2\nu/cS4EnAl4EfAw/ufMYy4HvAIcAfAdcCn+msn0eTKH+apvn/FcCdwOuGffyzbZqsPtt176FJah9N\ncxH8Hs2FcTfrc/Sm9hx6B83Q/L070+4DZe7XeRbYH1hPMwrtAJqfdNgMPH/KsQ77jzXbp7ZSbqAZ\n/nsF8IfDjsnpt+poKc1PJdxFM9rhc8BjOuvn0PzGxpr2C/evwCMGPmNfmt/HuLP9or4beNCwj+2B\nMtF0tN1Cc8u6O32yU+b09iK4kWZEymMHPmMB8Fma/zzvAP4FmDtQ5iDgm+1n3Aj87bCPfTZOk9Un\nsDvwdZrWv7uBnwJnM/DPpvU5OtMEdXkf8OpOmZ1ynqX5B3dFez7/MfDn2xOrD4SVJEnCPkWSJEmA\nSZEkSRJgUiRJkgSYFEmSJAEmRZIkSYBJkSRJEmBSJEmSBJgUSZIkASZFkjQlSbYkOXbYcUiaPiZF\nkkZekk+1Scl97evY+2XDjk3S7LHrsAOQpCn6GvAamiedj9k0nFAkzUa2FEmaKTZV1a1VdUtnWgu/\nvrV1UpJlSTYm+UmS47obJzkoyaXt+jVJPpZkj4EyJyb5QZK7k/wyyZkDMeyV5ItJNiS5LslLOtsu\nSHJeklvafVyb5IRp+2tI2ulMiiTNFmfQPFn7ycB5wOeTHACQZC5wEXAbsAg4Hng+zVO5acu8ETgL\n+CjN09OPBf5vYB//AHweeBKwDDgvyYJ23T8BBwJHta9vpHnit6QZIlU17BgkaVJJPgX8GXB3Z3EB\n76yqdyXZAiypqjd1trkCWFFVb0ryF8A/A4+qqrvb9S8C/h3Yp6puTfIL4BNVddoEMWwBzqiq09v5\nucCdwAur6uIk/wbcWlWv37lHL6kv9imSNFP8B3ASW/cpur3z/sqB8lcAB7fvDwSuGkuIWt+maS0/\nIAnAI9t9TObqsTdVtTHJOuAR7aKzgS8kWQRcDHy5qq7Y1kFJGh0mRZJmig1Vdf00ffZdUyx3z8B8\n0XZDqKqvJ9kPOBp4AXBJko9U1ak7L0xJ08k+RZJmi0PHmV/Vvl8FHJzkIZ31hwH3AT+qqjuBG4Aj\n7k8AVXVbVX2mql4NnAK84f58nqR+2VIkaaaYk2TvgWX3VtVt7fs/SbICuJym/9EhwIntuvOA04FP\nJ/lHmlteZwLnVtVYZ+jTgbOT3Eoz/H8e8MyqOmsqwbWfuwL4IbA78GLgmu09SEnDY1IkaaZ4IXDT\nwLJrgSe0708DXgl8BPgV8Mqq+hFAVd2V5CjgQ8ByYCNwAfA3Yx9UVecmmUPTwvNempFjF3T2Nd6o\nlOos3wy8E9if5nbct4DFO3CckobE0WeSZrx2ZNgfV9WFw45F0sxlnyJJkiRMiiTNDjZ5S7rfvH0m\nSZKELUWSJEmASZEkSRJgUiRJkgSYFEmSJAEmRZIkSYBJkSRJEmBSJEmSBJgUSZIkASZFkiRJAPw/\nuZ6mdWW6CAIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdb9a23d208>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(epochs, counts)\n",
    "plt.title('Simulation Steps vs. Episodes')\n",
    "plt.xlabel('Epochs')\n",
    "plt.ylabel('Total Simulation Steps')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Heat map of position exploration:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANQAAAFfCAYAAAA23uK3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAEvBJREFUeJzt3X/sXXV9x/Hn25a2Uq0EkFZnnUCBzCwyh4I4i2y4oZjg\nFgy/TAywxSBoHMmiM2HDwXRTA6s/1kWjAwlCgiKDbbRVUXGotQnIJiAysfyQ0q/8cC0BSi3fz/44\nt/HL/f64n/v9vu/3nts+H8lJuOe+7zlv+j2v+zn3nHPviVIKknK8YNgNSHsSAyUlMlBSIgMlJTJQ\nUiIDJSUyUFIiAyUlMlBSooXDbiAiDgBOBO4Hdgy3G2lKS4BXARtKKY/PWFlKGcgEnA9sBp4BNgKv\nn6buTKA4OY3AdGav7X4gI1REnAZcCrwH2ARcAGyIiMNLKY91ld8PsIjn73/u7MwbRfY+HIPqfbyz\nbDrb6kwGtct3AfC5UsqVABFxLvB24BzgE121O6AJ08RABaP7Ac/eh2Meeu/5kSR9/RGxD3AUcPPu\neaXZt/smcGz2+qQ2GUSgDwQWAGNd88eAFQNYn9QaQz/Kt9tOmiF7t+eAXbSoQe0VdtFsexOVPl4/\niO31MZqelnfNXw5sne5F3QclRjlMC4bdwBzs7b0vZPJ2N079+Zz0Xb5Syq+B24ATds+LiOg8/n7t\nckY1TGDvw9KG3gfVw2XAFRFxG785bL4vcMWA1jcQNe94me9ItYd8F1fUrKyoqf1Au6WiZv/KZd10\nSkXR0rplHXZl75r/fVvdspauq6vrZSCBKqVcGxEHAhfT7OrdAZxYSnl0EOuT2mJgo2QpZS2wdlDL\nl9poVM/hSa1koKREBkpKZKCkRAZKSmSgpEQGSkrUhqs1WmuU3232q6h5onJZG/+md82hl9Qt66Dr\netd0X5w6nX0rapYlXQFRa5S3Gal1DJSUyEBJiQyUlMhASYkMlJTIQEmJDJSUyBO7MxivqBnGO1LN\nV/N/XVFTewL1VRUnbZdULqtmnbV9PVlZN58coaREBkpKZKCkRAZKSmSgpEQGSkpkoKREBkpKZKCk\nRF4pMU/m+8YDNV+B/8q1dctaeWrvmv+rW1T1VRBZ5nt9jlBSIgMlJTJQUiIDJSUyUFIiAyUlMlBS\nIgMlJfLE7h5qZ0XNH1ecsIW6nwKoqamVuaz5lj5CRcRFETHeNd2dvR6pjQY1Qt0JnABE5/GuAa1H\napVBBWpXKeXRAS1baq1BHZQ4LCIejoj7IuKqiFg5oPVIrTKIQG0EzgJOBM4FDga+GxFLB7AuqVXS\nd/lKKRsmPLwzIjYBDwCnApdP97qd/OYD124LBtGgNINdTP7KR+nj9QPfXksp2yLiXmDVTHWL8KSY\nhm8hk0MxDuyofP3At+GIeBFwKPDIoNclDdsgzkN9MiKOi4jfjog3AtfTjKTXZK9LaptB7PK9Arga\nOAB4FLgVeEMp5fEBrGug2roLuk9FzYsrap6uXF/N7k7NzQn2BoM4KHFG9jKlUdHWN2FpJBkoKZGB\nkhIZKCmRgZISGSgpkYGSEnnt6Qzaehf4GleVRT1rDo2aL8rXX8em9m4P0kgyUFIiAyUlMlBSIgMl\nJTJQUiIDJSUyUFIiT+zOYL7fbWpubF1b91cVJ223Vq5P9RyhpEQGSkpkoKREBkpKZKCkRAZKSmSg\npEQGSkpkoKREXikxgmqulFgy8C40FUcoKZGBkhIZKCmRgZISGSgpkYGSEhkoKZGBkhJ5Ynee1JyM\nrX1327+i5u8r7iL9TzV3v1Zf+h6hImJ1RNwYEQ9HxHhEnDxFzcURsSUino6Ib0TEqpx2pXabzS7f\nUuAO4DygdD8ZER8C3ge8BzgaeArYEBG9bwchjbi+d/lKKeuB9QAREVOUfAC4pJTyH52adwNjwJ8C\n186+Van9Ug9KRMTBwArg5t3zSinbgR8Cx2auS2qj7KN8K2h2A8e65o91npP2aK05yrcT6N5/XECL\nGtReYRfwXNe8SQcKZpC9vW6lycVynj9KLQd+NNMLF+FJMQ3fQiaHYpz626KmbsOllM00oTph97yI\nWAYcA3w/c11SG/U9QkXEUmAVv9lDOyQijgSeKKU8BKwBLoyInwH3A5cAvwBuSOlYarHZ7PK9Dvg2\nza5lAS7tzP8ScE4p5RMRsS/wOWA/4L+At5VS6m45voeq2RWovVlAxUUQrPcqiKGYzXmoW+ixfZRS\nPgJ8ZHYtSaPL4wBSIgMlJTJQUiIDJSUyUFIiAyUlMlBSIq89HUHdF29O5e6Bd6GpOEJJiQyUlMhA\nSYkMlJTIQEmJDJSUyEBJiQyUlMhASYm8UqJFar8CX1O3ci6NaNYcoaREBkpKZKCkRAZKSmSgpEQG\nSkpkoKREBkpK5IndOao9GVvzU+O1P0e+cdJtwqewrnfJWTU/kq6+OEJJiQyUlMhASYkMlJTIQEmJ\nDJSUyEBJiQyUlMgTuy1Se5L4nBt715wxp040W32PUBGxOiJujIiHI2I8Ik7uev7yzvyJ0015LUvt\nNZtdvqXAHcB5QJmmZh2wHFjRmXzD1F6h712+Usp6YD1ARMQ0Zc+WUh6dS2PSKBrUQYnjI2IsIu6J\niLURsf+A1iO1yiAOSqwDrgM2A4cC/wDcFBHHllKm20WU9gjpgSqlXDvh4V0R8WPgPuB44NvTvW4n\n0L3/uGAQDUoz2MXkO0T2MwoM/DxUKWUz8Biwaqa6RcDirskwab4tZPJ2uKiP1w88UBHxCuAA4JFB\nr0satr4HgYhYSjPa7N5DOyQijgSe6EwX0XyG2tqp+zhwL7Aho2GpzWazV/U6ms9CpTNd2pn/JZpz\nU68B3g3sB2yhCdLfllJG7gvX4xU1/ewOZNlaUfPkwLvQVGZzHuoWZt5VfOvs25FGmxfHSokMlJTI\nQEmJDJSUyEBJiQyUlMhASYm8XG4Gme82NV9vr13fkoqal1YuS7kcoaREBkpKZKCkRAZKSmSgpEQG\nSkpkoKREBkpKZKCkRF4pMUe1d27PvLrha39dUXRDRc1PKleoao5QUiIDJSUyUFIiAyUlMlBSIgMl\nJTJQUiIDJSXyxG6L1P74+4X/2LvmtDl1otlyhJISGSgpkYGSEhkoKZGBkhIZKCmRgZISGSgpkYGS\nEvUVqIj4cERsiojtETEWEddHxOFdNYsj4p8j4rGIeDIivhoRB+W2vXd7omJ6rmJSvn5HqNXAZ4Bj\ngLfQ/KTC1yPihRNq1gBvB04BjgNeDlw391al9uvrWr5SykkTH0fEWcAvgaOAWyNiGXAOcHop5ZZO\nzdnATyLi6FLKppSupZaa62eo/YBCs5cBTbAWAjfvLiil/BR4EDh2juuSWm/WgYqIoNm9u7WUcndn\n9gpgZylle1f5WOc5aY82l69vrAVeDbwpo5GdQHTNW4DfL9H82sXkAzalj9fPanuNiM8CJwGrSylb\nJjy1FVgUEcu6RqnlneemtQiP4Wv4FjI5FOPAjsrX970Nd8L0DuAPSykPdj19G03IT5hQfwTwSuAH\n/a5LGjV9jVARsRY4AzgZeCoilnee2lZK2VFK2R4RXwQui4hfAU8Cnwa+5xE+7Q363eU7l2aX8jtd\n888Gruz89wU0u6FfBRYD64HzZ9/i8Mz3ndv3rVzW8t4l7F+5LOXq9zxUz+2nlPIs8P7OJO1VPA4g\nJTJQUiIDJSUyUFIiAyUlMlBSIgMlJfLa0xnUnNitOWELzfdcevlw5bKO/8uKop9X1NxYuUJVc4SS\nEhkoKZGBkhIZKCmRgZISGSgpkYGSEhkoKZGBkhJ5pUSLXNm7BIAH1vSuOXlOnWi2HKGkRAZKSmSg\npEQGSkpkoKREBkpKZKCkRAZKSuSJ3RnUvNssqlxWze+Wv6RyWS+tqFlauSzlcoSSEhkoKZGBkhIZ\nKCmRgZISGSgpkYGSEhkoKZGBkhL1FaiI+HBEbIqI7RExFhHXR8ThXTXfiYjxCdNzEbE2t+322Kdy\nWlAxLUmcNBz9jlCrgc8AxwBvodlWvh4RL5xQU4DPA8uBFcDLgA/OvVWp/fq6lq+UctLExxFxFvBL\n4Cjg1glPPV1KeXTO3UkjZq6fofajGZGe6Jr/roh4NCJ+HBEf6xrBpD3WrK82j4gA1gC3llLunvDU\nl4EHgC3Aa4BPAIcD75xDn9JImMvXN9YCrwb+YOLMUsoXJjy8KyK2At+MiINLKZunW9hOILrmLZhj\ng1K/dgHPdc0rfbx+VttrRHwWOAlYXUp5pEf5D2mysgqYNlCL8Bi+hm8hk0MxDuzo4/V96YTpHcCb\nSykPVrzktTQh7xU8aeT1FajO+aQzaH7p96mIWN55alspZUdEHAKcCdwEPA4cCVwG3FJKuTOvbamd\n+h2hzqUZbb7TNf9smp/m3klzfuoDNN/Cfgj4CvDROXU5JDV3bj+iclmfqqh5uHJZR41VFF1TUVNz\nN3n1pd/zUDN+zCml/AI4fi4NSaPM4wBSIgMlJTJQUiIDJSUyUFIiAyUlMlBSIq89nUH3RZJTqb3G\n67MVNcdULmvl8t41By2uXJhSOUJJiQyUlMhASYkMlJTIQEmJDJSUyEBJiQyUlMhASYm8UmIGNXdS\nX1G5rJUVNYdVLuugmlvK719R84vKFaqaI5SUyEBJiQyUlMhASYkMlJTIQEmJDJSUyEBJiTyxO4M3\nVtT8eeWyjqr5HfGa3ywHuPoLvWtu/oveNW+pXJ+qOUJJiQyUlMhASYkMlJTIQEmJDJSUyEBJiQyU\nlMhASYn6ulIiIs4F3gu8qjPrLuDiUsr6zvOLgcuA04DFwAbgvFLKL7Mank/bK2p+VrmsBWt61/xe\n7Q/8r6y4CuLpymUpVb8j1EPAh4DfB44CvgXcEBG/03l+DfB24BTgOODlwHU5rUrt19cIVUr5z65Z\nF0bEe4E3RMTDwDnA6aWUWwAi4mzgJxFxdCllU0rHUovN+jNURLwgIk4H9gV+QDNiLQRu3l1TSvkp\n8CBw7Bz7lEZC31ebR8Tv0gRoCfAk8GellHsi4rXAzlJK90ePMep/bUsaabP5+sY9wJHAS4B3AldG\nxHFzbWQnEF3zFuD3SzS/djH5zpWlj9f3vb2WUnYBP+88/FFEHA18ALgWWBQRy7pGqeXA1l7LXYTH\n8DV8C5kcinHqb/2asQ2/gOYQ+W00AT9h9xMRcQTwSppdRGmP1+95qI8B62gONLwYeBfwZuBPSinb\nI+KLwGUR8Suaz1efBr7nET7tLfrd5TsI+BLwMmAb8D80YfpW5/kLaHZBv0ozaq0Hzs9pdf5ddURF\n0UcrF3bKv1UUnVi5sCUVNRXNx72V61Otfs9DzXiKvpTyLPD+ziTtdTwOICUyUFIiAyUlMlBSIgMl\nJTJQUiIDJSUyUFIiL+aeyT0vryj6XuXCPl9RU3tb9ppLNT9dUfPWyvWpliOUlMhASYlaG6hdw25g\nDq655sZhtzBro/zv3obeWxuo7m9NjpJRDtQo/7u3offWBkoaRQZKSmSgpERtOA+1BJofwpioTDFv\nvt1++86Kqjsnzdm27Uluv717fs/fqaH+//jZiprev+s81dra8O8+W4PqfcIye35VOkrp50eS8kXE\nmcCXh9qEVOddpZSrZypoQ6AOoPkxhfup/7UmaT4toblBxoZSyuMzFQ49UNKexIMSUiIDJSUyUFIi\nAyUlMlBSotYFKiLOj4jNEfFMRGyMiNcPu6caEXFRRIx3TXcPu6+pRMTqiLgxIh7u9HnyFDUXR8SW\niHg6Ir4REauG0Wu3Xr1HxOVT/B1umq/+WhWoiDgNuBS4CHgt8N/Ahog4cKiN1buT5vY9KzrTm4bb\nzrSWAncA5zHF7Y8i4kPA+4D3AEcDT9H8HRbNZ5PTmLH3jnU8/+9wxvy0BpRSWjMBG4FPTXgcNN8L\n/+Cwe6vo/SLg9mH3MYu+x4GTu+ZtAS6Y8HgZ8Axw6rD7rej9cuBrw+qpNSNUROxDc5/eiffoLcA3\nGZ179B7W2RW5LyKuioiVw26oXxFxMM27+sS/w3bgh4zO3+H4iBiLiHsiYm1E7D9fK25NoIADae4C\nOtY1f1Tu0bsROIvmMqpzgYOB70bE0mE2NQsraHalRvXvsA54N/BHwAdp7l92U0R033F2INpwtfke\noZSyYcLDOyNiE/AAcCrNbojmQSnl2gkP74qIHwP3AccD3x70+ts0Qj1G8y3m5V3zq+7R2zallG3A\nvUArjo71YSvNZ9c95e+wmWbbmpe/Q2sCVUr5Nc19eifeozc6j78/rL5mKyJeBBwKPDLsXvrR2QC3\n8vy/wzLgGEbz7/AK4ADm6e/Qtl2+y4ArIuI2YBPNLUb3Ba4YZlM1IuKTwL/T7Ob9FvB3ND/Ec80w\n+5pK53PdKpqRCOCQiDgSeKKU8hCwBrgwIn5G87WaS2iOtt4whHafZ6beO9NFwHU0bwqrgI/T7Cls\nmLy0ARj2oc8pDoWeR/NHfIbm7vGvG3ZPlX1fQ7PRPUNzU++rgYOH3dc0vb6Z5pDzc13Tv06o+QjN\n4fOnOxvjqmH33at3mu8tracJ0w7g58C/AC+dr/78PpSUqDWfoaQ9gYGSEhkoKZGBkhIZKCmRgZIS\nGSgpkYGSEhkoKZGBkhIZKCnR/wNgCwBKvmrn+wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdb9a24f4e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Heat map of position exploration:')\n",
    "plt.imshow(np.flipud(position_map.T), cmap='hot', interpolation='nearest')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Convert Q (action-values) to pi (policy)\n",
    "pi = np.zeros((x_size, y_size, 5, 5), dtype=np.int16)\n",
    "for idx in np.ndindex(x_size, y_size, 5, 5):\n",
    "    a = np.argmax(Q[idx[0], idx[1], idx[2], idx[3], :, :])\n",
    "    a = np.unravel_index(a, (3, 3))\n",
    "    pi[idx] = track.tuple_to_action(a - np.array([1, 1]))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample trajectory on learned policy:\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAANQAAAFfCAYAAAA23uK3AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAEQRJREFUeJzt3X/sXXV9x/Hny0JB6hgRZqtTJ4iYmUWiVZBNkA0zNkxw\ni0YFEwNkMQgawz8aMzMcZi5qIExdF5dtIFGbMJnRbUBFVBwqNKGyCYhGBEGg5ZehBKi17Wd/nFv9\nctt+e+/3+77fe277fCQn6T0/7nnne8+rn8/5ndYakmo8a9oFSPsSAyUVMlBSIQMlFTJQUiEDJRUy\nUFIhAyUVMlBSoQOmXUCSw4FTgXuALdOtRtqtg4GXAOtaa4/OO2drbSIDcD5wN/A0cBPw2j3MdybQ\nHBxmYDhzb9v9RFqoJG8HLgbeDawHLgDWJTmmtfbI0Oz3ACznmf3PrYNxs8jap2NSte8YfDeDbXU+\nk+ryXQB8trV2BUCSc4E3AecAnxiadwt0YZobqDC7O3jWPh1LUPted0nK15/kQGA1cP3Oca3r230d\nOKF6fVKfTCLQRwDLgE1D4zcBqyawPqk3pn6Ub6etdE32TtuBbfSoQO0XttFte3O1MZafxPb6CF1N\nK4fGrwQ27mmh4YMSsxymZdMuYBH299oPYNftbgejn88p7/K11n4F3AKcsnNckgw+f3fU75nVMIG1\nT0sfap9UDZcAlye5hd8cNj8EuHxC65uIJ2+edgX7oOPG6UDtxfrsfZ6C9W3YsIHVq1ePNO9EAtVa\nuzLJEcBFdF29W4FTW2sPT2J9Ul9MrJVsra0B1kzq+6U+mtVzeFIvGSipkIGSChkoqZCBkgoZKKmQ\ngZIK9eFqjSXnFRBjGuVqg1GuWhhnviqjrq/oCg5bKKmQgZIKGSipkIGSChkoqZCBkgoZKKmQgZIK\n7ZcndjWmpT4ZO8NsoaRCBkoqZKCkQgZKKmSgpEIGSipkoKRCBkoqZKCkQvvclRLe3j6GUW/79kqJ\nkdlCSYUMlFTIQEmFDJRUyEBJhQyUVMhASYUMlFRoZk7sesJ2AjxhW668hUpyYZIdQ8Md1euR+mhS\nLdRtwCnAzv8Ct01oPVKvTCpQ21prD0/ou6XemtRBiZcluT/JXUk+n+RFE1qP1CuTCNRNwFnAqcC5\nwJHAt5OsmMC6pF4p7/K11tbN+XhbkvXAz4C3AZftabmt/GaHa6dlkyhQmsfatWtZu3btM8Y9/vjj\nIy+f1mpehTjvSrpQXdda++vdTHs1cMvBzN9cethcEzXPvWEbNmxg9erVAKtbaxvm+5qJn9hN8hzg\npcCDk16XNG2TOA/1ySQnJfm9JH8IfJnusPnavSwqzbxJ7KK8EPgicDjwMHAj8LrW2qMTWJfUK5M4\nKHFG9XdKs8KLY6VCBkoqZKCkQgZKKmSgpEIGSipkoKRCXnu6rxrlueUj3gK/4vhF1jIT9vy32DHG\nt9hCSYUMlFTIQEmFDJRUyEBJhQyUVMhASYUMlFTIE7v7qhFO2u4fJ2yXli2UVMhASYUMlFTIQEmF\nDJRUyEBJhQyUVMhASYUMlFTIQEmFDJRUyEBJhQyUVMhASYUMlFTIQEmFDJRUyEBJhbwFfl81yrPN\n53metxZm7BYqyYlJvprk/iQ7kpy+m3kuSvJAkqeSXJfk6JpypX5bSJdvBXArcB6wy3+DST4IvBd4\nN3Ac8CSwLsnyRdQpzYSxu3yttWuBawGS7K7P8H7go621/xrM8y5gE/AXwJULL1Xqv9KDEkmOBFYB\n1+8c11rbDNwMnFC5LqmPqo/yraLrBm4aGr9pME3ap/XmKN9Wdj3mtIweFaj9wjZg+9C4UY6X7lS9\nvW6ky8VKntlKrQS+P9+Cy/GkmKbvAHYNxQ5gy4jLl27DrbW76UJ1ys5xSQ4Fjge+W7kuqY/GbqGS\nrACO5jc9tKOSHAs81lq7D7gU+HCSnwD3AB8Ffg58paRiqccW0uV7DfBNuq5lAy4ejP8ccE5r7RNJ\nDgE+CxwG/A/w5621rQX1alQjvuFdtRZyHuoG9tJVbK19BPjIwkqSZpfHAaRCBkoqZKCkQgZKKmSg\npEIGSipkoKRCBkoqZKCkQgZKKmSgpEIGSipkoKRCBkoqZKCkQgZKKmSgpEIGSipkoKRCBkoqZKCk\nQgZKKmSgpEIGSipkoKRCBkoq5OuXZtEob3j32eZTYQslFTJQUiEDJRUyUFIhAyUVMlBSIQMlFTJQ\nUqGZObG74vjR5nvy5snW0QuetO2tsVuoJCcm+WqS+5PsSHL60PTLBuPnDlfXlSz110K6fCuAW4Hz\ngD1dA3MNsBJYNRjOWFB10owZu8vXWrsWuBYgyZ76Hr9srT28mMKkWTSpgxInJ9mU5M4ka5I8d0Lr\nkXplEgclrgGuAu4GXgr8PXB1khNaayNcJi3NrvJAtdaunPPx9iQ/AO4CTga+uafltgLD/cdlkyhQ\nmsc2YPvQuHFagYmfh2qt3Q08Ahw933zLgYOGBsOkpXYAu26Hy8dYfuKBSvJC4HDgwUmvS5q2sRuB\nJCvoWpudPbSjkhwLPDYYLqTbh9o4mO/jwI+BdRUFS322kF7Va+j2hdpguHgw/nN056ZeCbwLOAx4\ngC5If9Na+9Wiq5V6biHnoW5g/q7iny28HGm2eXGsVMhASYUMlFTIQEmFDJRUyEBJhQyUVMhASYUM\nlFTIQEmFDJRUyEBJhQyUVMhASYUMlFTIQEmFDJRUyAcLzSLfAt9btlBSIQMlFTJQUiEDJRUyUFIh\nAyUVMlBSIQMlFfLE7izypG1v2UJJhQyUVMhASYUMlFTIQEmFDJRUyEBJhQyUVMhASYXGClSSDyVZ\nn2Rzkk1JvpzkmKF5Dkryj0keSfJEki8leV5t2VI/jdtCnQh8GjgeeCNwIPC1JM+eM8+lwJuAtwAn\nAS8Arlp8qVL/jXUtX2vttLmfk5wFPASsBm5McihwDvCO1toNg3nOBn6Y5LjW2vqSqqWeWuw+1GFA\nAx4bfF5NF9Lrd87QWvsRcC9wwiLXJfXeggOVJHTduxtba3cMRq8CtrbWNg/NvmkwTdqnLeb2jTXA\nK4DXVxSyFRi+KWEZ3l+ipbUN2D40boSnIP7agrbXJJ8BTgNObK09MGfSRmB5kkOHWqmVg2l7tByP\n4Wv6DmDXUOwAtoy4/Njb8CBMbwb+uLV279DkW+hCfsqc+V8OvBj43rjrkmbNWC1UkjXAGcDpwJNJ\nVg4mPd5a29Ja25zkX4FLkvwCeAL4FPAdj/BpfzBul+9cui7lt4bGnw1cMfj3BXTd0C8BBwHXAucv\nvERpdox7HmqvXcTW2i+B9w0Gab/icQCpkIGSChkoqZCBkgoZKKmQgZIKGSipkNeeziJfWt1btlBS\nIQMlFTJQUiEDJRUyUFIhAyUVMlBSIQMlFTJQUqF97kqJFcfvfZ4nb558HRPlVRC9ZQslFTJQUiED\nJRUyUFIhAyUVMlBSIQMlFTJQUiEDJRUyUFIhAyUVMlBSIQMlFTJQUiEDJRUyUFIhAyUVMlBSobEC\nleRDSdYn2ZxkU5IvJzlmaJ5vJdkxZ9ieZE1t2VI/jdtCnQh8GjgeeCNwIPC1JM+eM08D/hlYCawC\nng98YPGlSv031kNaWmunzf2c5CzgIWA1cOOcSU+11h5edHXSjFnsPtRhdC3SY0Pj35nk4SQ/SPKx\noRZM2mct+DFiSQJcCtzYWrtjzqQvAD8DHgBeCXwCOAZ46yLqlGbCYp7LtwZ4BfBHc0e21v5lzsfb\nk2wEvp7kyNba3Xv6sq3A8NPmli2yQGlc24DtQ+NGeF/kry1oe03yGeA04MTW2oN7mf1muqwcDewx\nUMvxGL6m7wB2DcUOYMsYy49lEKY3A29ord07wiKvogv53oInzbyxAjU4n3QGcDrwZJKVg0mPt9a2\nJDkKOBO4GngUOBa4BLihtXZbXdlSP43bQp1L19p8a2j82cAVdLtCbwTeD6wA7gP+Hfi7RVWpZ/It\n8L017nmoeXdzWms/B05eTEHSLPM4gFTIQEmFDJRUyEBJhQyUVMhASYUMlFTIa09nkSdte8sWSipk\noKRCBkoqZKCkQgZKKmSgpEIGSipkoKRCBkoqZKCkQgZKKmSgpEIGSipkoKRCBkoqZKCkQgZKKmSg\npELeAj+LfLZ5b9lCSYUMlFTIQEmFDJRUyEBJhQyUVMhASYUMlFTIQEmFxrpSIsm5wHuAlwxG3Q5c\n1Fq7djD9IOAS4O3AQcA64LzW2kNVBVdYcfxo8z1582TrWDCvguitcVuo+4APAq8GVgPfAL6S5PcH\n0y8F3gS8BTgJeAFwVU2pUv+N1UK11v57aNSHk7wHeF2S+4FzgHe01m4ASHI28MMkx7XW1pdULPXY\ngvehkjwryTuAQ4Dv0bVYBwDX75yntfYj4F7ghEXWKc2Esa82T/IHdAE6GHgC+MvW2p1JXgVsba1t\nHlpkE7Bq0ZVKM2Aht2/cCRwL/DbwVuCKJCcttpCtwPCu9jK8v0RLaxuwfWjcCDfL/NrY22trbRvw\n08HH7yc5Dng/cCWwPMmhQ63USmDj3r53OR7D1/QdwK6h2AFsGXH5im34WXSHyG+hC/gpOyckeTnw\nYrouorTPG/c81MeAa+gONPwW8E7gDcCfttY2J/lX4JIkv6Dbv/oU8B2P8Gl/MW6X73nA54DnA48D\n/0cXpm8Mpl9A1wX9El2rdS1wfk2p+4FRbm0HT+z22Ljnof5qL9N/CbxvMEj7HY8DSIUMlFTIQEmF\nDJRUyEBJhQyUVMhASYUMlFTIi7mXyEMj3XbvFRCzzhZKKmSgpEK9DdS2aRewCGvXTbuChZvlv3sf\nau9toIbvmpwla6+bdgULN8t/9z7U3ttASbPIQEmFDJRUqA/noQ6G7kEYc7XdjFtqG+5c2HKPP7Hr\nso8tvpxyu/v79uHvvlCTqn3Odx68t3nT2jgPSaqX5EzgC1MtQhrNO1trX5xvhj4E6nDgVOAeRn9a\nk7SUDqZ7Qca61tqj88049UBJ+xIPSkiFDJRUyEBJhQyUVMhASYV6F6gk5ye5O8nTSW5K8tpp1zSK\nJBcm2TE03DHtunYnyYlJvprk/kGdp+9mnouSPJDkqSTXJTl6GrUO21vtSS7bze9w9VLV16tAJXk7\ncDFwIfAq4H+BdUmOmGpho7uN7vU9qwbD66dbzh6tAG4FzmM3rz9K8kHgvcC7geOAJ+l+h+VLWeQe\nzFv7wDU883c4Y2lKA1prvRmAm4B/mPM5wM+BD0y7thFqvxDYMO06FlD3DuD0oXEPABfM+Xwo8DTw\ntmnXO0LtlwH/Ma2aetNCJTmQ7j29c9/R24CvMzvv6H3ZoCtyV5LPJ3nRtAsaV5Ij6f5Xn/s7bAZu\nZnZ+h5OTbEpyZ5I1SZ67VCvuTaCAI+jeArppaPysvKP3JuAsusuozgWOBL6dZMU0i1qAVXRdqVn9\nHa4B3gX8CfABuveXXZ1kSZ6A04erzfcJrbW5N77flmQ98DPgbXTdEC2B1tqVcz7enuQHwF3AycA3\nJ73+PrVQj9DdxbxyaPxI7+jtm9ba48CPgV4cHRvDRrp9133ld7ibbttakt+hN4Fqrf2K7j29c9/R\nm8Hn706rroVK8hzgpcCD065lHIMNcCPP/B0OBY5nNn+HFwKHs0S/Q9+6fJcAlye5BVhP94rRQ4DL\np1nUKJJ8EvhPum7e7wJ/S/cgnrXTrGt3Bvt1R/ObJ2seleRY4LHW2n3ApcCHk/yE7raaj9Idbf3K\nFMp9hvlqHwwXAlfR/adwNPBxup7C0jyLatqHPndzKPQ8uh/xabq3x79m2jWNWPdauo3uabqXen8R\nOHLade2h1jfQHXLePjT825x5PkJ3+PypwcZ49LTr3lvtdPctXUsXpi3AT4F/An5nqerzfiipUG/2\noaR9gYGSChkoqZCBkgoZKKmQgZIKGSipkIGSChkoqZCBkgoZKKnQ/wNfUdM199NQCwAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fdb98078d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Run learned policy on test case\n",
    "\n",
    "pos_map = np.zeros((x_size, y_size))\n",
    "track.reset()\n",
    "for e in range(1000):\n",
    "    s = track.get_state()\n",
    "    s_x, s_y = s[0][0], s[0][1]\n",
    "    s_vx, s_vy = s[1][0], s[1][1]\n",
    "    pos_map[s_x, s_y] += 1  # exploration map\n",
    "    act = track.action_to_tuple(pi[s_x, s_y, s_vx, s_vy])\n",
    "    track.take_action(act)\n",
    "    if track.is_terminal_state(): break    \n",
    "\n",
    "print('Sample trajectory on learned policy:')\n",
    "pos_map = (pos_map > 0).astype(np.float32)\n",
    "pos_map +=  track.course  # overlay track course\n",
    "plt.imshow(np.flipud(pos_map.T), cmap='hot', interpolation='nearest')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}