CartPole v0 Deep Q-Network Solver in Python (reward: 199.74 ± 0.23)

CartPole_v0_DQN_Solver.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CartPole v0 Deep Q-Network\n",
    "\n",
    "CartPole v0 problem resolution using DQN (Q-Learning using Neural Network and experience replay) with Tensorflow.\n",
    "See [online example](https://gym.openai.com/evaluations/eval_CABwYtsAQ6C9F19XwGz2w) at Gym OpenAI."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import gym\n",
    "from gym import wrappers\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-08-26 17:31:51,543] Making new env: CartPole-v0\n",
      "[2017-08-26 17:31:51,562] Clearing 24 monitor files from previous run (because force=True was provided)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env = wrappers.Monitor(env, 'CartPole-record',force=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Env variables\n",
    "nb_actions = 2\n",
    "\n",
    "# Training tuning\n",
    "train_batch_size = 10\n",
    "e = 0.5\n",
    "e_decay = 0.99\n",
    "y = .9\n",
    "lr = 1.0\n",
    "\n",
    "# Training loop\n",
    "nb_max_episodes = 3000\n",
    "episode_count = 0\n",
    "total_steps = 0\n",
    "update_freq = 100\n",
    "test_freq = 100\n",
    "log_freq = 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_input = 4\n",
    "n_hidden_1 = 128\n",
    "n_hidden_2 = 128\n",
    "\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_1, nb_actions]))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1])),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2])),\n",
    "    'out': tf.Variable(tf.random_normal([nb_actions]))\n",
    "}\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "x = tf.placeholder(shape=[None,4],dtype=tf.float32)\n",
    "layer_1 = tf.nn.tanh(tf.add(tf.matmul(x, weights['h1']), biases['b1']))\n",
    "layer_2 = tf.nn.tanh(tf.add(tf.matmul(layer_1, weights['h2']), biases['b2']))\n",
    "Qout = tf.add(tf.matmul(layer_2, weights['out']), biases['out'])\n",
    "\n",
    "nextQ = tf.placeholder(shape=[None,nb_actions],dtype=tf.float32)\n",
    "loss = tf.reduce_sum(tf.square(nextQ - Qout))\n",
    "trainer = tf.train.AdamOptimizer(learning_rate=0.0001)\n",
    "update_model = trainer.minimize(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experience Replay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ExperienceBuffer():\n",
    "    def __init__(self, buffer_size = 10000):\n",
    "        self.buffer = []\n",
    "        self.buffer_size = buffer_size\n",
    "    \n",
    "    def add(self,experience):\n",
    "        if len(self.buffer) + len(experience) >= self.buffer_size:\n",
    "            self.buffer = self.buffer[:1]\n",
    "        self.buffer.append(experience)\n",
    "            \n",
    "    def sample(self,size):\n",
    "        return random.sample(self.buffer, size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test accurracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test_accurracy():\n",
    "    test_batch_size = 100\n",
    "    c = 0\n",
    "    nb_success = 0.0\n",
    "    max_time, min_time = 0.0, float('inf')\n",
    "    while c < test_batch_size:\n",
    "        j = 0\n",
    "        is_gameover = False\n",
    "        state = env.reset()\n",
    "        while not is_gameover:\n",
    "            j += 1\n",
    "            Q1 = sess.run(Qout, feed_dict={x: [state]})[0]\n",
    "            action = np.argmax(Q1)\n",
    "            next_state, reward, is_gameover, _ = env.step(action)\n",
    "#             env.render()\n",
    "            state = next_state\n",
    "        nb_success += j\n",
    "        max_time = max(j, max_time)\n",
    "        min_time = min(j, min_time)\n",
    "        c += 1\n",
    "    return nb_success / c, min_time, max_time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-08-26 17:32:01,306] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000000.mp4\n",
      "[2017-08-26 17:32:02,263] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000001.mp4\n",
      "[2017-08-26 17:32:03,709] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000008.mp4\n",
      "[2017-08-26 17:32:06,172] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000027.mp4\n",
      "[2017-08-26 17:32:09,057] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000064.mp4\n",
      "[2017-08-26 17:32:11,735] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000125.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step: 5196, episode: 100, mean time: 53.69. min: 22. max: 81. random move probability: 2.88593455884936e-23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-08-26 17:32:14,651] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000216.mp4\n",
      "[2017-08-26 17:32:26,861] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000343.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step: 16413, episode: 200, mean time: 188.67. min: 125. max: 200. random move probability: 8.644598353212519e-72\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-08-26 17:32:49,647] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000512.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step: 34317, episode: 300, mean time: 184.7. min: 124. max: 200. random move probability: 1.6817035253530329e-149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-08-26 17:33:13,544] Starting new video recorder writing to D:\\workspace\\rl-dojo\\gym_cartpole_v0\\qlearning_nn\\CartPole-record\\openaigym.video.0.41032.video000729.mp4\n",
      "[2017-08-26 17:33:20,589] Finished writing results. You can upload them to the scoreboard via gym.upload('D:\\\\workspace\\\\rl-dojo\\\\gym_cartpole_v0\\\\qlearning_nn\\\\CartPole-record')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step: 50321, episode: 400, mean time: 196.44. min: 130. max: 200. random move probability: 6.425165430218056e-219\n"
     ]
    }
   ],
   "source": [
    "mean_times = []\n",
    "rewards = []\n",
    "minus_hist = []\n",
    "plus_hist = []\n",
    "\n",
    "init = tf.global_variables_initializer()\n",
    "\n",
    "replay_buffer = ExperienceBuffer()\n",
    "\n",
    "finished = False\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    \n",
    "    while not finished:\n",
    "        state = env.reset()\n",
    "        is_gameover = False\n",
    "        reward_tot = 0\n",
    "        episode_step = 0\n",
    "        \n",
    "        while not is_gameover:\n",
    "            episode_step += 1\n",
    "\n",
    "            # Eval current state\n",
    "            Q1 = sess.run(Qout, feed_dict={x: [state]})[0]\n",
    "            \n",
    "            action_minus, action_plus = Q1\n",
    "            minus_hist.append(action_minus)\n",
    "            plus_hist.append(action_plus)\n",
    "            \n",
    "            if random.random() < e:\n",
    "                action = random.randint(0, nb_actions-1)\n",
    "            else:\n",
    "                action = np.argmax(Q1)\n",
    "                \n",
    "            # Play best move with chance of random\n",
    "            next_state, reward, is_gameover, _ = env.step(action)\n",
    "            \n",
    "            if is_gameover:\n",
    "                reward = -100\n",
    "                \n",
    "            # Eval next state\n",
    "            Q2 = sess.run(Qout, feed_dict={x: [next_state]})[0]\n",
    "            \n",
    "            # Update first state evaluation\n",
    "            Q1[action] += lr * (reward + y*np.max(Q2) - Q1[action])\n",
    "            sess.run(update_model,feed_dict={x: [state], nextQ:[Q1]})\n",
    "\n",
    "            # Register experience\n",
    "            replay_buffer.add((state,Q1))\n",
    "\n",
    "            reward_tot += reward\n",
    "            total_steps += 1\n",
    "            state = next_state\n",
    "\n",
    "            if is_gameover:\n",
    "                break\n",
    "\n",
    "            e *= e_decay\n",
    "        \n",
    "        if total_steps % (update_freq) == 0 and len(replay_buffer.buffer) > train_batch_size:\n",
    "            train_batch = replay_buffer.sample(train_batch_size)\n",
    "            train_x, train_Q = list(zip(*train_batch))\n",
    "            \n",
    "            #Update the network with our target values.\n",
    "            sess.run(update_model, feed_dict={\n",
    "                x: train_x,\n",
    "                nextQ: train_Q\n",
    "            })\n",
    "        \n",
    "        if episode_count % test_freq == 0 and episode_count != 0:\n",
    "            mean_time, min_time, max_time = test_accurracy()\n",
    "            mean_times.append(mean_time)\n",
    "\n",
    "            if episode_count % log_freq == 0:\n",
    "                print('step: %s, episode: %s, mean time: %s. min: %s. max: %s. random move probability: %s' \n",
    "                      % (total_steps, episode_count, mean_time, min_time, max_time, e))\n",
    "                \n",
    "            if mean_time > 196:\n",
    "                finished = True\n",
    "                \n",
    "        episode_count += 1\n",
    "        rewards.append(reward_tot)\n",
    "    env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt0VOW9//H3NxcSSLiaBLnfEkBUVIyKWit4A6wt9tRa\nPbZFj7fTWlu19dZzVj3nrNVVLFar9Xf00IpV2x/Wan+Vcw4XEalYKyhaL6gBAgiEWwJIIAnk+v39\nMRsIYSCTy8zOJJ/XWlnZ88yeme92cD55nmfPs83dERERaSol7AJERKRjUkCIiEhUCggREYlKASEi\nIlEpIEREJCoFhIiIRKWAEBGRqBQQIiISlQJCRESiSgu7gLbIycnx4cOHh12GiEhSeffdd3e6e25z\n+yV1QAwfPpyVK1eGXYaISFIxs42x7Be3ISYzG2JmS83sEzP72Mx+ELT3M7PFZrY2+N03aDcze8zM\nis3sQzObEK/aRESkefGcg6gDfuju44CJwG1mNg64D1ji7gXAkuA2wDSgIPi5BXgijrWJiEgz4hYQ\n7r7N3d8LtvcBnwKDgOnAM8FuzwBXBtvTgWc9YjnQx8wGxKs+ERE5voScxWRmw4EzgBVAf3ffFty1\nHegfbA8CNjd6WEnQ1vS5bjGzlWa2sqysLG41i4h0dXEPCDPLBl4C7nD3vY3v88jFKFp0QQp3n+3u\nhe5emJvb7CS8iIi0UlwDwszSiYTD7939T0HzjoNDR8Hv0qB9CzCk0cMHB20iIhKCeJ7FZMBTwKfu\n/nCju+YBM4LtGcDLjdq/HZzNNBEobzQUJSIiCRbP70GcD3wL+MjM3g/afgzMBF4wsxuBjcDVwX3z\ngcuBYqAKuCGOtYmIJJXa+gY27qqiuLSCdWUVjB/cmwsK4jvMHreAcPe/AnaMuy+Osr8Dt8WrHhGR\nZFBVU8f6skqKSysO/5RVsHFXJbX1h6dsvzNpVPIGhIiIHNvnlTUUl1UcGQSlFWzZs//QPikGw07I\nYlRuNpec1J/8vGzy87IZlZtFz8z0uNeogBARiRN3Z1v5gSN6AsWlFawrrWBXZc2h/TLSUhiZm82E\nYX25unDIoSAYntODjLTU0OpXQIiItFFdfQObdkfmB9YGAVBcFvldWVN/aL9emWnk52Vz8Ul5h0Ig\nP7cng/p2JzXlWCPy4VFAiIjE6EBtPesa9QIO9gg+21lFTX3Dof3698ogPy+bq84cHBkSCsIgNzuD\nyAmeyUEBISLSRHlVLcVl+46aKC75fD8ezBOnGAzt14P8vGwmj80jPzf7UBj0SsD8QCIoIESkS3J3\nduytDgJgX6MJ40p2VlQf2q9bWgojc7I4bXAfvjZh8OH5gROyyEwPb34gERQQItKp1Tf4ofmBxr2B\n9aUV7KuuO7Rfz4w0RuVlM2lMLgUH5wfyshnct0eHnB9IBAWEiHQKB2rrI98faDxHUFrBhp2VR8wP\n5PbMID83myvPGHR4ojgvm7yeyTU/kAgKCBFJKuX7a4+aJC4urWDz51WH5gfMYEjfyPzAhWNyyc89\nPFHcu3vnmB9IBAWEdBgzFxTx6ba9ZGekRX4yI797Zh55+4j7MtLJykglLTUhK9dLgrg7Zfuqj/ju\nwMFTSMv2NZofSE1hRE4Wpw7qfbhHkJvNyNzOPz+QCAoI6RD+tm4nT76+jpE5WZhBRXUdFQfqjjiH\n/Hi6p6ceGSBBiPRsHCxR708/Iogy0lI0zJBA9Q1OyedHzw8Ul1aw78Dh+YHsYH7giwW5RwwLDenb\nXX8cxJECQkLX0ODMXFDEwN6ZzP/BBUf85Vff4FTWBGFRXce+IDgOBsjh27VUVNcH7bVUVNexeXdV\n5HawT11D85ceSU2xQwEStedyVM8mPWrPJjsjrctObEZTXVfPhp2VRy0rsX5nJTV1h+cHcrIzyM/L\nYvrpA4PTRnuSn5dN/16aHwiDAkJC978fbePDknIe+vppRw0LpKYYvTLT23xeubtTXddwKCwqquvY\ndzBoqmuDtvpD242DaHdlDZt2Vx26XRVjr6ZHt9QjejJZUXs2BwMmNbIdJZiSqVez70DtET2BgxPF\nm3ZX0dBofmBw3+7k52ZzQUHOEd8o7t1D8wMdiQJCQlVT18CsRasZe2JPvnrGUVeYbTdmRmZ6Kpnp\nqeRkZ7TpuerqG6isqW8UNkHvJdje16iHUxH0eiqD25sqqxoFUx31MfRq0lPtcLgcESCHAyWrW/Qh\ntUPBFOzTHr0ad6esovqIM4UODgvt2Ht4fiA91RiRk8W4gb34ymkDD00Sj8zJpns3zQ8kAwWEhOr/\nrtjIpt1VPH3DWUkzJJOWmkLv7iltPhvG3TlQ28C+6loqg4DZd6g3c2Qvp7LJkNrOiho+23UwbGo5\nUNvQ/AsCWd0azdVkpgcBEum9NO25HAwegPU7jxwa2ttofiCrWyr5edmcn59zaJI4Py+bof16aH4g\nySkgJDT7DtTy2GvFnDvyBCaN7nrXFzczundLjfw13bNtz1VX30BldX0kYI6anwnmb6L0bCoO1FK6\n70DkscHczbE6NSdkdWNUXjZXnDbwUAjk52UzoHdm0gyBScsoICQ0s5etZ3dlDfdfPlYfMG2UlppC\n7x4pbR7Dd3f219YfETD17ow4IYu+Wd3aqVpJFgoICUXp3gP85o0NXDF+AOMH9wm7HAmYGT26pdGj\nWxp5YRcjoYvbAKGZzTGzUjNb1ajtdDNbbmbvm9lKMzs7aDcze8zMis3sQzObEK+6pGP45ZK11DU0\ncPeUMWGXIiLHEM8ZpN8CU5u0/Rz4d3c/HfhJcBtgGlAQ/NwCPBHHuiRk68oq+MM7m7nunGEMOyEr\n7HJE5BjiFhDuvgzY3bQZ6BVs9wa2BtvTgWc9YjnQx8wGxKs2CdfPFxbRPT2V2y/KD7sUETmORM9B\n3AEsMrOHiITTeUH7IGBzo/1KgrZtiS1P4u3djbtZ9PEOfnjpaE5o4/cRRCS+En2S8neAO919CHAn\n8FRLn8DMbgnmL1aWlZW1e4ESP+7Oz+YXkdczgxsvGBF2OSLSjEQHxAzgT8H2H4Gzg+0twJBG+w0O\n2o7i7rPdvdDdC3Nzu96588ls8Sc7WLnxc+64ZDQ9uukEOpGOLtEBsRW4MNi+CFgbbM8Dvh2czTQR\nKHd3DS91InX1DTy4sIhRuVlcXTg47HJEJAZx+zPOzOYCk4AcMysBHgBuBh41szTgAJEzlgDmA5cD\nxUAVcEO86pJw/PHdEtaVVfJf3zpTyy+IJIm4BYS7X3uMu86Msq8Dt8WrFglXVU0djyxew5nD+nLZ\nuP5hlyMiMdKfchJ3c/66gdJ91fxYS2qIJBUFhMTVropqnnx9PZeN68+Zw/qFXY6ItIACQuLqV68V\ns7+2nnumjg27FBFpIQWExM2mXVX8fsVGri4cQn5edtjliEgLKSAkbma9spq0lBTuvKQg7FJEpBUU\nEBIXH5bs4b8/2MqNXxhBXq/MsMsRkVZQQEi7c3dmLiiiX1Y3br1wZNjliEgrKSCk3b2+poy/rdvF\n7Rfl0zOzbVc4E5HwKCCkXdU3RHoPQ/v14LpzhoVdjoi0gQJC2tWf/76Fou37+NGUMXRL0z8vkWSm\n/4Ol3RyorefhxWs4dVBvrjhV13sSSXYKCGk3z771GVv27Of+aWNJSdGSGiLJTgEh7aK8qpb/s3Qd\nF47O5bz8nLDLEZF2oICQdvGfrxez90At903TkhoinYUCQtps6579PP3mZ3z1jEGcNKBX2OWISDtR\nQEibPbx4DQA/vGxMyJWISHtSQEibFG3fy0vvlXD9ecMZ1Kd72OWISDtSQEibPLigiJ4ZaXx30qiw\nSxGRdqaAkFZ7a90ulq4u47bJ+fTp0S3sckSkncUtIMxsjpmVmtmqJu23m1mRmX1sZj9v1H6/mRWb\n2WozmxKvuqR9RBbk+5SBvTOZcd7wsMsRkThIi+Nz/xZ4HHj2YIOZTQamA6e5e7WZ5QXt44BrgJOB\ngcCrZjba3evjWJ+0wf9+tI0PSsp56OunkZmeGnY5IhIHcetBuPsyYHeT5u8AM929OtinNGifDjzv\n7tXuvgEoBs6OV23SNjV1DcxatJqxJ/bkq2cMCrscEYmTRM9BjAYuMLMVZva6mZ0VtA8CNjfaryRo\nkw5o7tub2LirinunjSVVS2qIdFrxHGI61uv1AyYCZwEvmFmLrihjZrcAtwAMHTq03QuU49t3oJbH\nlqzl3JEnMGl0btjliEgcJboHUQL8ySPeBhqAHGALMKTRfoODtqO4+2x3L3T3wtxcfUAl2q+XrWdX\nZQ33Xz4WM/UeRDqzRAfEn4HJAGY2GugG7ATmAdeYWYaZjQAKgLcTXJs0o3TvAX79xgauGD+A8YP7\nhF2OiMRZ3IaYzGwuMAnIMbMS4AFgDjAnOPW1Bpjh7g58bGYvAJ8AdcBtOoOp4/nlkrXU1jfwIy2p\nIdIlxC0g3P3aY9z1zWPs/1Pgp/GqR9pmXVkFf3hnM988ZyjDc7LCLkdEEkDfpJaY/HxhEZlpKdx+\ncUHYpYhIgiggpFnvbtzNoo93cOuFo8jJzgi7HBFJEAWEHJe787P5ReT2zOCmC0aEXY6IJJACQo5r\n8Sc7WLnxc+64pIAe3RL9tRkRCZMCQo6prr6BBxcWMTI3i28UDmn+ASLSqSgg5JhefLeEdWWV3DNl\nLGmp+qci0tXo/3qJan9NPY+8uoYzh/Vlysn9wy5HREKggJCo5ry5gR17q7l/mpbUEOmqFBBylN2V\nNTz5l3VcOq4/hcP7hV2OiIREASFH+dVra6msqePeqVpSQ6QrU0DIETbtquJ3yzfyjbOGkJ/XM+xy\nRCRECgg5wkOvrCY1xbjjktFhlyIiIVNAyCEflZQz74Ot3PSFkfTvlRl2OSISMgWEAJElNWYu/JR+\nWd249cIWXeRPRDopBYQAsGztTt4s3sXtF+XTMzM97HJEpANQQAgNDc7MBUUM7deD684ZFnY5ItJB\nKCCEP7+/hU+37eVHU8bQLU3/JEQkIqZPAzMbZmaXBNvdzUznP3YSB2rr+cUrazh1UG+uOHVA2OWI\nSAfSbECY2c3Ai8B/BU2DgT/HsyhJnOfe2siWPfu5f9pYUlK0pIaIHBZLD+I24HxgL4C7rwXymnuQ\nmc0xs1IzWxXlvh+amZtZTnDbzOwxMys2sw/NbELLDkNao7yqlseXFnPh6FzOy88JuxwR6WBiCYhq\nd685eMPM0gCP4XG/BaY2bTSzIcBlwKZGzdOAguDnFuCJGJ5f2ug/Xy9m74Fa7p06NuxSRKQDiiUg\nXjezHwPdzexS4I/Afzf3IHdfBuyOctcjwD0cGTLTgWc9YjnQx8w0IB5HW/fs5+k3P+Orpw9i3MBe\nYZcjIh1QLAFxH1AGfATcCswH/rU1L2Zm04Et7v5Bk7sGAZsb3S4J2qI9xy1mttLMVpaVlbWmDAEe\nXrwGHO66TEtqiEh0zV5k2N0bgF8HP61mZj2AHxMZXmo1d58NzAYoLCyMZahLmijavpeX3ivhpi+M\nYHDfHmGXIyIdVCxnMV1hZn83s91mttfM9pnZ3la81ihgBPCBmX1G5Gyo98zsRGAL0Piix4ODNomD\nBxcU0TMjjdsm54ddioh0YLEMMf0SmAGc4O693L2nu7d40NrdP3L3PHcf7u7DiQwjTXD37cA84NvB\n2UwTgXJ339bS15DmvbVuF0tXl/Hdyfn06dEt7HJEpAOLJSA2A6vcvUXDOWY2F3gLGGNmJWZ243F2\nnw+sB4qJDGV9tyWvJbFxd2Yu+JQBvTO5/rzhYZcjIh1cs3MQRM44mm9mrwPVBxvd/eHjPcjdr23m\n/uGNtp3I9y0kjuZ/tJ0PSsqZddV4MtNTwy5HRDq4WALip0AFkAloTCJJ1dY3MGtREWNP7Mk/TBgc\ndjkikgRiCYh+7t6mM48kfHPf3sRnu6p4+vqzSNWSGiISg1jmIF41MwVEEquoruPRV9cycWQ/Jo3J\nDbscEUkSsfQgbgPuMbNqoBYwItMG+vptkpi9bD27KmuYM+0kzNR7EJHYxPJFOS3tncRK9x3gN2+s\n50vjB3DakD5hlyMiSeSYAWFmY9296Fgrq7r7e/ErS9rLo6+upaaugbsvGxN2KSKSZI7Xg7iLyMqq\nv4hynwMXxaUiaTfryip4/p3NfPOcoQzPyQq7HBFJMscMCHe/Jdic5u4HGt9nZplxrUraxayFq8lM\nS+H2iwvCLkVEklAsZzH9LcY26UDe3fg5Cz/ezq0XjiInOyPsckQkCR1vDuJEIktudzezM4icvQTQ\nC9ASoB3YwSU1cntmcNMFI8IuR0SS1PHmIKYA1xNZWfUXHA6IvUSW7ZYO6tVPS3nns8/56VdPoUe3\nWM5kFhE52vHmIJ4BnjGzr7n7SwmsSdqgrr6BBxcWMTI3i28UDmn+ASIix9DsHITCIbm8+G4JxaUV\n3DNlLGmpsUwxiYhEp0+QTmR/TT2PvLqGCUP7MOXk/mGXIyJJTgHRicx5cwM79lZz/+VaUkNE2i6m\nGUwzOw8Y3nh/d382TjVJK+yurOHJv6zjkpP6c9bwfmGXIyKdQLMBYWbPEbme9PtAfdDsgAKiA/nV\na2uprKnj3qlaUkNE2kcsPYhCYFxLLzkqibNpVxW/W76RqwuHUNBfayuKSPuIZQ5iFXBiS5/YzOaY\nWamZrWrUNsvMiszsQzP7f2bWp9F995tZsZmtNrMpLX29ruyhV1aTmmLceenosEsRkU4kloDIAT4x\ns0VmNu/gTwyP+y0wtUnbYuAUdx8PrAHuBzCzccA1wMnBY/7TzHTR5Bh8VFLOvA+2cuMXRtC/l5bI\nEpH2E8sQ07+15ondfZmZDW/S9kqjm8uBq4Lt6cDz7l4NbDCzYuBs4K3WvHZX8uDCIvr2SOfWC0eF\nXYqIdDKxXDDo9Ti99j8Bfwi2BxEJjINKgjY5jmVryvhr8U5+csU4emWmh12OiHQyzQ4xmdlEM3vH\nzCrMrMbM6s1sb1te1Mz+BagDft+Kx95iZivNbGVZWVlbykhqDQ3OzAVFDOnXnesmDg27HBHphGKZ\ng3gcuBZYC3QHbgraWsXMrgeuAK5rdGbUFqDxwkGDg7ajuPtsdy9098Lc3NzWlpH0Xv5gC59s28uP\nLhtDRpqma0Sk/cX0TWp3LwZS3b3e3Z8GJrXmxcxsKnAP8BV3r2p01zzgGjPLMLMRQAHwdmteoys4\nUFvPQ4vWcOqg3nx5/MCwyxGRTiqWSeoqM+sGvG9mPwe2Ac1ev9LM5hIJkhwzKwEeIHLWUgawOFgK\nYrm7/7O7f2xmLwCfEBl6us3d66M/s/xu+Ua27NnPz68aT0qKltQQkfiIJSC+RaSn8T3gTiJDQV9r\n7kHufm2U5qeOs/9PgZ/GUE+XVr6/lseXFvPF0bmcn58Tdjki0onFchbTRjPrDgxw939PQE1yHE/8\nZR3l+2u5b+rYsEsRkU4ulrOYvkxkHaaFwe3TY/yinLSzrXv28/SbG/jq6YMYN7BX2OWISCcXyyT1\nvxH50toeAHd/H9CFjkPwyOI1uMNdl2lJDRGJv1gCotbdy5u0aeG+BFu9fR8vvVfCjPOGMbhvj7DL\nEZEuIJZJ6o/N7B+BVDMrAL4P/C2+ZUlTDy4sIjsjjdsm54ddioh0EbH0IG4nsoheNTAX2AvcEc+i\n5EjL1+/itaJSvjs5nz49uoVdjoh0EbGcxVQF/EvwIwnm7vxsQREDemdy/XnDwy5HRLqQWK4oVwj8\nmKMvOTo+fmXJQfM/2s4Hm/cw66rxZKZrSQ0RSZxY5iB+D9wNfAQ0xLccaay2voFZi4oY078n/zBh\ncNjliEgXE0tAlLm7vvcQgrlvb+KzXVXMub6QVC2pISIJFktAPGBmvwGWEJmoBsDd/xS3qoSK6joe\nfXUt54zox+QxeWGXIyJdUCwBcQMwFkjn8BCTAwqIOJq9bD27Kmt46vKTCBY2FBFJqFgC4jR3PzXu\nlcghpfsO8Js31vOlUwdw+pA+YZcjIl1ULN+DWG5m4+JeiRzy6Ktrqalr4O4pY8IuRUS6sFh6EF8A\nZpjZBiJzEAa4TnONj/VlFTz/zmauO2cow3OaveyGiEjcxBIQU+NehRwya9FqMtNS+P7FBWGXIiJd\nXEzXg0hEIQLvbfqcBau2c+clo8nJzgi7HBHp4mK6JrXEn7szc34ROdkZ3HSBVlMXkfApIDqIJZ+W\n8vZnu7njkgKyMmIZ+RMRia+4BYSZzTGzUjNb1aitn5ktNrO1we++QbuZ2WNmVmxmH5rZhHjV1RHV\n1Tfw4MIiRuZk8Y2zhoRdjogIEN8exG85eoL7PmCJuxcQ+Wb2fUH7NKAg+LkFeCKOdXU4L71XwtrS\nCu6ZOob0VHXqRKRjiNunkbsvA3Y3aZ4OPBNsPwNc2aj9WY9YDvQxswHxqq0j2V9Tz8OL1zBhaB+m\nnHxi2OWIiByS6D9X+7v7tmB7O9A/2B4EbG60X0nQ1unNeXMDO/ZWc7+W1BCRDia08Qx3d1pxbWsz\nu8XMVprZyrKysjhUlji7K2t48i/ruOSk/pw1vF/Y5YiIHCHRAbHj4NBR8Ls0aN8CNJ6dHRy0HcXd\nZ7t7obsX5ubmxrXYeHv8tWIqa+q4d6qW1BCRjifRATEPmBFszwBebtT+7eBspolAeaOhqE5p8+4q\nnlv+GVcXDqGgf8+wyxEROUrcTrg3s7nAJCDHzEqAB4CZwAtmdiOwEbg62H0+cDlQDFQRWWK8U3vo\nldWkphh3Xjo67FJERKKKW0C4+7XHuOviKPs6cFu8auloVm0p5+X3t3Lb5FH075UZdjkiIlHppPsQ\nzFxQRN8e6dx64aiwSxEROSYFRIItW1PGX4t38r2LCuiVmR52OSIix6SASKCGBmfmgiIG9+3ONycO\nDbscEZHjUkAk0MsfbOGTbXu5e8oYMtJSwy5HROS4FBAJcqC2nocWreGUQb348viBYZcjItIsBUSC\n/G75Rrbs2c99U08iJUVLaohIx6eASIDy/bU8vrSYCwpy+EJBTtjliIjERAGRAE/8ZR3l+2u5b9rY\nsEsREYmZAiLOtpXv5+k3N3Dl6YM4eWDvsMsREYmZAiLOHlm8Bne4S0tqiEiSUUDE0Zod+3jx3RK+\nfe4whvTrEXY5IiItooCIowcXFJGVkcZtk/PDLkVEpMUUEHGyYv0ulhSV8t1J+fTN6hZ2OSIiLaaA\niAN352cLihjQO5Mbzh8edjkiIq2igIiDBau28/7mPdx56Wgy07WkhogkJwVEO6utb2DWotWM6d+T\nr00YHHY5IiKtpoBoZ8+/vYkNOyu5d9oYUrWkhogkMQVEO6qoruPRJWs5Z0Q/Jo/JC7scEZE2UUC0\no18vW8/Oihruv/wkzNR7EJHkFkpAmNmdZvaxma0ys7lmlmlmI8xshZkVm9kfzCypzg0t3XeAX7+x\nni+dOoDTh/QJuxwRkTZLeECY2SDg+0Chu58CpALXAA8Cj7h7PvA5cGOia2uLx5aspaaugbunjAm7\nFBGRdhHWEFMa0N3M0oAewDbgIuDF4P5ngCtDqq3F1pdVMPftzfzjOUMZnpMVdjkiIu0i4QHh7luA\nh4BNRIKhHHgX2OPudcFuJcCgaI83s1vMbKWZrSwrK0tEyc2atWg1mWkp3H5RQdiliIi0mzCGmPoC\n04ERwEAgC5ga6+Pdfba7F7p7YW5ubpyqjN17mz5nwart3PzFkeT2zAi7HBGRdhPGENMlwAZ3L3P3\nWuBPwPlAn2DICWAwsCWE2lrE3Zk5v4ic7AxuvmBk2OWIiLSrMAJiEzDRzHpY5FzQi4FPgKXAVcE+\nM4CXQ6itRZZ8Wsrbn+3mB5cUkJWR1vwDRESSSBhzECuITEa/B3wU1DAbuBe4y8yKgROApxJdW0vU\n1Tfw4MIiRuZkcc1ZQ8IuR0Sk3YXyZ6+7PwA80KR5PXB2COW0ykvvlbC2tIInrptAeqq+byginY8+\n2Vphf009Dy9ewxlD+zD1lBPDLkdEJC4UEK3w9N82sGNvNfdP05IaItJ5KSBa6PPKGp74yzouOSmP\ns0f0C7scEZG4UUC00ONLi6msruPeqWPDLkVEJK4UEC2weXcVz721ka+fOYSC/j3DLkdEJK4UEC3w\ni1dWk5ICd146OuxSRETiTgERo1Vbyvnz+1v5p/NHcGLvzLDLERGJOwVEjB5cWETfHun886RRYZci\nIpIQCogYvLG2jDfW7uR7FxXQKzM97HJERBJCAdGMhgZn5oIiBvftzjcnDg27HBGRhFFANGPeB1v5\neOte7p4yhoy01LDLERFJGAXEcVTX1fPQK6s5ZVAvvjx+YNjliIgklALiOJ57ayMln+/nvqknkZKi\nJTVEpGtRQBxD+f5aHl9azAUFOXyhICfsckREEk4BcQxPvr6O8v213DdNS2qISNekgIhiW/l+5vx1\nA1eePoiTB/YOuxwRkVAoIKJ4ZPEa3OEuLakhIl2YAqKJNTv28eK7JXzr3GEM6dcj7HJEREKjgGji\nwQVFZGWk8b3J+WGXIiISqlACwsz6mNmLZlZkZp+a2blm1s/MFpvZ2uB330TXtWL9LpYUlfKdSaPo\nm9Ut0S8vItKhhNWDeBRY6O5jgdOAT4H7gCXuXgAsCW4njLvzswVFnNgrk386f0QiX1pEpENKeECY\nWW/gi8BTAO5e4+57gOnAM8FuzwBXJrKuBau28/7mPdx16Wgy07WkhohIGD2IEUAZ8LSZ/d3MfmNm\nWUB/d98W7LMd6B/twWZ2i5mtNLOVZWVl7VJQbX0DsxatZnT/bL525uB2eU4RkWQXRkCkAROAJ9z9\nDKCSJsNJ7u6AR3uwu89290J3L8zNzW2Xgp5/ZzMbdlZy79SxpGpJDRERIJyAKAFK3H1FcPtFIoGx\nw8wGAAS/SxNRTGV1HY++upazR/TjorF5iXhJEZGkkPCAcPftwGYzGxM0XQx8AswDZgRtM4CXE1HP\nr99Yz86Kau6fNhYz9R5ERA5KC+l1bwd+b2bdgPXADUTC6gUzuxHYCFwd7yLK9lUze9l6Lj/1RM4Y\nmvCzakVEOrRQAsLd3wcKo9x1cSLreGzJWmrqGrh7ihbkExFpqst+k3rDzkrmvr2Ja88eyoicrLDL\nERHpcLrNf6yoAAAGFElEQVRsQMxaVERGWgrfv7gg7FJERDqkLhkQf9/0OfM/2s7NXxxJbs+MsMsR\nEemQumRAAFxQkMPNF4wMuwwRkQ4rrLOYQnXG0L48d+M5YZchItKhddkehIiIHJ8CQkREolJAiIhI\nVAoIERGJSgEhIiJRKSBERCQqBYSIiESlgBARkagscvG25GRmZUSWBm+NHGBnO5YTJh1Lx9RZjqWz\nHAfoWA4a5u7NXpIzqQOiLcxspbtHW3I86ehYOqbOciyd5ThAx9JSGmISEZGoFBAiIhJVVw6I2WEX\n0I50LB1TZzmWznIcoGNpkS47ByEiIsfXlXsQIiJyHJ0+IMxsqpmtNrNiM7svyv0ZZvaH4P4VZjY8\n8VXGJoZjud7Myszs/eDnpjDqbI6ZzTGzUjNbdYz7zcweC47zQzObkOgaYxXDsUwys/JG78lPEl1j\nLMxsiJktNbNPzOxjM/tBlH2S4n2J8ViS5X3JNLO3zeyD4Fj+Pco+8fsMc/dO+wOkAuuAkUA34ANg\nXJN9vgs8GWxfA/wh7LrbcCzXA4+HXWsMx/JFYAKw6hj3Xw4sAAyYCKwIu+Y2HMsk4H/CrjOG4xgA\nTAi2ewJrovz7Sor3JcZjSZb3xYDsYDsdWAFMbLJP3D7DOnsP4myg2N3Xu3sN8Dwwvck+04Fngu0X\ngYvNzBJYY6xiOZak4O7LgN3H2WU68KxHLAf6mNmAxFTXMjEcS1Jw923u/l6wvQ/4FBjUZLekeF9i\nPJakEPy3rghupgc/TSeO4/YZ1tkDYhCwudHtEo7+h3JoH3evA8qBExJSXcvEciwAXwu6/y+a2ZDE\nlNbuYj3WZHFuMESwwMxODruY5gRDFGcQ+Wu1saR7X45zLJAk74uZpZrZ+0ApsNjdj/m+tPdnWGcP\niK7mv4Hh7j4eWMzhvyokPO8RWdbgNOBXwJ9Drue4zCwbeAm4w933hl1PWzRzLEnzvrh7vbufDgwG\nzjazUxL12p09ILYAjf+KHhy0Rd3HzNKA3sCuhFTXMs0ei7vvcvfq4OZvgDMTVFt7i+V9Swruvvfg\nEIG7zwfSzSwn5LKiMrN0Ih+ov3f3P0XZJWnel+aOJZnel4PcfQ+wFJja5K64fYZ19oB4BygwsxFm\n1o3IBM68JvvMA2YE21cBr3kw29PBNHssTcaDv0Jk7DUZzQO+HZw1MxEod/dtYRfVGmZ24sHxYDM7\nm8j/cx3uD5CgxqeAT9394WPslhTvSyzHkkTvS66Z9Qm2uwOXAkVNdovbZ1haezxJR+XudWb2PWAR\nkbOA5rj7x2b2H8BKd59H5B/Sc2ZWTGSy8ZrwKj62GI/l+2b2FaCOyLFcH1rBx2Fmc4mcRZJjZiXA\nA0Qm33D3J4H5RM6YKQaqgBvCqbR5MRzLVcB3zKwO2A9c00H/ADkf+BbwUTDeDfBjYCgk3fsSy7Ek\ny/syAHjGzFKJhNgL7v4/ifoM0zepRUQkqs4+xCQiIq2kgBARkagUECIiEpUCQkREolJAiIhIVAoI\nkVYyszvMrEfYdYjEi05zFWklM/sMKHT3nWHXIhIP6kGIxMDMsszsf4PF3VaZ2QPAQGCpmS0N9rnM\nzN4ys/fM7I/BWkCY2Wdm9mCwrv/bZpYftH89eK4PzGxZeEcnEp0CQiQ2U4Gt7n6au58C/BLYCkx2\n98nBOj7/Clzi7hOAlcBdjR6/193PBh4PHgvwE2BKsGDcVxJ1ICKxUkCIxOYj4NKgJ3CBu5c3uX8i\nMA54M1jeYQYwrNH9cxv9PjfYfhP4rZndTGT5FJEOpVOvxSTSXtx9TXCJzcuBn5nZK012MSJr9V97\nrKdouu3u/2xm5wBfAt43s9PdvcMtGCddl3oQIjEws4FAlbv/DniIyGVG9xG5pCXAcuD8RvMLWWY2\nutFTfKPR77eCfUa5+wp3/wmwkyOX0hYJnXoQIrE5FZhlZg1ALfAdIkNFC81sazAPcT0w18wygsf8\nK5HrIQNkmNkKIn+UHexlzDKzAiK9jyVErjMu0mHoNFeRONPpsJKsNMQkIiJRqQchIiJRqQchIiJR\nKSBERCQqBYSIiESlgBARkagUECIiEpUCQkREovr/N+77qoCf6gkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x241bba18748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.plot(mean_times)\n",
    "plt.xlabel('steps')\n",
    "plt.ylabel('mean time')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEICAYAAABcVE8dAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XeY1NT6B/Dvu30XlmWXZSlbWHqXttKRXgQUG15U7F3U\naxcEe7ncq14Vyw+xXFGwYAELTUARUECW3vvSe+9sOb8/ktnNzE7JTJJJMnk/z8PDTCaTnOycvDk5\nLSSEAGOMscgXZXYCGGOMhQcHfMYYcwgO+Iwx5hAc8BljzCE44DPGmENwwGeMMYfggM8YYw7BAT9E\nRDSWiJ41Ox16I6LPiOgVs9PBzEdEc4noLrPTESwiuo2IFpidDivigO+BiAqI6CIRpXssX05Egohy\nAUAIcZ8Q4mUT0jeQiP4mojNEdISIJhBRpvxZe3l5RS/fW05ED4Y7vcza5Px+johOE9EB+YJfLv/o\nvM8mRPQTEZ0golNE9BsRtZc/SyCi40TUw8v33iKi74xMW6TjgO/ddgA3uN4QUXMASeYlpzQd1wH4\nEsDbANIBNAVwEcB8IqoshFgEYDeA6zy+1wxAEwBfhTfFzCauEEJUBNAaQB6AUUbtiIjqAvgTwGoA\ntQHUBDAFwCwiaiuEOA/gGwC3eHwvGtI5Od6otDkBB3zvvoB7hrsVwOfKFZRVH0TUjYh2E9HjRHSQ\niPYR0e2Kdd1ujZW3nCR5S/7eSSJaLQdoeOyPALwJ4BUhxJdCiHNCiP0A7gJwFsA/5VXHe6Qd8vtp\nQogj8ra+JaL9cglrHhE19fZH8HZrLN/l1JNfxxPRG0S0Uy4djiWiRPmzdCL6RS6tHSWi+UTE+c3C\nhBB7AEwH4C3/vUBEExTvc+W8ECO/v42Itskl9u1EdJOP3bwAYKEQYqQQ4qgQ4pQQYgyACQD+La8z\nHsC1RKQsZPWFFK+my/sbTkRb5f2tI6Krve3MM53yMs/z8Q4iWk9Ex4hoJhHVkperOjfthE9A7xYB\nqEREjeWSxRBIGdKf6gBSAGQCuBPA+0SUqmJffQBcBqCB/P3rARzxsl5DADkAvlUuFEKUAPhe3g4g\nXawuI6JsAJCD7I1wLxlNB1AfQAaAZQAmqkinN6PldLcEUA/SsT8nf/Y4pLuNqgCqAXgGAE/cZGFy\nnukPYHmQ36sAYAyAy4UQyQA6AljhY/Xe8MjDskkAuhBRghDiLwD7AFyj+PxmAF8KIYrk91sBdIF0\nzrwIYAIR1Qgm3XLaB0HKm9dAyqvzUXYnrPbctA0O+L65Svm9AawHsCfA+oUAXhJCFAohpgE4DSlI\nB1IIIBlAIwAkhFgvhNjnZT1Xm4K3z/ZByqwQQuwCMBfSCQIAPQHEA5jqWlkI8alcsroAqcTVgohS\nVKS1lHzHcQ+AR10lNQCvQbo4uo6rBoBa8t9kvuCZ+qxqChEdB7AAwB+QfsdglQBoRkSJQoh9Qoi1\nPtZLh+88HA0gTX7/OeQ7VSKqBGAQFIUWIcS3Qoi9QogSIcQ3ADYDaBtCuu8D8C/5vCuCdOwt5VK+\n2nPTNjjg+/YFpJLxbfCozvHhiKL0AUjVLAEbv4QQvwF4D8D7AA4S0Tg5g3s6LP/vrRRTQ/E5IJ0Y\nroB/M4CvhRCFgFQXSkSj5dvhkwAK5PXcGqlVqAqpXWOpXG1zHMAMeTkAvA5gC4Bf5Vv94UFun4XP\nVUKIykKIWkKIB4QQ54L5shDiDIB/QAqe+4hoKhE18rH6YfjOwwJlJegvAHQnopqQ2qS2CiFK7zyI\n6BYiWqHIe80QfB4GgFoA3lFs5ygAApAZxLlpGxzwfRBC7IDUeNsfwA8aN3cG7o2+1T32NUYI0QZS\nw2oDAE962cZGSFUkg5UL5SqbayGV6l1+AJBFRN0h3aoqq3NuhFRa6gXpNjXXtalA6SYiZboPAzgH\noKkcLCoLIVLkxj/IdxCPCyHqALgSwGNE1NPLPpg9BMrDM4UQvSEF7g0APvKxndnwyMOy6wEsku86\nXefffABDIRVaSvOwXPr+CMCDAKoIISoDWAPfeRh+0r4LwL2KPFxZCJEoVyupPTdtgwO+f3cC6CGX\nYLRYAeAaIkqSGzzvdH1ARJcSUTsiioWUOc9Duj12I1eHPAFgFBHdSFL3teoAPoZUsnlXse4ZAN8B\n+B+AHUKIfMWmkgFcgFSSSoL/2/eVAJoSUUsiSoBU/ePaRwmkk+4tIsqQjyWTiPrKrwcSUT256ucE\ngGJvx8VsYwWktqEcufpvhOsDIqpGRIPkuvwLkKozff3WLwLoSESvElEaESUT0UMAbkdZ+4/LeEhB\nvRPc25kqQLobOCTv/3Z4aWgGACHEIUjVsUPlu9s7ANRVrDIWwAhXxwUiSiGiwfJrVeemnXDA90MI\nsdUjWIbqLUjdJw9AysTKzFsJUuA8BmAHpED8uo/0fAOptPMopFvPfZC60XX1Urc4HtLtqmd11Ofy\nfvYAWAepgdorIcQmAC9BKpVthlTHq/Q0pGqbRXL10GyUtVvUl9+fBrAQwAdCiN997YtZmxBiFqTu\nkqsALAXwi+LjKACPAdgLKV92BXC/j+1sBtAZQAtI1YnHAbwM4GohxGyP1b+HVKc/R5m/hRDrIPVY\nWwjpnGoOqaunL3dDKpkfgdSV+S/FtiZD6h30tZyH1wC4XP5Y9blpF8TtaPZFRH0g9cvvJYTw1SuC\nMcsioixIhY7nhRCfmJ2eSMclfBsTQvwK6Va4vdlpYSwUQojdkErUNcjgEb6MS/iMMeYYXMJnjDGH\niAm8Svikp6eL3Nxcs5PBItTSpUsPCyGqBl5Tf5y3mZHU5m1LBfzc3Fzk5+vRKYax8ohoh1n75rzN\njKQ2b3OVDmOMOQQHfMYYcwgO+Iwx5hAc8BljzCE44DPGmENwwGeMMYfggM8YYw7BAZ8xC9hz/Bx+\n33DQ7GSwCMcBnzELGDBmPm7/bInZyWARjgM+YxZw/Gyh2UlgDmCLgP/FwgLkDp+KwmJbP2yGMcZM\nZYuA/58ZGwEA5wqLTU4JY4zZly0CPmORTHnn+umC7eBnVDCjcMBnzGQNRk0vff3SL+uwYtdxE1PD\nIpktAj6Xd1gk8yzQFxZzjmfGsEXAdyGzE8BYmExevht/bT1sdjJYhLHUA1AYc5oDJ897Xf7oNysB\nAAWjB4QzOSzC2aqEz1ik6fnmH+WWEd/KMoPYIuCb2WtBCIEFmw+jpITrVZn+Tl8oMjsJzEFsEfDP\nXJT637/566aw7nfZzmMYMm4Rhn6yGJ/9VRDWfTPGmN5sEfBdwh10r/ngLyzefhQAsGq3765yQgj8\nvvEgHvl6OYr5ToAx21i+8xg2HzhldjLCxlYB30xTVux1e19SIvDl4p24WFSCsX9sw+3/W4IpK/Zi\n3uZDJqXQOFsOnsLzP67hai2mu30nzqHg8BnT9n/1B3+h91vzTNt/uHHAD9FPK/fimcmr0WDUdLwz\np6yq6fb/LcEzk1cH/P7vGw5i5tr9RiZRN7d88jfGL9yBnUfPmp0UptHbszchv+AoNh04hemr96n6\nzpo9J3D87EUAwPGzF/Hzyr0BvqFeh3/9hm5vzNVte8HYcvC0Kfs1k+0C/vTV+8o14uYOn4oXflqr\n634+mLul3LLZ6w4AAK56/0/8Z8aG0uXnC90ndfty8c5y333i25XIHT4Vx85IJ87tny3BvV8sDSpN\n2w6dxpo9J4L6jh72npC6DnLvEft7e/ZmXDd2Ifq8NQ/3T1wGABjxw2pc+d4Cn98Z+O4CXPnenxg1\nZTVu/d8SPPTVcuw8EtrF/8yFImz3UqL/dMF2PPrNipC2GaoLRWVzc63YdRy5w6di4/5TGDVlNU5Y\naPbSkhKBP7foMybDdgH//onL8PnCHaXvj5y+AECq39ezysE1YZvSXZ/n461Zm7Bi1/HSIOhLkWJ+\nlKLiEny3dDcAYOuh0EsVPd78AwPf9X1issigvK6e0aEXT8HhM2j/2hzsO3HO6+df/b0Tq3afwJg5\nm3Hg5Hk8KRdOlHYePYsJi3ZipTztw/mi0CYyvPmTxejupUT/0i/rMHn5nqC3d+JcIfJemY1lO48B\nkErtoVQRvfebVMDr+/Y8TFi0E/+dVf78N8vHC7bhpo8XY876A5q3ZbuADwDzNpXVk7d5ZXbp6xd/\n1qeU78o83rwzZ7OqbdQbWTY/yru/ld0tXDd2IV6fWXZ3EOoJvenAKazbezKk7yodO3PRZxoKi0vw\nbf4ut/csvJo+P1PzNiYs2oH9J8+jl5c+/61fnlX6+r+zNqHda3PwrVw4CcXpC0XIe2U2Fm49Uu6z\nBZsPY9lOfecJWrrjKA6fvoAx8nnZ679/qKoiKiouwd7jZYW22R7B1ErNVdsPS3dT+30M0guGLUfa\nzpEfBbfJo3V9/MIdeHFQM83bv+aDvzRvAwA2HzjltUHo/d+3lr4+X1iMCvFlP8NPK/di1a7jGDWw\nidt3XHcyLn3k7QY7EvN8YTGGfrwYTWpWQmJsND6ct83ndsbM2ex2sZq59gDqZSSXvv90wXa0yE5B\nm1ppQaXh6JmLKCwuQbVKCUF9j2nj6t6sdFSuYvTmjZkbUTejgtfPXLWqC7cewcnzUvXHiXOF+H7p\nbhw+fQFv/roR393f0e07t/3v74BpPF9YjPOFxaicFBdwXQAguZ4x0FCdb/N3oUPdKshKTQIA3PDR\nIiwp8F2wExaYwWvB5sNIio/GV3+XryIOlS0DvsuQcYt03+aAMfN129bbswPfDbw2bQPevL5F6fuH\nv1oOAOUC/lRFA5uyVKbGo9+swKCWNXGhqATnC4uRv+MY8ne4Z/aLRSUoEQIlQiApLgZTlu9xC/YA\n8PrMjRjWvR6enbIGWw6exsJtUiku2IuOK/1LR/XCF4t24KEe9REdxQ0EejtxrhDPTF6NpNjokL7/\n3u/l27FcNuw/ieopCbjhI+/n4NZDp7H5wCnUr5bs9XNfbvxoEZbtPI4xN7TClS1qBlzflWtKvET8\nU+cLsaTgKDrWTceT361CZuVE/Dm8B2au3e832APAhEU78cpVzYNKu57yC45i6CeL3ZbpMf7U1gHf\nW+nkxNlCpCTFhrzNtTpUk7hMVdEL4vtlu5FWIRYjB7gH+P0nzqN6SlkJ+Lkfy6qrlMd99mIRkuL8\n/4yTl+8JWD/a75152HZIqvt8/oom+Fsef+Dpuv/7q9zFApBKZglBBhZXdVxKYixu71Q7qO+6CCHw\n+Lcr8cOyPaqDhFOM/6sAU1ep64kTrH9+7b+B9djZQvR+ax6+urs9OtStgoVbj6BIRT2Jq8rn4a+W\nq/oto+QS/l9bj5Rrd2j+wq8AgFmPXgYAOCzfJavtLDFpyS7UqpKEdnWqlC77aN42vDptPTa9cjni\nYoyrEf9xRfmeUKOmrMHQ9rU0bdeWdfj+HDilvZ4r3D6avx1Ldxx1a2xq/685+GnlXuQOn4px87b6\n/O7gsQvx7JQ1KDh8Bg9MXBryCe4K9gDw4s/rMH2N9y6j3oL9n1sOo9GzMzBh0Q50Gv1bueonpb3H\nyzccvvjzuhBSLJm/+TB+WCZdzD78Y6su7RpMP7+skgKXrzuBQB76arnfc8DVc8xzwKMy+Lu6kV4o\nKnHrmRPIU9+vwj8UtQgb9p/Eq9PWA0BpNdaaPSfcCkdCCJzX4cl8xQZNJxNxAd+urv2/heVK1a7q\nndembfD2FQDSHckXi3ag2xtzMW31fgz7clnpZ0XFJegRhj7Oz/64BoBUAtlz/Bx+23AQ/zd3K5o8\nNwNj5mx2O67O//4t6O3PWX8A4z1GWR87cxFbD53GLZ+W1Quv3XsS/XWskjPLvgA9wOxk4uKdPjsF\nBBqVfuV7C0qD9WvTNiB3+NTSXkIuUSr6Co9RVE02HDUj4Pq+9Hu7LG89Jw9EHPjuAlz/4cLS5R/M\n3YpGz87An1sOY7mfzh+exs3bitzhU7FmzwmcvViE1buN6X5t2yqdX3UetLSk4Chy0pJ03Wawnvp+\nlS7byR0+FfOf6o6E2GhsC8MoRuXdAQA8+V3Zcfx3ljQo7el+jXB/t7pB9344eb4Qd47PBwDc2jEX\ngFSqGvjuAjSqHlz9sF08JF/otThy+kLp395sb8/2no77J/ivWlnlJehNXLwDLbIrY+qqfW6Fm3Cb\ntno/6qyeVm75FLnq9KaPpfr3/FG9cP+EpXj/xtbI8NNJ4V/TpULdwHcX4KZ2OVht0HgbXUr4RPQp\nER0kojWKZWlENIuINsv/p+qxL5cRPwQezRqMwfJglEhx9+f5XqtPzPLvGRtw0E+3stzhU1FnxFQ8\nO2WN2/LPFSX7e7/IR+7wqXjiW2mu+A37nTMHSrDmb7bOw1M+mr+93LKhHy/Gr+uC71c+KX83LhQV\nhzXYF5cInDofeCBW7vCp2Owxevfrv3diScExjF9YAEAa87BA8dsMm7gM3yzZ6dYgu9hH+5ke9KrS\n+QxAP49lwwHMEULUBzBHfq+bI366k4XqxDnrjK7TasP+Uxj0/p9mJ8NN29fm+P28RABfLJIG1ZWU\nCBSXCLyhmCF15lopQHCgt78FGkaOnvPSvdRIdZ+ZVtoA7MvXPrpObjrgfgEY8cNqt943U1fvw9Pf\nuxdejSyo6VKlI4SYR0S5HosHAegmvx4PYC6Ap/XYnz/zNh1CgyC6gp0vLMaHf2wzMEUsWMMmLlPV\nw4m5KywuwbGzF3HuYjH+NX292ckxTMuXguuWHA7DfdQ4/ORj3qET5woR46Mr8lkDL2hGNtpWE0K4\nztr9AKp5W4mI7iGifCLKP3RI+0yTrkaenm/OxZBxCwOsDYybtw1v+ahjZOaIlGCvV95W+2zbZ6es\nQdtX56Dr63Nx4KTvnlIs/L7+exdmrCnL1y1e/BWfLihf1WW0sPTSEdJsZ16b64QQ44QQeUKIvKpV\nq2re18rdJzB73QFsPXQGi7YFrgs7p0MXKsa80Stv3/jRYjz13UrsCjBbaSh14iw8jpy5iPsmuLc7\nvGlCo7qRAf8AEdUAAPn/gwbuy03BEfPm12bMCJPyd+NpnXpxMecyMuD/BOBW+fWtAH40cF9uLhSp\nn+SLB/QzxpxCr26ZXwFYCKAhEe0mojsBjAbQm4g2A+glvw+L12daZ2pTFn6R1NtKSQipH7pnF8Hi\nEoHc4VP9ToTGGKBfL50bfHzUU4/ta7Fmzwk0y0zx+tnnCwvwwVzf0xYwe9p26DRa5eg67MMSlu48\nhoXbjmDk5DWY/VhX1MuoCAAoKuFpq5k6ET+1gudj+e7+PB/5BVJjrnJCMsas7qKiqnLFLn3nlWfO\nEPEBX2nd3pOYte4ArhsbuLsmsy/zZzI3nuecMoypYdu5dNRavvMYhn25DN0bZqCGYrrhT0zoA8uY\nXr5YtAPRUYQXrmxqdlKYjUR8Cf+XVfsgBPDbBvdeoS//EvqUvIxZwWd/FaCouATEfc2YShFfwldO\nNbt+H8+V7gQGTSVuSeMX7sBsHnDFVIr4Er6S3g9QZlblnIh//OzF0kdNMhaIowI+Y5HG87nDjPnD\nAZ9FHCdV6TAWDA74jDHmEBzwWcThAj5j3nHAZxHHLlU6wi4JZRGDAz5jjDkEB3wWcbjkzJh3HPAZ\nY8whOOAzxphDcMBnjDGH4IDPIk5cDGdrxrzhM4NFnIxKCYFXYsyBOOAzxphDcMBnEccu3TJtkkwW\nQTjgM8aYQ3DAZ4wxh+CAzxhjDsEBnzHGHIIDPmOMOQQHfBZxuPcLY95xwGeMMYfggM+YSfhGhIUb\nB3zGTFJcwiGfhRcHfMZMIriMz8KMAz5jjDkEB3zGGHMIDviMMeYQHPAZY45WrVK82UkIGw74jDFH\ne6pvI7OTEDaGB3wi6kdEG4loCxENN3p/jPFIWxaMy5tXNzsJYWNowCeiaADvA7gcQBMANxBREyP3\nyRhjwSCQ2UkIG6NL+G0BbBFCbBNCXATwNYBBBu+TMcZUI+fEe8MDfiaAXYr3u+VlpYjoHiLKJ6L8\nQ4cOGZwcxsKH8zazGtMbbYUQ44QQeUKIvKpVq5qdHMZ0EyhvRzupaMksweiAvwdAtuJ9lryMMceL\niTa9vMUcxugctwRAfSKqTURxAIYA+MngfTKH4zlqGPMuxsiNCyGKiOhBADMBRAP4VAix1sh9MsYY\n887QgA8AQohpAKYZvR/GGGP+cSUiY4w5BAd8xpijOamzFAd8FnF4agXGvOOAzxiztJva5ZidhIjB\nAZ8xZmkvDWpm6PbtMpdO29ppmrfBAZ8x5nhv/aOF2UkICw74jDFLC0f5OzUpLgx70UiHtikO+Czi\ncJstCxY5pKsOB3zGmKM5JNYD4IDPGLM4JwVko3HAZ4wxh+CAzxhzPDvcROgxCywHfMaYpRndoGqH\nYK8XDvgs4gieW4ExrzjgW9izA5uYnQTGdNWrcYbZSfDKKQ3DHPAt7M7OtVWvO/CSGgamhDHt+jSp\n5pj+7kbQ48aVA36E6FC3itlJYMwvq8Z6IkLD6slmJyMsOOBbQHSURc8ExhwiIzkBBaMH4OcHO5ud\nFENxwLeA6f/sonkbdpnxLxy4ydaahFBXLZEcb/iTV32qEB9t2r7DgQN+mLTzM7VpRRMzOGNGio+x\nV4iJ9DYGe/0aNvZIrwY+P4vwPMYcLJS8He47NLucfnr8XTjgh4m/jO/vNvfvkT3RoFpFTdvXQ05a\nEro2qGrsTlhEU5tHeRyFcTjgW1xGcgJeu7o5qlTwP193BY3VQvUy/F9UEmOjMf6Otpr24U9Gcrxh\n22bW4BnHr22dZU5CHIwDvgVVq+Qe/PJy07D02d4+13/lqmYY2FxbP/zoAMUvO1U72bmAeEWLmmYn\nQVf+OhPc1UX9OBO1qhpQcKiZkqD7Ns3CrYVhEky8/O3xbrhYVKJ6/aHtawWfIA96TMzEWCAvXNkE\nmw+ewrWts9C4RiWv6yhzYmblxKC2v2RkL+QOnxrUd5SFmRgvXaSjo21U2gmAS/g6iYvW709ZIT4G\nqQGqcFhksmL9dXZacEFXyfPOMCs1CX882R0P96wPwHsXTNef4LaOufhzeI+Q9x2K7LSkcsv06vKs\n9bm5euQNDvg6iY8N/U9ZpaL5wT1QprZgHGJhkp1aPgiqpcxVdqoWVKpVJfTjV8pJq4Au9dN12Vao\nOODrRUNAjI8JfrBHIw1DwZ/u18hy9ZJ2DQbMP+VpYdVCQ6C+995K/XbFAV+FZpne6xqDofeAjp8f\nCn0I+G0dc0NOz+A2duhZYdHI4kC+6un9sVp7UkIIBTKrsn3Af7y37wFN4fD9/R1N2W+sjm0GwXh9\nsLZ6SF94aojIlBgbfLB03QlY5a6vbe1UTd+vH6DLs1qOGXg18a52eOEK97nhZz92GaYM64Q4mwzd\nTk2KNTsJfpl9cpm9f7NMvKsdfnjAnEJDOFittK7G2KGt3d63rqUt4FuJLaJlp3rpuK2Te5/dehnJ\naJld2aQUKanL0PWrWWv6Vc8Aa9X61UjXqV46LslMMTsZfmm5GIeSr8zOiv2auY9pyUjWq73L7COz\nScA3m79MW/pZGEuoH9+Sp/s2gymJzX2iG65qGVkDhKwiMzUR93atY3YydFOrSoXgv2R+XIxYtg/4\nlqnSCWMm1XrMWqtPctMrIDGOx+zpRZl1CIQRlzc2LS16CzRlhz+R165j/vFYJFqG7sZ2ORjWva5p\n+7dr3bNnugOdXEYfp56b5+opm5Mzgx3r//3TdjyOe8RhnybVyi2Lj4nGk30bmZAaSTA/wsIRvkcN\nfnKr/tU0ejI6iEb6POShStAwoM8KQhodGmlx3kI05SYiGkxEa4mohIjyPD4bQURbiGgjEfXVlkzJ\nOAPqrsOpRkqiz1vcno3LX8x8MSI2Btpm+dIWn5VGi4kiLH6ml9nJME3kVemYT2tF7BoA1wD4ULmQ\niJoAGAKgKYCaAGYTUQMhRLHG/dlenybVsOXgaU3bMKK0beVqkMzKidhz/JzZyQgL14V19Qt9EEWE\nhBD6sevNyMCblZaE9ftOGrZ95k5TCV8IsV4IsdHLR4MAfC2EuCCE2A5gCwDjJlM3mN9eOuFLhq60\nn8ThK30lxpkf9IzkLX8lJ8RqfsaBHXx+R1uvVbWRRK87cisPvMoEsEvxfre8rBwiuoeI8oko/9Ch\nQwYlx3hqfwyrXiACZUozb6+D3bNV/sZq83bkNU6qVzU5Hn2bVndb5uS/h9ECBnwimk1Ea7z8G6RH\nAoQQ44QQeUKIvKpVrfkIPX/BMOhgpENeNqN9k9tUg6c2b+s5tXYksMLUCvOe7G7ezg0U8J5RCBFK\nq9EeANmK91nyMs1GDWhsqdnrQh13lRgbjXOFoTVpmHHR8LXPK1rUxM8r92pPkL99G7p18xERnu7X\nCP+escHspDBZjk5TIluNUUWLnwAMIaJ4IqoNoD6Av/XY8F1d6pS7BTRTsME30m5X29dJ02U7fAdh\nXeH+bSI12FqhY4TWbplXE9FuAB0ATCWimQAghFgLYBKAdQBmABhmtR46U4Z1Ur2u/6kVXPef6rbl\nmpejv4Zn0OpxAgZfLx44t2alhv5kJKdz1eoEerawGcIZqL6+pz2GXJodeEUn0uGH0NQNQAgxGcBk\nH5+9CuBVLds3kt6nldrt3dYxFxnJ8RjQvAa+X7Zb51SoZ8RAp8FtsvHW7E26bzfYwTtWKEkF6+b2\nudh7/Dwe6F7P7KToSpnPujb03o6hzIrt61TB6t0npOWGpsyZuLVIo2BjS3QU4YoWNRElPyxZWSUy\n4c52+PKudjqmzjetz8d8uGc9dKmfjitalE2idlWr0CdU89swbsFSr94S46LxwpVNUdGCXTHjQ5y7\n6eEe9XBj25zS98rX3lzdyr0jn5Wu21rmr7JSAcR6uStMor08nT7c1r/UDzHRZenoHOB5ly9f1QzP\nTlmDXHkGwsY1KqG5Dk/jCkWNlER8cWd4Lk7m/1LOlJWaiN3HziGjUmjTA2enJbmNoVB74bbk9V1D\n0LbS8Tgu4HdrWBWd66WjaU19AqWWq3ewA4qGtsvBta0zkSTPVDn9n138rt+9YVU82rsBrnzvz3Kf\neZ58TihFs+CkJMZi97FzYQ9YVioR68FKU0Q4LuCnVYjDXV30m2/c1ZgZjoBJRKXBXo3a6RVxSZY+\nD4kxOtPvGT5IAAATrUlEQVT6236Enf+OM/2fXXDk9MWgv2edMGkNepwHjgv4ofD7h+ZopAu+wYg8\nrlMjlAeZW06E5E9utNWJlQLWqAHBP0DDQsnXLNLGOgSrucUfmRiIVX6/UQMao39z/cb8WOGonBfw\ndf6rx8vzlXeqm26rx/55Bvjgp0cOzsQw9T5iQC2dBy6ZVadudiHqri518MFNbXTdptltZc4L+DpL\niI3G7090w5vXt8DbQ1qZnRz1dM53WoOMMjlVk+N9fsbCR3NsskKRVidvDm4R8ndL2/k0psFxT7wy\nU3ZaIhpVT/b6We30CpaYt9wsvzzUGVMeUD9yOZAlI5370A9PfLGzBl+DxoJhheuf4wJ+qH/0+U/1\nwLMDm5TfnhV+xRDoGUiaZaYgtUKcjltkLlFRhJmPXGZ2MsLKrueUL9wt02a0jkq1GqLwlzYC/Qn9\n1W0Gm9YI+7nQ0MedpafstERUSog1ODXhY3Z9dyRyXAnfCVkoTS5tu+rCczymkw4lIBrfD59pNf+p\nHpj6cBcOlBZjpZ8j4gN+QmwUYhTTKIwMocuii4V+N7+ubpWJd4a0xN1dagMAvr2vQ8Dv3NKhltHJ\n8svfNSjS7rCYf53qSVOM9GiUYXJKrEWP7qoRH/BXPt8Ha17sW/q+SsV4P2tHBiLCoJaZiJHn3K1W\nKQGVk/zf6jeq7n9wjNbMZqVSDgtOoN/usgbaGjSzUqU70MY1pKqrZpkpKBg9AO3rVAlpezVTQpv7\nx59IKXNEfMCPj4l2TA8aX72I1AjYD19Fhg9m2odgcBWFtY27uQ3+Gt4j5O+3rZ2Gnx/sjLs6B57y\n5I8nuwVcp3bVCiGnJdJFfMBXo1lmJcx9opvZydBs0n0d8NvjXQOuR2RM9ZRn//lg6JmefSfO6bg1\na3qiT4Ow7SvQxT4hNho1K5d/+E0wd4XNs1JKpwz3p1aVwMHcSr1iPGlJ2Zo9JzXvn3vpAHiiT0Pk\npqsoFXj5tax0q1cpIdawXhpmFrKDrcM/cPKCQSmxDs+7njQDusVaOXCGmx75Xwjz++JzCd8hkhMs\nfm3n2KKboe38P2iElbn3Mv1mzrUDDvgOEai0Fujz6in8vForU/567UJs7HSiuhkVVa2n5U7eSrUA\nHPChrVHQKjP7BeJZZeUt1elyDyZvj7QzugDOBXz/7u9W1+wkBM1Kgc4KrND3IGICvpYnFrq6gwVi\n5zrN925shaHtc9CpXhXERUd5jfjpFaV64Mk6zoujB+6lw8HTzlzZ1wq/ocUrdtVbOKInjp29iH5v\nzw/6uxnJ+vfbtZpKCbF45armpe+DzXtmxlwO9/bE12nriZiAX61SAqqF+LDlQPxdma1w1Q6F50Cs\nmpWNvejl5aYauv1I563qcOrDnXHibCFu/HixCSlidhQxVTpWlZEcj7f/0dLsZJTz4dCyBztse62/\n2whkb8FF64UtPib0wW82vaYaQllqblozBfWrJZdbrrdQR66aVRiyS7uaGRwf8FMS1fdbD+Wkap6Z\ngqtaZQb/RYNlyHdDHetWKR3w4q2uvFfj4OYzuSbEY+V6ev/i5WkyYqPUnbJvqHxgxyO96vv8zPWT\nXNagKu7rqr7R+JrW5uT3lwY1DbjOBze1dnvfIcw9mszO5Y4O+JPu7YBfH3XWXONK85/qjk9uvbT0\nvbcBTk2CfAD1HZ1ra06XJ548DbivW13c360ublY5yd11bbJUrXd7x9rY8HK/gOu1q52manuA+ouS\n3m7pkAvA/51FrSpJaJYp5elh3esiO0374yAn3Rt4ckKriLiA/+OwTqrn9WhbO01zvb+dQ1F2WhIS\n48pXtSh7IwV7fIEmaWOhSYqLwdP9GqFNLaktxNWzTI/qCzVzTUVKNYkQQL+mITyY3M/h1w0wd4+V\nyisR02jr0iK7smHbblgt9MnJ7OCKFjWxYf9GZFQqPyeO2hoX18yHeuLqnjL9m9fAvCe7I6fcM4SN\n/RuFErTMinPhDrAWiucBRVwJ3wiu0k1qhTgUjB5gcmqM80C3ulj7Yl+kV4xHqoaSur96YRfPh7Iw\n9coHe2sx+/psyN2IhmMy+++hxAFfo0iqXyYiVIiXbvqeu0J6fm+uitkJQzHvqe5u7/39HSPpb2yW\nrFRtU2Oo+QmGXJpdWj9uBXHRUbi2dfm2jLxcqT2iXW19GmwtFM8D4oDvhbepBZzm6lZZKBg9wJBZ\nGFl4fXb7pZj6UBdN28hKC3zBGH3tJfhF43704HqYz48PdsKb15fvrdS+ThWse6mv5ge32FHE1eFb\nRUwU4apWmXisd/jmLbczrqcPkYqSd7eGUtfax3o3wH9nbVK96Yry3V50FKFR9UqoEBeNMxeLA37v\nsT4NcOp8Ea5sUVP1vvT0TP/G6N+8Bhr76WEW9MN6VN5kVkqIQVJcDPafPB/c9sOEi7IKPz/YGQ92\nrxfUd3zlg9joKLwxuIXXB0MwpjcjrpdvD2mJ4Zc3QtOaUuBU8/ARQJqq5P2bWpdWD4ZbXEwU2iq6\nkepx4VHb0zQrNQmLnunp83OzyzUc8BWaZ6Xgib4Nyy0PpQq5Z5ADlhizmozkBNzXta7t7770qLpJ\nTojFLT7GQNiphYkDvs4GtayJLvXTvdYdstDYPeDYgZrxE22DGHxlBD2yQaqGNqnr87IDrmP14K8p\n4BPR60S0gYhWEdFkIqqs+GwEEW0hoo1E1Fd7UsPn0lz1Gduz9P/OkFb44s52muaOiVQ/P9jZ7CRE\nHNfFMCFWW9ltycheWP5sbywa4bs6YuSAxpj9WOBnJhvhy7vbYd6T3QOvGECmTlWsQ9uH9lSxUDuc\nVYyP0adqSuP3ZwFoJoS4BMAmACMAgIiaABgCoCmAfgA+ICLbRMAPb27j93PuxROa5lkpAde5oW35\nEynYbplO6sVZNTkeT/ZtiAl3ttO0ndjoKKRWiEN1PxOlxUZHoV5GxdLG3HDqWDfd6zQIb6qYMyhR\nxUjiYMVFh7bNULPmE30aYMwNrUL8dhlNkUsI8asQokh+uwiAq9PrIABfCyEuCCG2A9gCoK2WfYVT\nhfgY5I/q5fPzJaN6oZbFB7/oRY9BLAtHBJ7qggBsefVyvHZ1M037uq5NlmmTd5llWPd6PhtUW+WU\njTzXq9fMtIe74L0btQcfrXLSknCtijmDLm8mTaWQXrH8CHI9BFPTZPaYEj0v1XcA+EZ+nQnpAuCy\nW15mmugoQnGJ+j+2v8xRKSEW1SolYMeRs6a3uoeLlsNUWyKMidZ+56R2lkgnWPdSX8Qoupd4PuYy\nVDlVkpBTJQlHTl9Euzrm1uurERVF2Ppaf8MGSCmjSjj2oUXAM4yIZhPRGi//BinWGQmgCMDEYBNA\nRPcQUT4R5R86dCjYr/ulnFcnf2QvVSVNb7yV5t8Z0hL3dq2DllnGzd1jJaFkuKHtc/DxLXlITihr\nEGybm4Zq8lw9n96Wp2o7t3bMDWHv5jMyb6uRFBeDOI/qx9HXNEffptV02f6tHXNLBzmZIZjpOaKj\nqHQacCN5O09GDmiMRtWT0SyzkunVjQGLXkII33UbAIjoNgADAfQUZfcrewAom7Sz5GXetj8OwDgA\nyMvL0+3Pse6lvohVlBhDaZ3PH9UL8zcfQq/G5U+QGimJGHF5Y01ptAMtz/FNS4pDrybuf7tJ95VN\nJVs7vaKq7VQIdpCMRRiVt7UY0jYHqRXiMHPtAbOTotn7N7YOvFIYBAriLbIrY8Yj0jTs2SpGLBtJ\nay+dfgCeAnClEOKs4qOfAAwhongiqg2gPoC/tewrWElxMW4BPxTpFeNxdasstxIqC0KA+q7cKkkY\neEkNr58VjB6Aq+WHqTil2ixc+jTxX8L/5p72mPxAxzClJnQpFpmKO1A7lzL7Pn9FU4wdat6FSmul\n6XsAkgHMIqIVRDQWAIQQawFMArAOwAwAw4QQgcdkM8sxch50IsLDPQPPrMn0RURl1SFeLqbt6lRB\nqxzrPoP49esuwRUmTNsQah9+5RmUEBuNfs28F3LCQdO9shDC5zwEQohXAbyqZfvMOgxrjPJzPbm5\nQy1MXr4HHeumG7R355p4VzvM3XgwqEd8WsXgvGwMVjEISm+ZlRPxj7xsfJO/CwAwdmgb6eEnijxs\ndi+cQLhDOTOV6w7CW7VN65xUFIweUK5vePeGzpvlUG/ZaUm4WX4kIFOvfrWydqd+zaqXPkTeLjjg\nM91d0yoLlRJicG0Q/eHVNg5vefVytA/zg6cZ86dKxfjSbtxWnwaEAz7TXU6VJKx6oa+q2RVdQ93v\n6uL/4eeP926ASfd20KWvPmOhcj0fompy2Tid6CjC+DsuNXS/etUU2bO/GwubivFSHa+/IfdaJCfE\nqnps5EPcuMss4OpWmYiOIgxo7r/hVe2zA8KNAz7zq23tNLwzpCX6NKludlIYMx0RYVBL31WVVm+0\n5YDPAvKXwRkz25gbWiHd5EdxahmgGE5cIcoYs7UrW9REx3rW6ro7/o6yuSKtdCngEj5jjGnkOUAx\nLzcNC57ujrkbD3kdqf947wY4daEI4+ZtC1cSAXDAZ4wx3Si7ZWalJmFoe++PRXR1Qgh3wOcqHcYY\n04nVG2054DPbycu17jwvzJm40ZYxg7SplYb1L/UzOxmMhU3YHoDCmBUlxtnmEcmMWQYHfBYx/nPd\nJWYngTFD6FVhxAGf2da21/qXvi4YPQDXmzBlLmMAUKdqBdTLqIjnr2hqdlL84m6ZzLbC8YxSxtRI\niI3G7Me6GrZ9vSbh5BI+Y4w5BAd8xhhzCK7SYbb23MAmaFs7zexkMGYovSovOeAzW7ujs/8HpzDG\nynCVDmOMWcy8J7u7vW+n02M9OeAzxphJVr/Qx+vynCpJpa9/e7wrGteopMv+OOAzxphJvE2d7KlO\n1Yq67Y8DPmOMOQQHfMYYcwgO+IwxZqIPbmodtn1xwGeMMRP1b14jbPvigM8YYw7BAZ8xxkw2uE1W\n6esZj3QxbD8c8BljzGT3dq0DAPjoljw0qq5Pn3tveGoFxhgzWb2MZBSMHmD4friEzxhjDsElfMYY\ns6Dv7++ALQdP67pNDviMMWZBbWqloU0tfaf+5iodxhhzCA74jDHmEJoCPhG9TESriGgFEf1KRDXl\n5UREY4hoi/x5+MYOM8YY80prCf91IcQlQoiWAH4B8Jy8/HIA9eV/9wD4P437YYwxppGmgC+EOKl4\nWwGAkF8PAvC5kCwCUJmIwjdhBGOMsXI099IholcB3ALgBADXc7kyAexSrLZbXrbPy/fvgXQXgJyc\nHK3JYcwyOG8zqwlYwiei2US0xsu/QQAghBgphMgGMBHAg8EmQAgxTgiRJ4TIq1q1avBHwJhFcd5m\nVhOwhC+E6KVyWxMBTAPwPIA9ALIVn2XJyxhjjJmEhBCB1/L1ZaL6QojN8uuHAHQVQlxHRAMglfb7\nA2gHYIwQoq2K7R0CsMPHx+kADoecWGvjYwuPWkIIU4ranLcjkpWOTVXe1lqHP5qIGgIogZSZ75OX\nT4MU7LcAOAvgdjUb85dgIsoXQuRpS6418bFFPs7bkceOx6Yp4AshrvWxXAAYpmXbjDHG9MUjbRlj\nzCHsFPDHmZ0AA/GxOVsk/4342CxEU6MtY4wx+7BTCZ8xxpgGHPAZY8whbBHwiagfEW2UZ98cbnZ6\nvCGibCL6nYjWEdFaIvqnvDyNiGYR0Wb5/1R5uc8ZRYnoVnn9zUR0q2J5GyJaLX9nDBFRmI8xmoiW\nE9Ev8vvaRLRYTs83RBQnL4+X32+RP89VbGOEvHwjEfVVLLf8b6w3uxwz5+0IyttCCEv/AxANYCuA\nOgDiAKwE0MTsdHlJZw0AreXXyQA2AWgC4D8AhsvLhwP4t/y6P4DpAAhAewCL5eVpALbJ/6fKr1Pl\nz/6W1yX5u5eH+RgfA/AlgF/k95MADJFfjwVwv/z6AQBj5ddDAHwjv24i/37xAGrLv2u0XX5jJ+Zr\nOa2ctyMkb9uhhN8WwBYhxDYhxEUAX0OajdNShBD7hBDL5NenAKyHNGHcIADj5dXGA7hKfu1rRtG+\nAGYJIY4KIY4BmAWgn/xZJSHEIiHlsM8V2zIcEWUBGADgY/k9AegB4Dsfx+Y65u8A9JTXHwTgayHE\nBSHEdkgD89rCJr+xzmxzzJy3Iydv2yHg+5p507Lk27xWABYDqCaEcM0Suh9ANfm1r+Pyt3y3l+Xh\n8jaApyCNqgaAKgCOCyGKvKSn9Bjkz0/I6wd7zJHMlsfMedveedsOAd9WiKgigO8BPCLcnxcAufRi\nu36wRDQQwEEhxFKz08LMw3nb/uwQ8G0z8yYRxUI6ISYKIX6QFx+Qb1kh/39QXu7ruPwtz/KyPBw6\nAbiSiAog3ZL2APAOpFt11/QcyvSUHoP8eQqAIwj+mCOZrY6Z83aE5G2zGg/U/oM03882SA0hrkaP\npmany0s6CVLd49sey1+He8PWf+TXA+DesPW3vDwNwHZIjVqp8us0+TPPhq3+JhxnN5Q1bH0L94at\nB+TXw+DesDVJft0U7g1b2yA1atniN9b572ibY+a8HTl52/TMpPKH6A+pZ8BWACPNTo+PNHaGdEu7\nCsAK+V9/SPV7cwBsBjBbkcEJwPvyMa0GkKfY1h2QGn22ALhdsTwPwBr5O+9BHikd5uNUnhR15BN1\ni3yCxMvLE+T3W+TP6yi+P1JO/0YoemLY4Tc24G9pi2PmvB05eZunVmCMMYewQx0+Y4wxHXDAZ4wx\nh+CAzxhjDsEBnzHGHIIDPmOMOQQHfMYYcwgO+Iwx5hD/DwF/iY44D2IhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x241c297ec88>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "f, (ax1, ax2) = plt.subplots(1, 2, sharey=True)\n",
    "ax1.plot(minus_hist)\n",
    "ax1.set_title('Minus QValues')\n",
    "ax2.plot(plus_hist)\n",
    "ax2.set_title('Plus QValues')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Upload results on Gym OpenAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-08-26 17:33:59,265] [CartPole-v0] Uploading 801 episodes of training data\n",
      "[2017-08-26 17:34:02,189] [CartPole-v0] Uploading videos of 10 training episodes (75956 bytes)\n",
      "[2017-08-26 17:34:02,793] [CartPole-v0] Creating evaluation object from CartPole-record with learning curve and training video\n",
      "[2017-08-26 17:34:03,145] \n",
      "****************************************************\n",
      "You successfully uploaded your evaluation on CartPole-v0 to\n",
      "OpenAI Gym! You can find it at:\n",
      "\n",
      "    https://gym.openai.com/evaluations/eval_yii4hTCcRky6P5E2MIWihw\n",
      "\n",
      "****************************************************\n"
     ]
    }
   ],
   "source": [
    "gym.upload('CartPole-record', api_key='your_key')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}