Policy gradient code written by PyTorch where the number of batches is larger than one

policy_gradient_by_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from IPython.display import clear_output\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "from torch.distributions import Categorical, Bernoulli\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import gym\n",
    "\n",
    "from itertools import count\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorNet(nn.Module):\n",
    "    def __init__(self, state_size: int, action_size: int):\n",
    "        super(ActorNet, self).__init__()\n",
    "        self.fc1 = nn.Linear(state_size, 24)\n",
    "        self.fc2 = nn.Linear(24, 36)\n",
    "        #self.fc3 = nn.Linear(36, action_size)\n",
    "        self.fc3 = nn.Linear(36, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        \n",
    "        #prob = F.softmax(self.fc3(x), dim=-1)\n",
    "        #policy = Categorical(prob)\n",
    "        prob = torch.sigmoid(self.fc3(x))\n",
    "        policy = Bernoulli(prob)        \n",
    "        return policy\n",
    "    \n",
    "class CriticNet(nn.Module):\n",
    "    def __init__(self, state_size: int):\n",
    "        super(CriticNet, self).__init__()\n",
    "        #self.state_size = state_size\n",
    "        self.fc1 = nn.Linear(state_size, 128)\n",
    "        self.fc2 = nn.Linear(128, 256)\n",
    "        self.fc3 = nn.Linear(256, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        value = self.fc3(x)\n",
    "        return value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_discount_reward(reward_buff, gamma=0.99, normalize=True, done_flag=True):\n",
    "    prev_dr = 0\n",
    "    for ii in reversed(range(len(reward_buff))):\n",
    "        if done_flag and reward_buff[ii] == 0:\n",
    "            prev_dr = 0            \n",
    "        else:\n",
    "            reward_buff[ii] += prev_dr * gamma\n",
    "            prev_dr = reward_buff[ii]\n",
    "    \n",
    "    if normalize:\n",
    "        mean, std = np.mean(reward_buff), np.std(reward_buff)\n",
    "        for ii in range(len(reward_buff)):\n",
    "            reward_buff[ii] = (reward_buff[ii] - mean) / std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:1999\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO2deXxU5dXHfycrCWtIAoQlhCUiCAIaWRQUxYVFRWytIlVc3qKtttXaKmpbbasVfau21lctrlh3i1QrbohaF0QMOwjIFggQSdgjgZBkzvvH3Al3Jndm7szd75zv55NP7jz3ufeeeWbmd889z/Och5gZgiAIgr9Ic9oAQRAEwXxE3AVBEHyIiLsgCIIPEXEXBEHwISLugiAIPiTDaQMAoKCggEtKSpw2QxAEwVMsWbJkNzMXau1zhbiXlJSgvLzcaTMEQRA8BRFtjbZPwjKCIAg+RMRdEATBh4i4C4Ig+BARd0EQBB8i4i4IguBD4oo7EfUgoo+JaC0RrSGiXyrlHYloPhFtUP7nKeVERI8Q0UYiWklEJ1n9JgRBEIRw9HjujQBuYeb+AEYAuIGIBgCYAWABM5cCWKC8BoDxAEqVv+kAHjfdakEQBCEmcce5M3MVgCplu5aI1gLoBmASgDFKtdkAPgFwm1L+PAdzCS8iog5EVKScx1Yqdh/C9n2HMaq0AABQXXsEP3thKcq37sOkIV3RtUMOXv26EnddMACThnRrPm7voaO4/Y2VmDCoKKzczVQdOIxvdh7E2P6dnTYlLtv21GHr3kMYUNQOc5ftwN8WbEDd0SY0BRgje+fjy817cN0ZvbFz/xHceGZf9OvS1lF7l1fuxw8fX4ix/TuhurYey7btBwDccGYfpBNFPW7T7kNoaAygdXYGeuTl2GWuLSyr3I+N1d/jZ2f2xdAeHTCwW3vNevNWVuGh+euxqeYQpg4vxrJt+1FSkIt3Vn2HKcN6YOueOpT1zLPZendxXJe2OP/ErqaflxLJ505EJQA+BTAQwDZm7qDat4+Z84jobQAzmflzpXwBgNuYuTziXNMR9OxRXFx88tatUcfiJ03JjHkAgM9uPRO7Dh7BTa8ux/Z9hzXrVsyc2Lx9/T+X4L0137UodzPD7v0Q1bX1nrA39LmcVNwBSxWhjIXT7ylkrxbRtD3yZxXjHuA5tCRD6zNqbAqg753v6jqnn9onUc4/sSv+PmVoUscS0RJmLtPap3uGKhG1ATAHwE3MfJCifxpaO1p8HZh5FoBZAFBWVmbpiiGjH/hYs7x/UTusrTrYovzgkQYrzbGE6tp6p01ImMooN1qvMGFQFzw29WTNfeP/9lnYd2vLfe6/6erl8NEm9P/9ezHrMDM+XLtL1/nW/WkcWmWmm2GaoELXaBkiykRQ2F9k5jeU4l1EVKTsLwJQrZRvB9BDdXh3ADvNMddcot2e0lLZjRB0E+uhNxDw7wpnGenxfx//Xr4D17+wVNf5sjNk0J4V6BktQwCeBrCWmR9S7XoLwDRlexqAN1XlVyqjZkYAOOBEvN0IAVl6UDCIn79DGWnxxf27A/qfJGNEAQQD6AnLnAbgCgCriGi5UnYHgJkAXiOiawFsA3CJsu8dABMAbARQB+BqUy22AT//MN2En5vZz98hM8V4058nmHYuIRw9o2U+R/QIxliN+gzgBoN22UK076iPn6gFm1B/hTq1zXbMDreTruMpQEgOV6T8dYqo4q5Sd2aWx0ZBk1hfi9B36NmrTsHIPvk2WeQe5CfjPCnbk1HcMRcU5YGkidXibpdFgteI2aGq7GuXk5GSI0Hkd+M8KSvurbOjP7SowzLyHRWSIRRz9+vIq3suGui0CUIcUlbc0yjWBBTW3BYEvYS+Nhlp/vyJdWnXKuZ+n97TPIU/v3k6iBkvVYu7DbYI3iTWdyjkFEiHYZBHLx+KX4wtddqMlCJ1xT1qxB1oChzbFsddSIZQv42eCT9+ZOa768Jel3Zqi/zWWQ5Zk5qkrrjr8LoAgMV3F6Kgp0PVrzH3RJFmsJ+UHQoZa3hjU0BGywjGOBZzF1ULIQJvL6nruSP6zCwvzy50ewfwK4u3OW2CaUjMXXAzKSvu7XMydaVr3Vj9vT0GpQgz3ljltAm2EEjxmHskLvc5fEnKivtDPxocdZ/ac3/+ywrrjTHIwk27m7flR+QOQpG9WIt5+JUXv4q9NkP/onYAgF4Fre0wJ2VJWXHPbxM934d6hqoX8szcNmel0yYkibsa9+N11bjxJX1pagEgMz36zyeQwmGZO+eublGmHpjQsXUmgOhhUcEcUlbcgehfLrX36/a83Dv2H0blXm8veuEWrn7ua7y9Ul926mtO64W7Ljgh6n6/T2IS3E9Kf/OijZg5paRj87bbO1fve2dt2Gt3W+sf7pzYHx1jjNtu9twl5i44REqLu5qnPtuMZdv2AQj36F3uuKN9TqbTJqQk8ST7WG4Z623xAi73kXxJSou7+nd3z7y1mPzYQgDhMfcml38rcyIyDrp9KKRfiNdPmmqTmDbVyKgyt5Ha4q5jsQ63i2WKaIfriJfjf9YVJ+OM4wpTZn3QsQ/+F4s273HaDEGFnjVUnyGiaiJarSp7lYiWK38VoeX3iKiEiA6r9j1hpfFWoe5EbXJ7XCYCb1nrX8b064TZ1wxLqYVeLpu1KOo+ZmDcwC7o2r4Vrjq1l41WpS560g88B+BRAM+HCpj50tA2ET0I4ICq/iZmHmKWgVYSLXVYwENDIVNJPMzm64q9yEpPw+AeHZw2xfeUFOQiNysDC28fKyEcm9CzhuqnRFSitY+CyvIjAGeZa5azqL11tw+FFJLnkie+BABUzJzosCX+5rXrRiI3K2XTWDmG0YDgaAC7mHmDqqwXES0jov8S0ehoBxLRdCIqJ6Lympoag2YkiY6Yu9uHQkbiMXMFn1FTW9+i7HBDkwOWCEbFfQqAl1WvqwAUM/NQAL8C8BIRtdM6kJlnMXMZM5cVFhYaNCM59CQOa3K5WEpQRnATlzyxsEVZn8LwNAOdlVWcrh0tsXcrSfpZiYgyAFwM4ORQGTPXA6hXtpcQ0SYAxwEoN2inJUQfLePdsIyX8s/LU4Z3Ob6orWZ5xZ66sNdpBHTPyw0ra5OdIaEwGzDiuZ8NYB0zbw8VEFEhEaUr270BlALYbMxE+/HUCBkPue5bdh9y2gTBJLrn5aJi5kT0Loyd/CtVxvm7ET1DIV8G8CWAfkS0nYiuVXZdhvCQDACcDmAlEa0A8C8A1zPzXjMNtgO1R+klT9jt7K876rQJunD73AY3Ec8RunBIV5ssESLRM1pmSpTyqzTK5gCYY9wse4j2G/bSSkyRwzndbq/gLxrjdErdNPY4mywRIkmN6XMR/Prc2F84tbd+qL7RanMEwbM0BgIx90tSTOdIyaaPl2xL/aS5YvuB6BUFQ9S69MYpTz/6iReWkUl2zpGS4h6Lo40BT83hb2iK7Tm5maON3rVdCNIQJyyTKWkxHSM1xV3xJrSciuN++65nJi5V1x7B059vCStzs+le8eJc3ISuI57nnoorUbmF1BT3OHhF3HfskxWYBGeJF3PPiLEUoWAtKdny8XwJb0i7dzxhwb/EGy2TKStROUZKinuIaA6612alqpFx+caRce76aZSwjGtJSXGP5/DWR3T0ScefICRHpoyFdAxpeQ0ixfx7lw7Z00KcTuOom/Anz7syLZJnSBPP3TFSUtyjLdIRor4xgFNK8ppfy6Nl8ny7qxan3PshqmuPOG1KUsz/ZpfTJghCUqSkuIeIFp5pDATQPicrbj034jbH/ZnPt6Cmth4L1lY7bYpgAbF+G+efWGSfIUILUlLc9axcr67zxYbd1hokuAoJbemnXavos71vG3e8jZYIkaSkuMejprYe6kjMT19c6pwxHkctlG59ADoiKwUlzb+uHxl1X/e8HBstESJJSXHXIzLx4vJuQMtCtw7jc3Nrvr/mu4TqPz2tzCJLvEdpZ+1FOwCZh+E0qSnuOr5zMoLLHLww7j7AHJ7mOY7NY/t3ttokz3PfxYOcNiHlSWkJi+XkesFz18KtUupmJy4QABZv8dyaMq5myrBip01IeVJS3PUIt5vFKIQXbPQCAeaw7Jqxbvq5Wek2WCQIxtGzzN4zRFRNRKtVZXcT0Q4iWq78TVDtu52INhLReiI6zyrDzSCWOEq80Bxc2gUQRoBZd7K4W87tZ7E1gmAOejz35wCM0yh/mJmHKH/vAAARDUBwbdUTlGMeCy2Y7Sr0xNw9qu1uFVM3h7kCrD8TqHvfhSCEE1fcmflTAHoDkpMAvMLM9cy8BcBGAMMM2OcY8bLdCf5hReV+6F3zRB7oonPZKT2cNkFQYSTmfiMRrVTCNqG5+t0AVKrqbFfKWkBE04monIjKa2pqDJiROHp+n/NWVVluh1E0vWEX35PcKoyvfF0pnrsJuPXzTVWSFffHAfQBMARAFYAHlXLNoddaJ2DmWcxcxsxlhYWFSZohuB0X32vCUKd5jjmKShQsKtI27iIpcWfmXczcxMwBAE/iWOhlOwD1s1l3ADuNmWg+er+EN51darEl5uPaceUu/9036fTcvdoXYwfSNu4iKXEnInVGoMkAQiNp3gJwGRFlE1EvAKUAFhsz0TncuGZHxe5DOHC4AYA3HoPd1MHLzFi1/YDmPr2fdU5WhokW+Ys0L3whUwg9QyFfBvAlgH5EtJ2IrgXwABGtIqKVAM4EcDMAMPMaAK8B+AbAewBuYGbXJe7Q8xXMykhzlzIpjPnLJ5j4yGdOm+FJXvm6Ehc8+jk+Wtcyja/6OxHr6ae1jHOPioi7u4jrhjDzFI3ip2PUvxfAvUaMcgPZGWluDXBgu7Iwtta9x4X3I9ew/rtaAMCW3XXYvq8ubJ9eXYqVS0UQ3ERqzlDVNc6ddI+gEOLjJp/u9fJKjLr/47AytdcZ62Pv26mNVWZ5nqHFHZw2QVCRkuKuFzfG3NVo3aTcZrKbOnhD7bVO8eDD9tlsix+ZNERz1LPgECkp7nofwcVzNw8G0KjzbvnCtcMtsSHWLFkJFwt+IyW7/nVPhfegtrs1n/sD763D7u+P6qrbMz/XYmu0UIVlHLi6IJhNSnruepEfuXnoFXag5YLkJTPm4f731pltUhjiuQt+IyXFXc8PmSh81qKQJCY14ZOfbjZ8jphZQFXb3x9pRMmMeXh58bawOu1zoq8XKghuIyXFXS9elHYv2hyJVV70059v0VVv54HDmvX7dZFhkIJ3EHGPgXSopg7qT1o+dsEPiLjHQP0jP9Lguom2mogwJUd4uwVfSBhe8DIpKe56EocRwkee7D2kv0PQLrwg5MmY6MzCHscs/cHjX0aUCIL3SElxD5GTGTtPCEfZdgtaE4TeXV2F6/5Z7oA13sYLN0pBSISUFPeQX/jADwfHrKeOuXtl5Mzv31yD99e0TIzlJczsUGVm/PKVZVi8Re9iYoLgD1JyElOIwrbZGNStPVbt0E4D63Zvzu32uYFDR5vw5vKdmP9N7BueVlO6dUKYIOghNT13lWcYa0SM2ll3w8gZtdgcbQzgd2+ujlHbHTgtkKGPOp4Zmhk2TbdGEOwjJcVdTbQfPRGFCZMLtD3sZvP+mu+wMsrCE17HzO7UULbHeAnM3JTgTBDMICXFfUy/Ts3bsX7SN57VV1c9q/m6Yi/+s2Jn2M3GDU8SenDaytBTWrwuE83mdNp4QTBASsbc22Qfe9uxwgbd844lsHJSTC95Ijg079t7xjeX+XoxYhNTGTd/bEmcQLRd8DJ6ltl7hoiqiWi1qux/iWgdEa0korlE1EEpLyGiw0S0XPl7wkrjzSBqWEZnPTtRhw58LO2WED8so1Hmhg9dEJJET1jmOQDjIsrmAxjIzCcC+BbA7ap9m5h5iPJ3vTlmWodej9wNP3QXmOAYyd7MQqIePyzTskLFnjqNmoLgDeKKOzN/CmBvRNkHzNyovFwEoLsFttmCXr10g66q9ccrUZlkbkhaM1SNhmXccHMWBDsxo0P1GgDvql73IqJlRPRfIhod7SAimk5E5URUXlNTY4IZyaHXc3dDB6bahnhT9EXMwonXGnqaa9rIEjNMEQRbMCTuRHQngEYALypFVQCKmXkogF8BeImI2mkdy8yzmLmMmcsKCwuNmGEIvRroBq3Uu0wd4A57k0XrqST5sIzyP15YRsezwcQTi5K0QhDsJ2lxJ6JpAM4HMJUVN5GZ65l5j7K9BMAmAMeZYahV6PVw3SCWsz7d1LwdLyzjAnNNJfmwjN9aQhD0kZS4E9E4ALcBuJCZ61TlhUSUrmz3BlAKwPgSOhYSzRm+pKxHRD3nRaJit/4OPjfYC3jnJuOS5hIE04g7zp2IXgYwBkABEW0HcBeCo2OyAcxXxlsvUkbGnA7gj0TUCKAJwPXM7KqMTaNLC8JeR3scv/W8fnaYkxCJDIX0mli1ykzDkYYAAO33ZjQsE7eex9pLEOIRV9yZeYpG8dNR6s4BMMeoUVby5JVlYa8DAe16aRGLNLvBE1abUHukMXpFAE0eyWIZ4tmrhmHKk4ui7jc8iSlevSTPLwhuJeXSD2SkJecDukDbw7h1zsqY++96y/1JxdRkph/7XJxo6qONUe7yguBRUk7cI6fte2koZCImvFa+3TpDEiCZDk2tQ5Ie1q/z8nfMXZXsFQTBlaScuEeie4aqxXZEo6HpmEcZa7je9Wf0scOchDGr3ZIOy0jARUhRUk7ck80Z48SQuiMNTSi989j8sGgmzLx4EPJyM22yShAEL5ByWSEjx4fr7Xd0IiqzfZ++oY/d83Kxr877ud0/uuUMZGVo+xvJhGVeWbwNM96QcIuQmqScuLfEvWGZo43hV41mQxOzL8IPvQvbAACqa4+Ycr6XFm8z5TyC4EVSLywT4bp3U+Vs1+KPk04A4MwC2ZGCHe3pITOdXDeaZ0XlfpTMmIfNNYcSP9ik9+KR3GqCYAkpJ+6RPD2tLOb+ru1zAAAfro29wLIVtBRsbdXLzkh33TT7uct2AADWVh005XzueneC4H5SXtwL2mTH3P9tdS0A4MnPtthhTlK0yc5wneduBB+9FUFwjJQX93g0NQWlpm22/d0TkYIdTcCP69zGemNsxLRx7l5Jei8IFiDiHodLhwUTiF0/xv5x5Ho7SYnI8zoWb01Y8eYFITFE3OOQk5kOAJizxP4Zny089xh1vb5g9knFHZq3tW5qTQHG3kNHEzrnisr9hu0SBK8i4h6HNEU0N+9OYtSHQSIlzm2dpmai5+Y0e2GF9YYIgk8QcY9DmoMecaSY+1faw4l2D0uV9y8IZiDiHgcnox2JiJnHozL68PGTiyCYjW/FXZ1wywhqz/3gkQZTzqkXvaNlAO0FsyscCCWFMHKz6dyuFbI10hCItAuCfnwr7nOX7jDlPOmq/O/3vP2NKefUy1vLw99DxZ7oYq2Vpv4Hjy802yRbSE8jzL5mWItyJx33yUO7OXdxQUgCXeJORM8QUTURrVaVdSSi+US0Qfmfp5QTET1CRBuJaCURnWSV8bGoN81zP7Z9uMHeBR1mf7k17PXWPdETiWl5yvvqEhtd4ia0hNzJ/DmpEPUS/IVez/05AOMiymYAWMDMpQAWKK8BYDyCC2OXApgO4HHjZjqHV4YYaoVlnFxpzwovW0LugqAfXeLOzJ8CiFzoehKA2cr2bAAXqcqf5yCLAHQgoiIzjBWi45F7kG60vHRHtd1n7Sv4HyMx987MXAUAyv9OSnk3AJWqetuVsjCIaDoRlRNReU1NjQEz/EmjSWElz6IVljFZ3UeXFph7QkFwEVZ0qGr5OC3zGzLPYuYyZi4rLCy0xQij2DmJ6PUEZ8S6LXxkhTlWtL/enEFaYS9BcDNGxH1XKNyi/K9WyrcD6KGq1x3ATgPXSYpEZKC0k77EW002BrGrD9YnVH9YSUeLLHEGO1qaiPCniwbacCVBsB8j4v4WgGnK9jQAb6rKr1RGzYwAcCAUvnErH9x8uq56dnboZaQn5ikO6t4eU4b1iF/RI2i1dYDZtPzwIRqd7HUWBAvROxTyZQBfAuhHRNuJ6FoAMwGcQ0QbAJyjvAaAdwBsBrARwJMAfma61TpIRBrdFtIAgK4dWiV8TEaab6ctAADmLN2B8X/7DAtMWjiFoH+y26jSfFOuKQh2oSvgyMxTouwaq1GXAdxgxCi3Yuc46/zWsRcR0cKF96ik0WrrUFbIFdsPYEiPDmjbKjPqgtoA0DM/N+bcAAA4eFjfrOPJQ7vrqicIbsHfrl4Ef7jwhKSOK2ybuNAaJZBEDGhTzfcWWOIMsd7+Iws24OR7PsTPX14a8xwZWtN2VRABJQWtkzFPEFxPSol7x9ZZSR2XrxxnZ8xd77Xa52Q2b8fzUr2Enrf//hpj4RkCcN4JXQydQxDcSkqJu1Hs7HrTGwL672/GNG87mZ7YbPycu14Q7MD+hUFdzme3ntkijutEh2tARz9fz/xcdMg99jTiI2035UYa7xxu7EgXBLMQcY+gR8dcp00AoC/m7lZpuurZxfhkvbFZxx1U4Sar6Jnvjs9aEKzAt2EZLacsWW8wdCo7IwV6hl9HhmHcEJa55+1v4gr7/T8YFPc8Q4vzzDJJk94FrTFj/PEAgBeuHR6z7g9PlpEygvfwrbibKcTHNNM+ddcTc47UcuelHXjq8y1x65TkmztCpbEpgJIZ81AyYx7qjjYe2xGjCcf064TsjODi58cXtQ3b1zdixvLxXcL3C4IX8GVYZtKjn2PrXm+PHNFzG2nhqbtB3R2grqGpebv6YD1KCuJ/rWN1WBe1b4WN1ceGldqZdkIQzMKXnvuK7Qewv878JfHsDcvEv1hebvjQTq9oe1qc8eeJsHDjbpx230fNr7/YtBujH/gIP3h8ITZrLDN4sbKiUm5WenNZpDWnROTpkRQFghfxpeduNiEH2c6fuB49eXTq0LDXWqM/qg4cRlH7HLPMMgX1E0dZzzzccGbfpM/1x7e/QW39sVDMnXODi4VV7j2sWf+BH56IPp3a4NpRvTTteerKshb5azITzPMjCG7Al5672YTSvdo59lrPtTq1Dc8/oyVBr5cnljrYDtSO+/PXDsOZx3eKXjkO9Y2J5b3PSE/DDWf2RavMY557XussjC4twN0XDMDZAzo338yJgJ+f1RdXjixJ2j5BcArx3HXgxCCUZO4jLhgsowv1ouO5Wca+gvWqeLsR/qkaMZOuJGD72Zg+uOXcfqacXxDsxlfi/uwXW2KOXTbqedsblkn8aloLSrhR780cspmo566Haaf2xPZ9dbj+jD6mn1sQ7MJX4v6H/3xjyXmdEMhYMfeczHRcfVpJi3Itzfzrgg0YP6gL+nZyz3A+M8X9qAXLEeZmZeDeyfHH4guCm5GYux4oFHO375KxPPe1fxqHW8cd36Jcq0O1KcC4bNZXptpmlHQTR8vIMEVB0EbEXQfNM1RtvOa6qtqEj4kmmfWN5sSlzSLdxG+dDFMUBG18FZYxwh0TjseKygNOm9HMM1/En+mpl9ojjfEr2YqJYRkLYu6C4AeSFnci6gfgVVVRbwC/B9ABwE8AhBKM3MHM7yRtoU1MP90dnWcLN+5G+9xM9O3UJmyWpB6848N6x1JB8CpJizszrwcwBACIKB3ADgBzAVwN4GFm/ospFrqA5klMNgTdL38qGB//8YjihMVd73qgTsMMXD68GOu/Szz0JAiCPswKy4wFsImZt/oxR7YT7+iFRdsSPsYz4g7gzzIaRRAsxayurcsAvKx6fSMRrSSiZ4hIM3crEU0nonIiKq+pMZb7WwjidPx5eeV+XfWcWmSpYuZEZy4sCA5gWNyJKAvAhQBeV4oeB9AHwZBNFYAHtY5j5lnMXMbMZYWFhUbNsAW3r/zm9DT5i/7vC1319C4hKAhC8pjhuY8HsJSZdwEAM+9i5iZmDgB4EsAwE67hKKFQk9tF6adj3NEpHI/OETlx7CBHlUtGEFIBM8R9ClQhGSIqUu2bDGC1CdcwhWQ9b//1IjhLXuus+JVMZvUfzrP9moLgJIY6VIkoF8A5AK5TFT9AREMQ7DeriNjnadwelnE7f710CMYN7OLItc2cFSsIXsCQuDNzHYD8iLIrDFnkQnw4AMgRsjPSwlLtCoJgHZJ+IAHEczeG3CQFwT48L+71jU2WTy5qXqzDoQ7VD391hiPXTQQ9Cbz8OAdCENyKp8X9UH0j+v32PTw8/1trL9Q8Q9Xay0SjuGP0HPVuYeyDn8StY2aqX0EQYuNpcT9wOLgI9utL3LeUnFm8/fNRyMpIQ7tWwe6RT349xlmDNKjcW4eKPXVx60mfpiDYh6fFPVFHOtmwihMpf0MM7NYeAPDhLWfg7Z+PQklBawes0ObZL7agZMY8LN22T1f9RD33z249MxmzBEGAx8U9hNUOITmp7gqd2rZqFnq38NRnwbTEe74/qqt+olGZHh1zMaRHBwDAY1NPSuxgQUhxJJ+7DrTWJvUazGx6h+aO/YeD59ZZP5mY+9PTyrC8cj8GKyIvCII+fOG524Xb0w8AwLxfjMKUYcUtym9+dTlu+9dKByw6RjLint8mG2P7d066M/YXZ/XFC9cOT+pYQfAynhZ3O/KrA94an31C1/Y447iWidj+vXwnXi2vtOSaf3pb38LkRjpUkz30uC5tMaq0IPkLC4JH8bS4h7Br/LQd95LsDOMfybkDOptgifkY+ZzSkrwzjB9YFL+SIPgQX4i7XpJOHBYa526eKVFhBvINJtZKSyM8f417knGe0LUdAGOee/ucTPTomJPwcZJTRkhVUkrck8XODtUAMy4b1sPweU7XCM04wYzxxzen203W+w7x4rUjzDBJEFICT4u7H3O9BJhdP5Pzq8178NgnG3XVvf6MPggoH5RRJ9rlzSIIrkKGQurAjgWya2rrMevTTQiw+3OwXDprUUL1H/zREDz60UYM7i7DGQXBLkTcE8DKB4Xf/nsV3l+zC4C7p+kns05rr4LWePBHgw1f+3BDU0L1rzmtl+FrCoJXEXF3CeqsimlEmDq8GMd1bmv6dT5ZX42s9DSc2je54YEbqmtNtkg/PfO1E6hlpBEaNbJS/v6CAVabJAiuxdMxd7vZse8w5i6zKknZMXc9jYB7Jw/CtFNLTL/KVc9+jcuf+njaLFcAAA+PSURBVCrp4/Wk9rWK7Ix0PDJlaIvyqzTaafLQbjZYJAjuxbC4E1EFEa0iouVEVK6UdSSi+US0QfmfZ9zUWDboq5f8UMjgBapr63HzqysQsEDg1KEYs2LuN5wZfcHs2iMNSZ3zNYsmQunlyNGWoRmt4Y4PXzrEDnMEwbWY5bmfycxDmLlMeT0DwAJmLgWwQHntWSKlwwrfVT1CxqzRMr857/io+5Zs1ZfJUc2W3YfwwqJtcev17dQm4XPr5dS++S3KcrMkuigIkVgVlpkEYLayPRvARRZdxxHijZr5dlctvt2VWGxa7X3a0aF61bNfJ3xMecVeXfVaZaZhZO98DCvpmPA14tE9LxcVMyfizgn9m8suGFyELu1a4dIy4/MDBMEvmOHyMIAPiIgB/IOZZwHozMxVAMDMVUTUKfIgIpoOYDoAFBe3THSl68I2hX8jHel4UZlzH/4UAFAxc6Ku81cfPIJ5q6qaX7tpnPvqHQdQ2DYbndu1wm90Jh7LyUzHy9OtnXAUyhfz+vUj0buwDRbdMRYALMufIwhewwzP/TRmPgnAeAA3ENHpeg5i5lnMXMbMZYWFxmZTWq2FF5zYNey10eyQ3x04gordhwAA6747iGF/XhC2f1+dvvzodnD+3z/H6Q98jOraI7qPaaXMSLWS/kXtUDFzIk6JeDqYMKiL5dcWBC9g2HNn5p3K/2oimgtgGIBdRFSkeO1FAKqNXkfz2jal4D33hM7A66rrGrzsiPuCYl4xcyLG/fWzFvudHJGiRX1jAKPu/1h3/fNPdC5Z12NTT3bs2oLgJgx57kTUmojahrYBnAtgNYC3AExTqk0D8KaR60TDvrBM+KOBWdfdGGXMuNaYbSvYqSy2oYdEJi9dOFiGIQqC0xgNy3QG8DkRrQCwGMA8Zn4PwEwA5xDRBgDnKK9NJ/E1VM26rr4z1TfGnlF59kOfapY3NiU+CzQa834xKuq+0IzPmtp61B1tNO2agiA4j6GwDDNvBtBiXjkz7wEw1si5dV4fgPVZGyPPrtexvuLpxXjtupFhZQcOxx9ffrTJPM89J0b8uynAqKmtxyn3foh+ndvitetGolVWGrIzjMXMJc2uIDiPpwcIhySw7mhTzDh1j445qNyrPwQR97o64zKLtwSHDjY0BdDQFEBTgDH4Dx/EPc5Mz/1IQ/RzhUb1AMD6XbUY/McPMKJ3R7wyfWTUY+Ix7oQuyDJhwRFBEIzhbXFXNHb39/W4NcYwvc5tWxkS90SHQqqpb2zCxY8txJqdB3Uf02CiuOdmBb3wa07rhWe+2BK3/qLN+sayR6NbXuILagiCYD4ed7GOqeycpdFzvpiesjeB0/T77XsJCTsANJjYoVpS0Br/uXEU7pgQfbaqEX53fnhyrgwJyQiCK/CF5241kTH9gMUXNjMsAwCDurdP6ri3VuyMW6e4Y3imxh4dtTM3CoJgL5723BOVWLMScll9T2kwsUPVCK/rmO2Zl5sZ9vryYcnNNhYEwVy8Le46NTDkeScblom8J8Q6jxmhn05tsw2fwwglM+bhwOGGuE8o/7jiZJRFzBA1uk6qIAjm4G1x1+tDm6w3sULiRicgPXjJYFcsMlG5tw5fbNwTs855JwSn+n/86zEAgG4dpDNVENyCxNyTuW6Mm4rRkS4XDe3minHiiSQv61XQGo9NPQlDi2WNVEFwC9723J1x3GNet6HR2B3HBboOIPpEpJN7aq+7MmFQEYrai+cuCG7B2557gl2bycpuy5h7yzofr6vGPz7dhKoD+rMnal/LOnVPTyPdSclWbN+vWf7rc/uBmVFv8ogeQRDMxdvirtdzN6iXkdeJ7Ghc/10trn4u8cUv7GZE745x4+ghok0KG1rcwZaUvoIgGMPTYRmnOHXmR3j0ow0AgP+ZXY7z/qqdACxR/jx5kCnnicZfLhmsuZi0Xv5nVC8RdkHwCJ4W90SHQiZLdkYaxg8MXwTiLx98CwD4cO0uQ+cOMfb4Trh8uLVjxIva5+CuCwbgp2P64DSNtUjj0eRUD7YgCAnjbXG3abEOIsLjPzZ/EQj1VH0rY+1qiAi3jTseV53aK+FjYyUhEwTBXaREzP2ms0vRNJ8xbqC5S7Dd/dYaQ8d3ad8Ko0sL8PLiSmRn2nufTSbFQcBlK0QJ7uOeiwaicl+d02YI8Lq466zXvWNui7zqZvDcwgpDx2dlpGFE73y8vLjS9kWx63WurFTWMw/TTi3B7IUVuPGsvhZbJXidH4/o6bQJgoKnxV1vBkKXDB1vCR97+rDbxtGlBZrlV47siee/3AoAuHx4cXMn7wWDu2rWFwTBnSQdCyCiHkT0MRGtJaI1RPRLpfxuItpBRMuVvwnmmRvOwG76sh3mt8myyoSYnN2/U4uypb87J+x1qN/A7slL+W2yUTFzIu6dPBCtVCGhq087Fou3evSOIAjWYcRzbwRwCzMvVRbJXkJE85V9DzPzX4ybZw5Gl40LQZRYyoMAA5OHdsPcZTuayzq2zsI3fzwPA37/Pn55dilOURJvTXXocXbq8J6YOlwepQXBbyTtuTNzFTMvVbZrAawF4Otl77fcNzGh+gFmPHzpEFTMDD8uNysDFTMnYtKQbujaIQcVMyc2i7wgCIIZmDJEg4hKAAwF8JVSdCMRrSSiZ4hIMxkJEU0nonIiKq+pqTHDDEd58sqyFiJ+yck9HLJGEIRUx7C4E1EbAHMA3MTMBwE8DqAPgCEAqgA8qHUcM89i5jJmLissLDRqhuOcM6Bz8/blw4tRMXMiJp5Y1FzmtQ7Ji0/y9UOYYCK9C1ujTbanx2b4EkOfCBFlIijsLzLzGwDAzLtU+58E8LYhC13K0OIOWLatZXKtipkTNRfseOSyIfj7lKF2mGaYLfdZ1gcu+JCPbhnjtAmCBkmLOwWnVD4NYC0zP6QqL2LmKuXlZACrjZnoLjbcOx71jQEQgBPuel+zjtZsU7tmoJqBl2wVBEEbI577aQCuALCKiJYrZXcAmEJEQxCcY1QB4DpDFrqMzPQ0ZKan4UhDk9OmCIIgRCVpcWfmz6E99+ad5M3xDnbPKBUEQUgETycOcxK9s2MFQRCcQMQ9SdJE3AVBcDG+FPf81vakG3j/ptNtuY4gCEKi+E7c3/75KHxwsz2i269LW1uuIwiCkCien3lw89nH4eEPv21+XVIQPqHiw1+dYen1F844C3sPHbX0GoIgCInieXHvVdg67HXkBKK+ndpYev2uHXLQtUOOpdcQBEFIFM+HZeyKrwuCIHgJz3vup/bJR8/8XGzdU4fO7bLRtlUmAOClnwxH2+xMh60TBEFwBs977kSEy4cVAwAmDTmW7OrUPgUY1F3fYh6CIAh+w/PiDgRTAgT/y9hzQRAEwAdhGSCYYnfXwSP42RhZwFkQBAHwibi3ykzH7RP6O22GIAiCa/BFWEYQBEEIR8RdEATBh4i4C4Ig+BARd0EQBB8i4i4IguBDRNwFQRB8iIi7IAiCDxFxFwRB8CEUmSLXESOIagBsNXCKAgC7TTLHTMSuxBC7EkPsSgw/2tWTmQu1drhC3I1CROXMXOa0HZGIXYkhdiWG2JUYqWaXhGUEQRB8iIi7IAiCD/GLuM9y2oAoiF2JIXYlhtiVGCllly9i7oIgCEI4fvHcBUEQBBUi7oIgCD7E0+JOROOIaD0RbSSiGTZfuwcRfUxEa4loDRH9Uim/m4h2ENFy5W+C6pjbFVvXE9F5FtpWQUSrlOuXK2UdiWg+EW1Q/ucp5UREjyh2rSSikyyyqZ+qTZYT0UEiusmJ9iKiZ4iomohWq8oSbh8imqbU30BE0yyy63+JaJ1y7blE1EEpLyGiw6p2e0J1zMnK579Rsd3Q+pNR7Er4czP79xrFrldVNlUQ0XKl3M72iqYN9n7HmNmTfwDSAWwC0BtAFoAVAAbYeP0iACcp220BfAtgAIC7Afxao/4AxcZsAL0U29Mtsq0CQEFE2QMAZijbMwDcr2xPAPAuAAIwAsBXNn123wHo6UR7ATgdwEkAVifbPgA6Atis/M9TtvMssOtcABnK9v0qu0rU9SLOsxjASMXmdwGMt8CuhD43K36vWnZF7H8QwO8daK9o2mDrd8zLnvswABuZeTMzHwXwCoBJdl2cmauYeamyXQtgLYBuMQ6ZBOAVZq5n5i0ANiL4HuxiEoDZyvZsABepyp/nIIsAdCCiIottGQtgEzPHmpVsWXsx86cA9mpcL5H2OQ/AfGbey8z7AMwHMM5su5j5A2ZuVF4uAtA91jkU29ox85ccVIjnVe/FNLtiEO1zM/33Gssuxfv+EYCXY53DovaKpg22fse8LO7dAFSqXm9HbHG1DCIqATAUwFdK0Y3K49UzoUcv2GsvA/iAiJYQ0XSlrDMzVwHBLx+ATg7YFeIyhP/onG4vIPH2caLdrkHQwwvRi4iWEdF/iWi0UtZNscUOuxL53Oxur9EAdjHzBlWZ7e0VoQ22fse8LO5acTHbx3USURsAcwDcxMwHATwOoA+AIQCqEHw0BOy19zRmPgnAeAA3ENHpMera2o5ElAXgQgCvK0VuaK9YRLPD7na7E0AjgBeVoioAxcw8FMCvALxERO1stCvRz83uz3MKwh0I29tLQxuiVo1igyHbvCzu2wH0UL3uDmCnnQYQUSaCH96LzPwGADDzLmZuYuYAgCdxLJRgm73MvFP5Xw1grmLDrlC4RflfbbddCuMBLGXmXYqNjreXQqLtY5t9Skfa+QCmKqEDKGGPPcr2EgTj2ccpdqlDN5bYlcTnZmd7ZQC4GMCrKnttbS8tbYDN3zEvi/vXAEqJqJfiDV4G4C27Lq7E9J4GsJaZH1KVq+PVkwGEevLfAnAZEWUTUS8ApQh25JhtV2siahvaRrBDbrVy/VBv+zQAb6rsulLpsR8B4EDo0dEiwjwqp9tLRaLt8z6Ac4koTwlJnKuUmQoRjQNwG4ALmblOVV5IROnKdm8E22ezYlstEY1QvqNXqt6LmXYl+rnZ+Xs9G8A6Zm4Ot9jZXtG0AXZ/x4z0Cjv9h2Av87cI3oXvtPnaoxB8RFoJYLnyNwHAPwGsUsrfAlCkOuZOxdb1MNgjH8Ou3giORFgBYE2oXQDkA1gAYIPyv6NSTgD+T7FrFYAyC9ssF8AeAO1VZba3F4I3lyoADQh6R9cm0z4IxsA3Kn9XW2TXRgTjrqHv2BNK3R8on+8KAEsBXKA6TxmCYrsJwKNQZqKbbFfCn5vZv1ctu5Ty5wBcH1HXzvaKpg22fsck/YAgCIIP8XJYRhAEQYiCiLsgCIIPEXEXBEHwISLugiAIPkTEXRAEwYeIuAuCIPgQEXdBEAQf8v8VVma+u5/SHQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final results\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO2deXxU5dXHfycrCWtIAoQlhCUiCAIaWRQUxYVFRWytIlVc3qKtttXaKmpbbasVfau21lctrlh3i1QrbohaF0QMOwjIFggQSdgjgZBkzvvH3Al3Jndm7szd75zv55NP7jz3ufeeeWbmd889z/Och5gZgiAIgr9Ic9oAQRAEwXxE3AVBEHyIiLsgCIIPEXEXBEHwISLugiAIPiTDaQMAoKCggEtKSpw2QxAEwVMsWbJkNzMXau1zhbiXlJSgvLzcaTMEQRA8BRFtjbZPwjKCIAg+RMRdEATBh4i4C4Ig+BARd0EQBB8i4i4IguBD4oo7EfUgoo+JaC0RrSGiXyrlHYloPhFtUP7nKeVERI8Q0UYiWklEJ1n9JgRBEIRw9HjujQBuYeb+AEYAuIGIBgCYAWABM5cCWKC8BoDxAEqVv+kAHjfdakEQBCEmcce5M3MVgCplu5aI1gLoBmASgDFKtdkAPgFwm1L+PAdzCS8iog5EVKScx1Yqdh/C9n2HMaq0AABQXXsEP3thKcq37sOkIV3RtUMOXv26EnddMACThnRrPm7voaO4/Y2VmDCoKKzczVQdOIxvdh7E2P6dnTYlLtv21GHr3kMYUNQOc5ftwN8WbEDd0SY0BRgje+fjy817cN0ZvbFz/xHceGZf9OvS1lF7l1fuxw8fX4ix/TuhurYey7btBwDccGYfpBNFPW7T7kNoaAygdXYGeuTl2GWuLSyr3I+N1d/jZ2f2xdAeHTCwW3vNevNWVuGh+euxqeYQpg4vxrJt+1FSkIt3Vn2HKcN6YOueOpT1zLPZendxXJe2OP/ErqaflxLJ505EJQA+BTAQwDZm7qDat4+Z84jobQAzmflzpXwBgNuYuTziXNMR9OxRXFx88tatUcfiJ03JjHkAgM9uPRO7Dh7BTa8ux/Z9hzXrVsyc2Lx9/T+X4L0137UodzPD7v0Q1bX1nrA39LmcVNwBSxWhjIXT7ylkrxbRtD3yZxXjHuA5tCRD6zNqbAqg753v6jqnn9onUc4/sSv+PmVoUscS0RJmLtPap3uGKhG1ATAHwE3MfJCifxpaO1p8HZh5FoBZAFBWVmbpiiGjH/hYs7x/UTusrTrYovzgkQYrzbGE6tp6p01ImMooN1qvMGFQFzw29WTNfeP/9lnYd2vLfe6/6erl8NEm9P/9ezHrMDM+XLtL1/nW/WkcWmWmm2GaoELXaBkiykRQ2F9k5jeU4l1EVKTsLwJQrZRvB9BDdXh3ADvNMddcot2e0lLZjRB0E+uhNxDw7wpnGenxfx//Xr4D17+wVNf5sjNk0J4V6BktQwCeBrCWmR9S7XoLwDRlexqAN1XlVyqjZkYAOOBEvN0IAVl6UDCIn79DGWnxxf27A/qfJGNEAQQD6AnLnAbgCgCriGi5UnYHgJkAXiOiawFsA3CJsu8dABMAbARQB+BqUy22AT//MN2En5vZz98hM8V4058nmHYuIRw9o2U+R/QIxliN+gzgBoN22UK076iPn6gFm1B/hTq1zXbMDreTruMpQEgOV6T8dYqo4q5Sd2aWx0ZBk1hfi9B36NmrTsHIPvk2WeQe5CfjPCnbk1HcMRcU5YGkidXibpdFgteI2aGq7GuXk5GSI0Hkd+M8KSvurbOjP7SowzLyHRWSIRRz9+vIq3suGui0CUIcUlbc0yjWBBTW3BYEvYS+Nhlp/vyJdWnXKuZ+n97TPIU/v3k6iBkvVYu7DbYI3iTWdyjkFEiHYZBHLx+KX4wtddqMlCJ1xT1qxB1oChzbFsddSIZQv42eCT9+ZOa768Jel3Zqi/zWWQ5Zk5qkrrjr8LoAgMV3F6Kgp0PVrzH3RJFmsJ+UHQoZa3hjU0BGywjGOBZzF1ULIQJvL6nruSP6zCwvzy50ewfwK4u3OW2CaUjMXXAzKSvu7XMydaVr3Vj9vT0GpQgz3ljltAm2EEjxmHskLvc5fEnKivtDPxocdZ/ac3/+ywrrjTHIwk27m7flR+QOQpG9WIt5+JUXv4q9NkP/onYAgF4Fre0wJ2VJWXHPbxM934d6hqoX8szcNmel0yYkibsa9+N11bjxJX1pagEgMz36zyeQwmGZO+eublGmHpjQsXUmgOhhUcEcUlbcgehfLrX36/a83Dv2H0blXm8veuEWrn7ua7y9Ul926mtO64W7Ljgh6n6/T2IS3E9Kf/OijZg5paRj87bbO1fve2dt2Gt3W+sf7pzYHx1jjNtu9twl5i44REqLu5qnPtuMZdv2AQj36F3uuKN9TqbTJqQk8ST7WG4Z623xAi73kXxJSou7+nd3z7y1mPzYQgDhMfcml38rcyIyDrp9KKRfiNdPmmqTmDbVyKgyt5Ha4q5jsQ63i2WKaIfriJfjf9YVJ+OM4wpTZn3QsQ/+F4s273HaDEGFnjVUnyGiaiJarSp7lYiWK38VoeX3iKiEiA6r9j1hpfFWoe5EbXJ7XCYCb1nrX8b064TZ1wxLqYVeLpu1KOo+ZmDcwC7o2r4Vrjq1l41WpS560g88B+BRAM+HCpj50tA2ET0I4ICq/iZmHmKWgVYSLXVYwENDIVNJPMzm64q9yEpPw+AeHZw2xfeUFOQiNysDC28fKyEcm9CzhuqnRFSitY+CyvIjAGeZa5azqL11tw+FFJLnkie+BABUzJzosCX+5rXrRiI3K2XTWDmG0YDgaAC7mHmDqqwXES0jov8S0ehoBxLRdCIqJ6Lympoag2YkiY6Yu9uHQkbiMXMFn1FTW9+i7HBDkwOWCEbFfQqAl1WvqwAUM/NQAL8C8BIRtdM6kJlnMXMZM5cVFhYaNCM59CQOa3K5WEpQRnATlzyxsEVZn8LwNAOdlVWcrh0tsXcrSfpZiYgyAFwM4ORQGTPXA6hXtpcQ0SYAxwEoN2inJUQfLePdsIyX8s/LU4Z3Ob6orWZ5xZ66sNdpBHTPyw0ra5OdIaEwGzDiuZ8NYB0zbw8VEFEhEaUr270BlALYbMxE+/HUCBkPue5bdh9y2gTBJLrn5aJi5kT0Loyd/CtVxvm7ET1DIV8G8CWAfkS0nYiuVXZdhvCQDACcDmAlEa0A8C8A1zPzXjMNtgO1R+klT9jt7K876rQJunD73AY3Ec8RunBIV5ssESLRM1pmSpTyqzTK5gCYY9wse4j2G/bSSkyRwzndbq/gLxrjdErdNPY4mywRIkmN6XMR/Prc2F84tbd+qL7RanMEwbM0BgIx90tSTOdIyaaPl2xL/aS5YvuB6BUFQ9S69MYpTz/6iReWkUl2zpGS4h6Lo40BT83hb2iK7Tm5maON3rVdCNIQJyyTKWkxHSM1xV3xJrSciuN++65nJi5V1x7B059vCStzs+le8eJc3ISuI57nnoorUbmF1BT3OHhF3HfskxWYBGeJF3PPiLEUoWAtKdny8XwJb0i7dzxhwb/EGy2TKStROUZKinuIaA6612alqpFx+caRce76aZSwjGtJSXGP5/DWR3T0ScefICRHpoyFdAxpeQ0ixfx7lw7Z00KcTuOom/Anz7syLZJnSBPP3TFSUtyjLdIRor4xgFNK8ppfy6Nl8ny7qxan3PshqmuPOG1KUsz/ZpfTJghCUqSkuIeIFp5pDATQPicrbj034jbH/ZnPt6Cmth4L1lY7bYpgAbF+G+efWGSfIUILUlLc9axcr67zxYbd1hokuAoJbemnXavos71vG3e8jZYIkaSkuMejprYe6kjMT19c6pwxHkctlG59ADoiKwUlzb+uHxl1X/e8HBstESJJSXHXIzLx4vJuQMtCtw7jc3Nrvr/mu4TqPz2tzCJLvEdpZ+1FOwCZh+E0qSnuOr5zMoLLHLww7j7AHJ7mOY7NY/t3ttokz3PfxYOcNiHlSWkJi+XkesFz18KtUupmJy4QABZv8dyaMq5myrBip01IeVJS3PUIt5vFKIQXbPQCAeaw7Jqxbvq5Wek2WCQIxtGzzN4zRFRNRKtVZXcT0Q4iWq78TVDtu52INhLReiI6zyrDzSCWOEq80Bxc2gUQRoBZd7K4W87tZ7E1gmAOejz35wCM0yh/mJmHKH/vAAARDUBwbdUTlGMeCy2Y7Sr0xNw9qu1uFVM3h7kCrD8TqHvfhSCEE1fcmflTAHoDkpMAvMLM9cy8BcBGAMMM2OcY8bLdCf5hReV+6F3zRB7oonPZKT2cNkFQYSTmfiMRrVTCNqG5+t0AVKrqbFfKWkBE04monIjKa2pqDJiROHp+n/NWVVluh1E0vWEX35PcKoyvfF0pnrsJuPXzTVWSFffHAfQBMARAFYAHlXLNoddaJ2DmWcxcxsxlhYWFSZohuB0X32vCUKd5jjmKShQsKtI27iIpcWfmXczcxMwBAE/iWOhlOwD1s1l3ADuNmWg+er+EN51darEl5uPaceUu/9036fTcvdoXYwfSNu4iKXEnInVGoMkAQiNp3gJwGRFlE1EvAKUAFhsz0TncuGZHxe5DOHC4AYA3HoPd1MHLzFi1/YDmPr2fdU5WhokW+Ys0L3whUwg9QyFfBvAlgH5EtJ2IrgXwABGtIqKVAM4EcDMAMPMaAK8B+AbAewBuYGbXJe7Q8xXMykhzlzIpjPnLJ5j4yGdOm+FJXvm6Ehc8+jk+Wtcyja/6OxHr6ae1jHOPioi7u4jrhjDzFI3ip2PUvxfAvUaMcgPZGWluDXBgu7Iwtta9x4X3I9ew/rtaAMCW3XXYvq8ubJ9eXYqVS0UQ3ERqzlDVNc6ddI+gEOLjJp/u9fJKjLr/47AytdcZ62Pv26mNVWZ5nqHFHZw2QVCRkuKuFzfG3NVo3aTcZrKbOnhD7bVO8eDD9tlsix+ZNERz1LPgECkp7nofwcVzNw8G0KjzbvnCtcMtsSHWLFkJFwt+IyW7/nVPhfegtrs1n/sD763D7u+P6qrbMz/XYmu0UIVlHLi6IJhNSnruepEfuXnoFXag5YLkJTPm4f731pltUhjiuQt+IyXFXc8PmSh81qKQJCY14ZOfbjZ8jphZQFXb3x9pRMmMeXh58bawOu1zoq8XKghuIyXFXS9elHYv2hyJVV70059v0VVv54HDmvX7dZFhkIJ3EHGPgXSopg7qT1o+dsEPiLjHQP0jP9Lguom2mogwJUd4uwVfSBhe8DIpKe56EocRwkee7D2kv0PQLrwg5MmY6MzCHscs/cHjX0aUCIL3SElxD5GTGTtPCEfZdgtaE4TeXV2F6/5Z7oA13sYLN0pBSISUFPeQX/jADwfHrKeOuXtl5Mzv31yD99e0TIzlJczsUGVm/PKVZVi8Re9iYoLgD1JyElOIwrbZGNStPVbt0E4D63Zvzu32uYFDR5vw5vKdmP9N7BueVlO6dUKYIOghNT13lWcYa0SM2ll3w8gZtdgcbQzgd2+ujlHbHTgtkKGPOp4Zmhk2TbdGEOwjJcVdTbQfPRGFCZMLtD3sZvP+mu+wMsrCE17HzO7UULbHeAnM3JTgTBDMICXFfUy/Ts3bsX7SN57VV1c9q/m6Yi/+s2Jn2M3GDU8SenDaytBTWrwuE83mdNp4QTBASsbc22Qfe9uxwgbd844lsHJSTC95Ijg079t7xjeX+XoxYhNTGTd/bEmcQLRd8DJ6ltl7hoiqiWi1qux/iWgdEa0korlE1EEpLyGiw0S0XPl7wkrjzSBqWEZnPTtRhw58LO2WED8so1Hmhg9dEJJET1jmOQDjIsrmAxjIzCcC+BbA7ap9m5h5iPJ3vTlmWodej9wNP3QXmOAYyd7MQqIePyzTskLFnjqNmoLgDeKKOzN/CmBvRNkHzNyovFwEoLsFttmCXr10g66q9ccrUZlkbkhaM1SNhmXccHMWBDsxo0P1GgDvql73IqJlRPRfIhod7SAimk5E5URUXlNTY4IZyaHXc3dDB6bahnhT9EXMwonXGnqaa9rIEjNMEQRbMCTuRHQngEYALypFVQCKmXkogF8BeImI2mkdy8yzmLmMmcsKCwuNmGEIvRroBq3Uu0wd4A57k0XrqST5sIzyP15YRsezwcQTi5K0QhDsJ2lxJ6JpAM4HMJUVN5GZ65l5j7K9BMAmAMeZYahV6PVw3SCWsz7d1LwdLyzjAnNNJfmwjN9aQhD0kZS4E9E4ALcBuJCZ61TlhUSUrmz3BlAKwPgSOhYSzRm+pKxHRD3nRaJit/4OPjfYC3jnJuOS5hIE04g7zp2IXgYwBkABEW0HcBeCo2OyAcxXxlsvUkbGnA7gj0TUCKAJwPXM7KqMTaNLC8JeR3scv/W8fnaYkxCJDIX0mli1ykzDkYYAAO33ZjQsE7eex9pLEOIRV9yZeYpG8dNR6s4BMMeoUVby5JVlYa8DAe16aRGLNLvBE1abUHukMXpFAE0eyWIZ4tmrhmHKk4ui7jc8iSlevSTPLwhuJeXSD2SkJecDukDbw7h1zsqY++96y/1JxdRkph/7XJxo6qONUe7yguBRUk7cI6fte2koZCImvFa+3TpDEiCZDk2tQ5Ie1q/z8nfMXZXsFQTBlaScuEeie4aqxXZEo6HpmEcZa7je9Wf0scOchDGr3ZIOy0jARUhRUk7ck80Z48SQuiMNTSi989j8sGgmzLx4EPJyM22yShAEL5ByWSEjx4fr7Xd0IiqzfZ++oY/d83Kxr877ud0/uuUMZGVo+xvJhGVeWbwNM96QcIuQmqScuLfEvWGZo43hV41mQxOzL8IPvQvbAACqa4+Ycr6XFm8z5TyC4EVSLywT4bp3U+Vs1+KPk04A4MwC2ZGCHe3pITOdXDeaZ0XlfpTMmIfNNYcSP9ik9+KR3GqCYAkpJ+6RPD2tLOb+ru1zAAAfro29wLIVtBRsbdXLzkh33TT7uct2AADWVh005XzueneC4H5SXtwL2mTH3P9tdS0A4MnPtthhTlK0yc5wneduBB+9FUFwjJQX93g0NQWlpm22/d0TkYIdTcCP69zGemNsxLRx7l5Jei8IFiDiHodLhwUTiF0/xv5x5Ho7SYnI8zoWb01Y8eYFITFE3OOQk5kOAJizxP4Zny089xh1vb5g9knFHZq3tW5qTQHG3kNHEzrnisr9hu0SBK8i4h6HNEU0N+9OYtSHQSIlzm2dpmai5+Y0e2GF9YYIgk8QcY9DmoMecaSY+1faw4l2D0uV9y8IZiDiHgcnox2JiJnHozL68PGTiyCYjW/FXZ1wywhqz/3gkQZTzqkXvaNlAO0FsyscCCWFMHKz6dyuFbI10hCItAuCfnwr7nOX7jDlPOmq/O/3vP2NKefUy1vLw99DxZ7oYq2Vpv4Hjy802yRbSE8jzL5mWItyJx33yUO7OXdxQUgCXeJORM8QUTURrVaVdSSi+US0Qfmfp5QTET1CRBuJaCURnWSV8bGoN81zP7Z9uMHeBR1mf7k17PXWPdETiWl5yvvqEhtd4ia0hNzJ/DmpEPUS/IVez/05AOMiymYAWMDMpQAWKK8BYDyCC2OXApgO4HHjZjqHV4YYaoVlnFxpzwovW0LugqAfXeLOzJ8CiFzoehKA2cr2bAAXqcqf5yCLAHQgoiIzjBWi45F7kG60vHRHtd1n7Sv4HyMx987MXAUAyv9OSnk3AJWqetuVsjCIaDoRlRNReU1NjQEz/EmjSWElz6IVljFZ3UeXFph7QkFwEVZ0qGr5OC3zGzLPYuYyZi4rLCy0xQij2DmJ6PUEZ8S6LXxkhTlWtL/enEFaYS9BcDNGxH1XKNyi/K9WyrcD6KGq1x3ATgPXSYpEZKC0k77EW002BrGrD9YnVH9YSUeLLHEGO1qaiPCniwbacCVBsB8j4v4WgGnK9jQAb6rKr1RGzYwAcCAUvnErH9x8uq56dnboZaQn5ikO6t4eU4b1iF/RI2i1dYDZtPzwIRqd7HUWBAvROxTyZQBfAuhHRNuJ6FoAMwGcQ0QbAJyjvAaAdwBsBrARwJMAfma61TpIRBrdFtIAgK4dWiV8TEaab6ctAADmLN2B8X/7DAtMWjiFoH+y26jSfFOuKQh2oSvgyMxTouwaq1GXAdxgxCi3Yuc46/zWsRcR0cKF96ik0WrrUFbIFdsPYEiPDmjbKjPqgtoA0DM/N+bcAAA4eFjfrOPJQ7vrqicIbsHfrl4Ef7jwhKSOK2ybuNAaJZBEDGhTzfcWWOIMsd7+Iws24OR7PsTPX14a8xwZWtN2VRABJQWtkzFPEFxPSol7x9ZZSR2XrxxnZ8xd77Xa52Q2b8fzUr2Enrf//hpj4RkCcN4JXQydQxDcSkqJu1Hs7HrTGwL672/GNG87mZ7YbPycu14Q7MD+hUFdzme3ntkijutEh2tARz9fz/xcdMg99jTiI2035UYa7xxu7EgXBLMQcY+gR8dcp00AoC/m7lZpuurZxfhkvbFZxx1U4Sar6Jnvjs9aEKzAt2EZLacsWW8wdCo7IwV6hl9HhmHcEJa55+1v4gr7/T8YFPc8Q4vzzDJJk94FrTFj/PEAgBeuHR6z7g9PlpEygvfwrbibKcTHNNM+ddcTc47UcuelHXjq8y1x65TkmztCpbEpgJIZ81AyYx7qjjYe2xGjCcf064TsjODi58cXtQ3b1zdixvLxXcL3C4IX8GVYZtKjn2PrXm+PHNFzG2nhqbtB3R2grqGpebv6YD1KCuJ/rWN1WBe1b4WN1ceGldqZdkIQzMKXnvuK7Qewv878JfHsDcvEv1hebvjQTq9oe1qc8eeJsHDjbpx230fNr7/YtBujH/gIP3h8ITZrLDN4sbKiUm5WenNZpDWnROTpkRQFghfxpeduNiEH2c6fuB49eXTq0LDXWqM/qg4cRlH7HLPMMgX1E0dZzzzccGbfpM/1x7e/QW39sVDMnXODi4VV7j2sWf+BH56IPp3a4NpRvTTteerKshb5azITzPMjCG7Al5672YTSvdo59lrPtTq1Dc8/oyVBr5cnljrYDtSO+/PXDsOZx3eKXjkO9Y2J5b3PSE/DDWf2RavMY557XussjC4twN0XDMDZAzo338yJgJ+f1RdXjixJ2j5BcArx3HXgxCCUZO4jLhgsowv1ouO5Wca+gvWqeLsR/qkaMZOuJGD72Zg+uOXcfqacXxDsxlfi/uwXW2KOXTbqedsblkn8aloLSrhR780cspmo566Haaf2xPZ9dbj+jD6mn1sQ7MJX4v6H/3xjyXmdEMhYMfeczHRcfVpJi3Itzfzrgg0YP6gL+nZyz3A+M8X9qAXLEeZmZeDeyfHH4guCm5GYux4oFHO375KxPPe1fxqHW8cd36Jcq0O1KcC4bNZXptpmlHQTR8vIMEVB0EbEXQfNM1RtvOa6qtqEj4kmmfWN5sSlzSLdxG+dDFMUBG18FZYxwh0TjseKygNOm9HMM1/En+mpl9ojjfEr2YqJYRkLYu6C4AeSFnci6gfgVVVRbwC/B9ABwE8AhBKM3MHM7yRtoU1MP90dnWcLN+5G+9xM9O3UJmyWpB6848N6x1JB8CpJizszrwcwBACIKB3ADgBzAVwN4GFm/ospFrqA5klMNgTdL38qGB//8YjihMVd73qgTsMMXD68GOu/Szz0JAiCPswKy4wFsImZt/oxR7YT7+iFRdsSPsYz4g7gzzIaRRAsxayurcsAvKx6fSMRrSSiZ4hIM3crEU0nonIiKq+pMZb7WwjidPx5eeV+XfWcWmSpYuZEZy4sCA5gWNyJKAvAhQBeV4oeB9AHwZBNFYAHtY5j5lnMXMbMZYWFhUbNsAW3r/zm9DT5i/7vC1319C4hKAhC8pjhuY8HsJSZdwEAM+9i5iZmDgB4EsAwE67hKKFQk9tF6adj3NEpHI/OETlx7CBHlUtGEFIBM8R9ClQhGSIqUu2bDGC1CdcwhWQ9b//1IjhLXuus+JVMZvUfzrP9moLgJIY6VIkoF8A5AK5TFT9AREMQ7DeriNjnadwelnE7f710CMYN7OLItc2cFSsIXsCQuDNzHYD8iLIrDFnkQnw4AMgRsjPSwlLtCoJgHZJ+IAHEczeG3CQFwT48L+71jU2WTy5qXqzDoQ7VD391hiPXTQQ9Cbz8OAdCENyKp8X9UH0j+v32PTw8/1trL9Q8Q9Xay0SjuGP0HPVuYeyDn8StY2aqX0EQYuNpcT9wOLgI9utL3LeUnFm8/fNRyMpIQ7tWwe6RT349xlmDNKjcW4eKPXVx60mfpiDYh6fFPVFHOtmwihMpf0MM7NYeAPDhLWfg7Z+PQklBawes0ObZL7agZMY8LN22T1f9RD33z249MxmzBEGAx8U9hNUOITmp7gqd2rZqFnq38NRnwbTEe74/qqt+olGZHh1zMaRHBwDAY1NPSuxgQUhxJJ+7DrTWJvUazGx6h+aO/YeD59ZZP5mY+9PTyrC8cj8GKyIvCII+fOG524Xb0w8AwLxfjMKUYcUtym9+dTlu+9dKByw6RjLint8mG2P7d066M/YXZ/XFC9cOT+pYQfAynhZ3O/KrA94an31C1/Y447iWidj+vXwnXi2vtOSaf3pb38LkRjpUkz30uC5tMaq0IPkLC4JH8bS4h7Br/LQd95LsDOMfybkDOptgifkY+ZzSkrwzjB9YFL+SIPgQX4i7XpJOHBYa526eKVFhBvINJtZKSyM8f417knGe0LUdAGOee/ucTPTomJPwcZJTRkhVUkrck8XODtUAMy4b1sPweU7XCM04wYzxxzen203W+w7x4rUjzDBJEFICT4u7H3O9BJhdP5Pzq8178NgnG3XVvf6MPggoH5RRJ9rlzSIIrkKGQurAjgWya2rrMevTTQiw+3OwXDprUUL1H/zREDz60UYM7i7DGQXBLkTcE8DKB4Xf/nsV3l+zC4C7p+kns05rr4LWePBHgw1f+3BDU0L1rzmtl+FrCoJXEXF3CeqsimlEmDq8GMd1bmv6dT5ZX42s9DSc2je54YEbqmtNtkg/PfO1E6hlpBEaNbJS/v6CAVabJAiuxdMxd7vZse8w5i6zKknZMXc9jYB7Jw/CtFNLTL/KVc9+jcuf+njaLFcAAA+PSURBVCrp4/Wk9rWK7Ix0PDJlaIvyqzTaafLQbjZYJAjuxbC4E1EFEa0iouVEVK6UdSSi+US0QfmfZ9zUWDboq5f8UMjgBapr63HzqysQsEDg1KEYs2LuN5wZfcHs2iMNSZ3zNYsmQunlyNGWoRmt4Y4PXzrEDnMEwbWY5bmfycxDmLlMeT0DwAJmLgWwQHntWSKlwwrfVT1CxqzRMr857/io+5Zs1ZfJUc2W3YfwwqJtcev17dQm4XPr5dS++S3KcrMkuigIkVgVlpkEYLayPRvARRZdxxHijZr5dlctvt2VWGxa7X3a0aF61bNfJ3xMecVeXfVaZaZhZO98DCvpmPA14tE9LxcVMyfizgn9m8suGFyELu1a4dIy4/MDBMEvmOHyMIAPiIgB/IOZZwHozMxVAMDMVUTUKfIgIpoOYDoAFBe3THSl68I2hX8jHel4UZlzH/4UAFAxc6Ku81cfPIJ5q6qaX7tpnPvqHQdQ2DYbndu1wm90Jh7LyUzHy9OtnXAUyhfz+vUj0buwDRbdMRYALMufIwhewwzP/TRmPgnAeAA3ENHpeg5i5lnMXMbMZYWFxmZTWq2FF5zYNey10eyQ3x04gordhwAA6747iGF/XhC2f1+dvvzodnD+3z/H6Q98jOraI7qPaaXMSLWS/kXtUDFzIk6JeDqYMKiL5dcWBC9g2HNn5p3K/2oimgtgGIBdRFSkeO1FAKqNXkfz2jal4D33hM7A66rrGrzsiPuCYl4xcyLG/fWzFvudHJGiRX1jAKPu/1h3/fNPdC5Z12NTT3bs2oLgJgx57kTUmojahrYBnAtgNYC3AExTqk0D8KaR60TDvrBM+KOBWdfdGGXMuNaYbSvYqSy2oYdEJi9dOFiGIQqC0xgNy3QG8DkRrQCwGMA8Zn4PwEwA5xDRBgDnKK9NJ/E1VM26rr4z1TfGnlF59kOfapY3NiU+CzQa834xKuq+0IzPmtp61B1tNO2agiA4j6GwDDNvBtBiXjkz7wEw1si5dV4fgPVZGyPPrtexvuLpxXjtupFhZQcOxx9ffrTJPM89J0b8uynAqKmtxyn3foh+ndvitetGolVWGrIzjMXMJc2uIDiPpwcIhySw7mhTzDh1j445qNyrPwQR97o64zKLtwSHDjY0BdDQFEBTgDH4Dx/EPc5Mz/1IQ/RzhUb1AMD6XbUY/McPMKJ3R7wyfWTUY+Ix7oQuyDJhwRFBEIzhbXFXNHb39/W4NcYwvc5tWxkS90SHQqqpb2zCxY8txJqdB3Uf02CiuOdmBb3wa07rhWe+2BK3/qLN+sayR6NbXuILagiCYD4ed7GOqeycpdFzvpiesjeB0/T77XsJCTsANJjYoVpS0Br/uXEU7pgQfbaqEX53fnhyrgwJyQiCK/CF5241kTH9gMUXNjMsAwCDurdP6ri3VuyMW6e4Y3imxh4dtTM3CoJgL5723BOVWLMScll9T2kwsUPVCK/rmO2Zl5sZ9vryYcnNNhYEwVy8Le46NTDkeScblom8J8Q6jxmhn05tsw2fwwglM+bhwOGGuE8o/7jiZJRFzBA1uk6qIAjm4G1x1+tDm6w3sULiRicgPXjJYFcsMlG5tw5fbNwTs855JwSn+n/86zEAgG4dpDNVENyCxNyTuW6Mm4rRkS4XDe3minHiiSQv61XQGo9NPQlDi2WNVEFwC9723J1x3GNet6HR2B3HBboOIPpEpJN7aq+7MmFQEYrai+cuCG7B2557gl2bycpuy5h7yzofr6vGPz7dhKoD+rMnal/LOnVPTyPdSclWbN+vWf7rc/uBmVFv8ogeQRDMxdvirtdzN6iXkdeJ7Ghc/10trn4u8cUv7GZE745x4+ghok0KG1rcwZaUvoIgGMPTYRmnOHXmR3j0ow0AgP+ZXY7z/qqdACxR/jx5kCnnicZfLhmsuZi0Xv5nVC8RdkHwCJ4W90SHQiZLdkYaxg8MXwTiLx98CwD4cO0uQ+cOMfb4Trh8uLVjxIva5+CuCwbgp2P64DSNtUjj0eRUD7YgCAnjbXG3abEOIsLjPzZ/EQj1VH0rY+1qiAi3jTseV53aK+FjYyUhEwTBXaREzP2ms0vRNJ8xbqC5S7Dd/dYaQ8d3ad8Ko0sL8PLiSmRn2nufTSbFQcBlK0QJ7uOeiwaicl+d02YI8Lq466zXvWNui7zqZvDcwgpDx2dlpGFE73y8vLjS9kWx63WurFTWMw/TTi3B7IUVuPGsvhZbJXidH4/o6bQJgoKnxV1vBkKXDB1vCR97+rDbxtGlBZrlV47siee/3AoAuHx4cXMn7wWDu2rWFwTBnSQdCyCiHkT0MRGtJaI1RPRLpfxuItpBRMuVvwnmmRvOwG76sh3mt8myyoSYnN2/U4uypb87J+x1qN/A7slL+W2yUTFzIu6dPBCtVCGhq087Fou3evSOIAjWYcRzbwRwCzMvVRbJXkJE85V9DzPzX4ybZw5Gl40LQZRYyoMAA5OHdsPcZTuayzq2zsI3fzwPA37/Pn55dilOURJvTXXocXbq8J6YOlwepQXBbyTtuTNzFTMvVbZrAawF4Otl77fcNzGh+gFmPHzpEFTMDD8uNysDFTMnYtKQbujaIQcVMyc2i7wgCIIZmDJEg4hKAAwF8JVSdCMRrSSiZ4hIMxkJEU0nonIiKq+pqTHDDEd58sqyFiJ+yck9HLJGEIRUx7C4E1EbAHMA3MTMBwE8DqAPgCEAqgA8qHUcM89i5jJmLissLDRqhuOcM6Bz8/blw4tRMXMiJp5Y1FzmtQ7Ji0/y9UOYYCK9C1ujTbanx2b4EkOfCBFlIijsLzLzGwDAzLtU+58E8LYhC13K0OIOWLatZXKtipkTNRfseOSyIfj7lKF2mGaYLfdZ1gcu+JCPbhnjtAmCBkmLOwWnVD4NYC0zP6QqL2LmKuXlZACrjZnoLjbcOx71jQEQgBPuel+zjtZsU7tmoJqBl2wVBEEbI577aQCuALCKiJYrZXcAmEJEQxCcY1QB4DpDFrqMzPQ0ZKan4UhDk9OmCIIgRCVpcWfmz6E99+ad5M3xDnbPKBUEQUgETycOcxK9s2MFQRCcQMQ9SdJE3AVBcDG+FPf81vakG3j/ptNtuY4gCEKi+E7c3/75KHxwsz2i269LW1uuIwiCkCien3lw89nH4eEPv21+XVIQPqHiw1+dYen1F844C3sPHbX0GoIgCInieXHvVdg67HXkBKK+ndpYev2uHXLQtUOOpdcQBEFIFM+HZeyKrwuCIHgJz3vup/bJR8/8XGzdU4fO7bLRtlUmAOClnwxH2+xMh60TBEFwBs977kSEy4cVAwAmDTmW7OrUPgUY1F3fYh6CIAh+w/PiDgRTAgT/y9hzQRAEwAdhGSCYYnfXwSP42RhZwFkQBAHwibi3ykzH7RP6O22GIAiCa/BFWEYQBEEIR8RdEATBh4i4C4Ig+BARd0EQBB8i4i4IguBDRNwFQRB8iIi7IAiCDxFxFwRB8CEUmSLXESOIagBsNXCKAgC7TTLHTMSuxBC7EkPsSgw/2tWTmQu1drhC3I1CROXMXOa0HZGIXYkhdiWG2JUYqWaXhGUEQRB8iIi7IAiCD/GLuM9y2oAoiF2JIXYlhtiVGCllly9i7oIgCEI4fvHcBUEQBBUi7oIgCD7E0+JOROOIaD0RbSSiGTZfuwcRfUxEa4loDRH9Uim/m4h2ENFy5W+C6pjbFVvXE9F5FtpWQUSrlOuXK2UdiWg+EW1Q/ucp5UREjyh2rSSikyyyqZ+qTZYT0UEiusmJ9iKiZ4iomohWq8oSbh8imqbU30BE0yyy63+JaJ1y7blE1EEpLyGiw6p2e0J1zMnK579Rsd3Q+pNR7Er4czP79xrFrldVNlUQ0XKl3M72iqYN9n7HmNmTfwDSAWwC0BtAFoAVAAbYeP0iACcp220BfAtgAIC7Afxao/4AxcZsAL0U29Mtsq0CQEFE2QMAZijbMwDcr2xPAPAuAAIwAsBXNn123wHo6UR7ATgdwEkAVifbPgA6Atis/M9TtvMssOtcABnK9v0qu0rU9SLOsxjASMXmdwGMt8CuhD43K36vWnZF7H8QwO8daK9o2mDrd8zLnvswABuZeTMzHwXwCoBJdl2cmauYeamyXQtgLYBuMQ6ZBOAVZq5n5i0ANiL4HuxiEoDZyvZsABepyp/nIIsAdCCiIottGQtgEzPHmpVsWXsx86cA9mpcL5H2OQ/AfGbey8z7AMwHMM5su5j5A2ZuVF4uAtA91jkU29ox85ccVIjnVe/FNLtiEO1zM/33Gssuxfv+EYCXY53DovaKpg22fse8LO7dAFSqXm9HbHG1DCIqATAUwFdK0Y3K49UzoUcv2GsvA/iAiJYQ0XSlrDMzVwHBLx+ATg7YFeIyhP/onG4vIPH2caLdrkHQwwvRi4iWEdF/iWi0UtZNscUOuxL53Oxur9EAdjHzBlWZ7e0VoQ22fse8LO5acTHbx3USURsAcwDcxMwHATwOoA+AIQCqEHw0BOy19zRmPgnAeAA3ENHpMera2o5ElAXgQgCvK0VuaK9YRLPD7na7E0AjgBeVoioAxcw8FMCvALxERO1stCvRz83uz3MKwh0I29tLQxuiVo1igyHbvCzu2wH0UL3uDmCnnQYQUSaCH96LzPwGADDzLmZuYuYAgCdxLJRgm73MvFP5Xw1grmLDrlC4RflfbbddCuMBLGXmXYqNjreXQqLtY5t9Skfa+QCmKqEDKGGPPcr2EgTj2ccpdqlDN5bYlcTnZmd7ZQC4GMCrKnttbS8tbYDN3zEvi/vXAEqJqJfiDV4G4C27Lq7E9J4GsJaZH1KVq+PVkwGEevLfAnAZEWUTUS8ApQh25JhtV2siahvaRrBDbrVy/VBv+zQAb6rsulLpsR8B4EDo0dEiwjwqp9tLRaLt8z6Ac4koTwlJnKuUmQoRjQNwG4ALmblOVV5IROnKdm8E22ezYlstEY1QvqNXqt6LmXYl+rnZ+Xs9G8A6Zm4Ot9jZXtG0AXZ/x4z0Cjv9h2Av87cI3oXvtPnaoxB8RFoJYLnyNwHAPwGsUsrfAlCkOuZOxdb1MNgjH8Ou3giORFgBYE2oXQDkA1gAYIPyv6NSTgD+T7FrFYAyC9ssF8AeAO1VZba3F4I3lyoADQh6R9cm0z4IxsA3Kn9XW2TXRgTjrqHv2BNK3R8on+8KAEsBXKA6TxmCYrsJwKNQZqKbbFfCn5vZv1ctu5Ty5wBcH1HXzvaKpg22fsck/YAgCIIP8XJYRhAEQYiCiLsgCIIPEXEXBEHwISLugiAIPkTEXRAEwYeIuAuCIPgQEXdBEAQf8v8VVma+u5/SHQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_episodes = 2000\n",
    "durations_list = []\n",
    "render_flag = False\n",
    "lr = 3e-2\n",
    "num_batch = 5\n",
    "\n",
    "def init_buff():\n",
    "    global state_buff, action_buff, reward_buff\n",
    "    state_buff = []\n",
    "    action_buff = []\n",
    "    reward_buff = []\n",
    "    \n",
    "def run(env):\n",
    "    actor = ActorNet(env.observation_space.shape[0], env.action_space.n)    \n",
    "    critic = CriticNet(env.observation_space.shape[0])\n",
    "    optimizer_actor = optim.Adam(actor.parameters(), lr=lr)\n",
    "    \n",
    "    init_buff()\n",
    "    for ep in range(num_episodes):\n",
    "        state_numpy = env.reset()\n",
    "        for t in count():\n",
    "            state_torch = torch.FloatTensor(state_numpy)\n",
    "            policy, value = actor(state_torch), critic(state_torch)\n",
    "            action_torch = policy.sample()\n",
    "            action_int = action_torch.numpy().astype(int)[0]\n",
    "            next_state_numpy, reward, done, _ = env.step(action_int)\n",
    "            if render_flag:\n",
    "                env.render()            \n",
    "            if done:\n",
    "                reward = 0\n",
    "                \n",
    "            state_buff.append(state_torch)\n",
    "            action_buff.append(action_torch)\n",
    "            reward_buff.append(reward)\n",
    "            \n",
    "            if done:\n",
    "                durations_list.append(t+1)\n",
    "                break\n",
    "            state_numpy = next_state_numpy\n",
    "            \n",
    "        if ep % num_batch == num_batch - 1:\n",
    "            calc_discount_reward(reward_buff)\n",
    "            optimizer_actor.zero_grad()\n",
    "            for state_torch, action_torch, reward in zip(state_buff, action_buff, reward_buff):\n",
    "                policy = actor(Variable(state_torch))\n",
    "                loss = -policy.log_prob(Variable(action_torch)) * reward  \n",
    "                loss.backward()\n",
    "            optimizer_actor.step()        \n",
    "            init_buff()\n",
    "        \n",
    "        if ep % 100 == 99:\n",
    "            clear_output()\n",
    "            print(f'Episode:{ep}')\n",
    "            plt.plot(durations_list)\n",
    "            plt.show()\n",
    "            \n",
    "    print('Final results')\n",
    "    plt.plot(durations_list)\n",
    "    plt.show()    \n",
    "\n",
    "env = gym.make('CartPole-v0')\n",
    "run(env)\n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}