jskDr/Actor-Crtic_no_detached_pytorch.ipynb

## Actor-Crtic_no_detached_pytorch.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from IPython.display import clear_output\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "from torch.distributions import Categorical, Bernoulli\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import gym\n",
    "\n",
    "from itertools import count\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorNet(nn.Module):\n",
    "    def __init__(self, state_size: int, action_size: int):\n",
    "        super(ActorNet, self).__init__()\n",
    "        self.fc1 = nn.Linear(state_size, 24)\n",
    "        self.fc2 = nn.Linear(24, 36)\n",
    "        self.fc3 = nn.Linear(36, action_size)\n",
    "        #self.fc3 = nn.Linear(36, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        \n",
    "        prob = F.softmax(self.fc3(x), dim=-1)\n",
    "        policy = Categorical(prob)\n",
    "        #prob = torch.sigmoid(self.fc3(x))\n",
    "        #policy = Bernoulli(prob)        \n",
    "        return policy\n",
    "    \n",
    "class CriticNet(nn.Module):\n",
    "    def __init__(self, state_size: int):\n",
    "        super(CriticNet, self).__init__()\n",
    "        #self.state_size = state_size\n",
    "        self.fc1 = nn.Linear(state_size, 24)\n",
    "        self.fc2 = nn.Linear(24, 36)\n",
    "        self.fc3 = nn.Linear(36, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        value = self.fc3(x)\n",
    "        return value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_discount_reward(reward_buff, gamma=0.99, normalize=True, done_flag=True):\n",
    "    prev_dr = 0\n",
    "    for ii in reversed(range(len(reward_buff))):\n",
    "        if done_flag and reward_buff[ii] == 0:\n",
    "            prev_dr = 0            \n",
    "        else:\n",
    "            reward_buff[ii] += prev_dr * gamma\n",
    "            prev_dr = reward_buff[ii]\n",
    "    \n",
    "    if normalize:\n",
    "        mean, std = np.mean(reward_buff), np.std(reward_buff)\n",
    "        for ii in range(len(reward_buff)):\n",
    "            reward_buff[ii] = (reward_buff[ii] - mean) / std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:99\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nOy9eZgcd3Xv/T3Ve/d0zz7SaGa0jCTL8qLNsmW8BdsxMTgXJybBGF7iJOR1eGMSEnhvQu7NfcNzSQIJF0hCEicGE2wWE4LtBwM2AYxZDLaMJFuLJdnapVk0mr1npvfq3/tH1a+6urqquma6epme3+d59Gimuqf71z3Tp771Ped3DjHGIBAIBILmQqr3AgQCgUDgPiK4CwQCQRMigrtAIBA0ISK4CwQCQRMigrtAIBA0Id56LwAAurq62Pr16+u9DIFAIFhW7N+/f4Ix1m12W0ME9/Xr12Pfvn31XoZAIBAsK4jonNVtwpYRCASCJkQEd4FAIGhCRHAXCASCJkQEd4FAIGhCRHAXCASCJqRscCeiASJ6noiOEdFrRPRB9XgHEX2fiE6o/7erx4mI/pGIThLRISLaVe0XIRAIBIJinCj3HIAPM8a2ArgewINEdAWAjwB4jjG2GcBz6vcA8FYAm9V/DwB4yPVVCwQCgcCWsnXujLFRAKPq13NEdAxAH4C7AbxZvdujAH4E4M/U448xpZfwS0TURkS96uMIBAKBY+Q8w7//7AziySwAwOeR8H9dvw7tEb/lz/zg6BgODc2Y3nbT5m5ct6Gj6NjJS/N4+uAIoLY/39jTgrt39BXdZzaRxZdeOotMLl/ymEG/B79zwwaE/J5FvbZqs6hNTES0HsBOAHsBrOIBmzE2SkQ96t36AFzQ/diQeqwouBPRA1CUPdauXbuEpQsEgmbn+MU4/uo7x4qOdUUDuO8665jxv755BKOzKRAVH2cMeOHkBJ78gxuLjj/ywmk8/vIFECn3kQh4+/Y1IN0D/ODYGP7P994AgKLH5eMwLl8dxW2Xr1rCK6wejoM7EbUAeALAHzPG4mR853R3NTlWMhGEMfYwgIcBYPfu3WJiiEAgKGE+lQMAfOX39uDq/lZs++j3sJDO2f5MIiPjt29Yj4++/cqi4x/46gEcHY2X3H8mkcXmnhZ8/0O/hId+dAp/+93jSGXzRUp8IaM85/6/+GV0tgS046fG53H7p36MuZT9muqBo2oZIvJBCexfYYw9qR4eI6Je9fZeAJfU40MABnQ/3g9gxJ3lCgSClUQiIwMAwn4PQj4l2CbVY1akczIC3tLQFgv5EE+WBuF4KotYyAcACPmUn0tmi5+DP6fReokGFH28LIM7KRL9EQDHGGOf1t30NID71a/vB/BN3fHfUqtmrgcwK/x2gUCwFHhwjwS88HkkeCUqCbx6GGNIZfMI+Er971jQh3gyC+No0dlkFrGgEqR58C4J7ur3QW/x47aoPzdf5mqiHjixZW4E8F4Ah4noVfXY/wDwCQBfJ6L3ATgP4DfV254B8DYAJwEkAPyOqysWCAQrBm6HhNWgG/J7tIBvRkZWEp5myr015ENGziOdyyOoC/7xZA6bulsAQDtuvDpIZpWrAUkqdp1DPg8kQlmrqB44qZZ5AeY+OgDcbnJ/BuDBCtclEAgESKR5cFeVtc+DlI1yT2WV4B40U+4h5TFmk9ni4F5ky3jUxyl+jlRGNq2GISJEAt7lacsIBAJBvVjQee78fztbJq3eZuq5B5UAzssqASCfZ4gns9ptdrZMyOSEASi+eyPaMiK4CwSChiWZkeGRSAvWQZ+9LZPOWSv3VlWdx1OF4L6QySHPCrfxk4jxORIWyh1Q8gGNaMuI4C4QCBqWhUwOYb9HqzkP+8vZMjbKXQ3gszrlHlftFG7ZWHnuKRvl3hIUyl0gEAgWRSIta2oaKJ9QtVPuvCJGXw45m1ACfWsZz93OlmkRnrtAIBAsjoVMDhF/oe4j5PPa1rnzoBz0mVfLAMW2DP+6rOduY8u0CFtGIBAIFkcyIyMcKFbutgnVHC+FLA3EUTWAc7UOFJKrxmqZ0lLIvOnVAKAEd2HLCAQCwSJQPPeCcg/7PEtW7n6vhJDPU6Tcuf/OVb3muRtLIct57sKWEQgEAuckMjIiJZ67dSC1U+6AEsRNE6qqqg94JUhUqtwTmZyt5z6fyZXsfK03IrgLBIKGZSFdrNxDfo+2UckMO+UOKFUxRQlVNdDzNgJEhJCv1Pop57kzVlo+WW9EcBcIBA1LMmOolvF5kJHzyMnmAZ4HfivlHgv6ihOqySyiQS88urYCZr6+sUuknkbtLyOCu0AgaFgWMjIiAX21jLknzknn7JV7qS1T2J3KCfo8SOlUeE7OIyPnbW0ZoPE6Q4rgLhAIGpaEuomJo5UqWlggZZV7qFS582Sq9hwGWyal+vjlgnujlUOK4C4QCBqSTC6PrMxKbBmgvHI326EKKMpd77nHkzltdyrH2L+Gn0iCNp47IGwZgUAgcERSaxqmK4W02GTESWXz8Ju05uXEgl7EU1nk80pli5Uto0+OaoM6LJR7RNgyAoFA4Bzeyz2i28QUtGjsxbGawsSJhXxgDJhXH3vWzJYx9K/hJ5KwhXKPBoUtIxAIBI5JZIp7uQPKJiYARQlPPSmbnaSArnmYuks1niz0cueEDBuleHAv57kLW0YgEAgcsJAuVcyhSpV7sNBfJifnsZCRS2wZY0JV89zL2DLLLrgT0ReI6BIRHdEd+w8ielX9d5aP3yOi9USU1N32r9VcvEAgaF4SS/Dc02WVe6EzJN+d2mpIqAYNtgz/2qrOPeCV4PNQwwV3JzNUvwjgnwA8xg8wxu7lXxPRpwDM6u5/ijG2w60FCgSClUnCzHO3aOzFSWXtlXurrqe7sWkYx9i/ppwtQ0RKC4IGS6g6maH6EyJab3YbKR303wngNneXJRAIVjoLpspd+dq6FLKMctfZMsZ2v5yQ34NEVgZjDESkXUFYBXdAsWYaTblX6rnfDGCMMXZCd2wDEb1CRD8moputfpCIHiCifUS0b3x8vMJlCASCZqMwHLu0zt3Kcy+n3LlKjyezhY6Q4dJSSMYKTciSZWwZoDHb/lYa3O8D8Lju+1EAaxljOwF8CMBXiShm9oOMsYcZY7sZY7u7u7srXIZAIGg2uHLXD+vggXupyj0a8IJICe58M5NZQhUoeO28MscuuEcbsO3vkoM7EXkB3APgP/gxxliaMTapfr0fwCkAl1W6SIFAsPJIqp67PqhKktq10aLtbznlLkmEaMCLeCqn2TJmde5A4QTC/w/aPG6zKfdfBnCcMTbEDxBRNxF51K8HAWwGcLqyJQoEgpXIQkaG3yPBbwiqdtOYyil3QLFmZnW2jLH9gHEaUzKrrMPrsQ6XkQYcteekFPJxAC8C2EJEQ0T0PvWmd6HYkgGAWwAcIqKDAL4B4P2MsSk3FywQCFYGiXTO1ApRlLtVy1/ZsiMkR+kvo1TLeNUrgaLHNyr3TPnHjAa9mGuw4O6kWuY+i+O/bXLsCQBPVL4sgUCw0lkwTGHiKMrdPJCmc3nLjpAc3tM9nlJaDyhFf7rHNyp3m0EdnEYshRQ7VAUCQUOiDMcu1Z9hv/Uc1VRWRqCMyo6FvKotkyupcQfMPXd9OaYZkYAXyawMOd84o/ZEcBcImpRfnJ3C9X/zXFH/8uXEQiZnqtyNXRs5jDFHyp23/Y0ns4gFS4O2medezsdvxP4yIrgLBE3KibF5XIynMDGXrvdSlkQiba6Yw4b2ABxel17OH9fbMmbKPWjoGZ/Kygg58NwBEdwFAkENyKiDK7Jy41gFi2HBMIWJE7JQ7ukyU5g4sZAPiYyMyfmMrS2Tyjr33CMNOI1JBHeBoEnJqEOkMznzypJGx8pztyqFLDc/lcPr2kdmkiU17kChrbDelrFrPQA05hxVEdwFgiaFB3Ue5JcbVp67sd86p9z8VA6va8/lWcnuVEDXVrioFNL+MYUtIxAIaoYW3Jepcrfz3CtR7vqAbtzABBRaHKQWodyFLSMQCGpGWlXs2WWo3Bljtp57Uu3aqIcr96CDahmzrzlEVDSwQymFdFgtI2wZgUBQbZazck/n8sgzIBwwKYX0F3dtLPyMEozL17nrlLuJLQMU+/rJjKzNbrUiGlAep5F2qYrgLhA0KTyoV1O5P3N4FM8dG3P9cRMmHSE5YYu2v5pyL9dbpsiWsQjuaouDfF6pnS9vyyi3N5JydzKJSSAQLENqkVD9lx+dRCzow+1bV7n6uAsmvdw5xh2kHF66aNcVEihvy/DnSGVlpHLlB3UAgNcjIeiTsGDRrbIeCOUuEDQptSiFTGZk0w1FlWI2P5UT4tOYDMq9sInJPhAHfcrMUwCmO1QBXkufK0xhKmPLAEBLwKeVQjLG8MWfncHF2VTZn6sWIrgLBE1KLZS7Etzdf3w+P9XMcze2B+A4Ve5EpCl2W1smK2vPUe6EAQAtAY9WCnlqfB4f/dZRPPri2bI/Vy1EcBcIXCAn5/HkgSHkG6hxlOa5V1O5Z6ur3E09dwtbxqlyBwq+u1VCNej3IJnNa6+tXLUMALQECz3d95+bBgAcUP+vByK4CwQu8PKZKXzo6wfxyoX6fZiNaLZMFZV7okq2jJ3nHtQSqsX+tlPlDgDRkA8hn6dkEAgn5JOQysiF+amOlHuh7S8P7geHZupWiiqCu0DgAlxpWg1urgdprVqmOlcTslpJkqrClUHBcy8NqmFD7xfOYpR7a8hnmUxVnsNbZMs4De5zOuUe9ElIZfM4Nhov+7PVQAR3gcAFsg3Yx4WvxVgP7hYpXddEt9FsGbPeMpalkM6V++aeFmxe1WJ5e5B77nx+qqOEqmLLTC9kcGp8Ab95zQCAgoqvNU7G7H2BiC4R0RHdsY8S0TARvar+e5vutj8nopNE9DoR/Uq1Fi4QNBLc+qhWIF0K1a5z58E1ZbJbtPLHtrZl7Dx3v1cqmaxkxl/ctRWP/s51lrfz/jWLUu5BZUg2t+bu2taLNa3Bxg3uAL4I4E6T459hjO1Q/z0DAER0BZTZqleqP/MvfGC2QNDMNOJu0GqXQnKlnGfuWz8LaetSSK6izaplgg5UO6BUzEiS9Ukg5JeQzMqFUkgHwT2ieu77z03DIxG297dh17r2uiVVy74TjLGfAHA65PpuAF9jjKUZY2cAnARgfXoUCJoEHtz4FvhGoFbKHShV0ZU/dg4BrwSPSQC2KoVM52QEHARhJ4R8Hsh5hjl1ipWTaplowIuMnMdLp6dw5ZoYQn4PrlnXjpHZFEZnk66sazFU4rl/gIgOqbZNu3qsD8AF3X2G1GMCQVPDB2M0lHKv8tWEvlol7Xpwl039dgDweZRNSAmjLZPNl+0I6RSelJ1KKMHdqecOAK+cn8autUpIvGad8v+BczOurGsxLPWdeAjARgA7AIwC+JR63Ow6x/R6jYgeIKJ9RLRvfHx8icsQCBqDgnJvoOBe5VJIvVp3eyOTVUdITtCkp3sqJ5ft5e4UbgdNL2QAOLdlAMWm4kF9a28MQZ9UF999ScGdMTbGGJMZY3kAn0PBehkCMKC7az+AEYvHeJgxtpsxtru7u3spyxAIGoZGTqhWS7nrg2vKZTsqkZZNNzBxzOaouqncQ37lcaYSGXglgs/joHZe18qAB3efR8L2/jbsP79MgjsR9eq+/XUAvJLmaQDvIqIAEW0AsBnAy5UtUSBofBoyoVplz71Yubsb3BcyOdPWAxyzOapuKneu1KfmM45UO6D0lgGA3tYg1rSFtOPXrGvHa8OzVSkZtaNsV0giehzAmwF0EdEQgL8E8GYi2gHFcjkL4PcBgDH2GhF9HcBRADkADzLGGifDJBBUiWyDKXfGWNWrZfTB1W1bJpGxH5ARUjcZ6amK576QceS3A4W2v9xv5+xa245cnuHQ0Cyu29DhyvqcUDa4M8buMzn8iM39/xrAX1eyKIFgudFom5j0Pnu1dqjqlaj71TIyOiJ+y9tDPsnUc7fbdboYNOWeyDiqlAEKTch2rTMEd/X7feemahrcxQ5VgcAFCh0YG+NCVX+SqY1yd78U0mw4NidsodzLTWFyCm/xO73g3JYZ7Irgb99xNe69dqDoeEfEj8GuSM0rZkRwFwhcIMOrZarQ/nYpFAX3GtS5u+65p2WELUohAcU2MfPcy81PdQpX67k8c9SrBlA2Rt177VqtJFLPrnXteOX8tOs7ee0QwV0gcIFa9E5fDPp1VHuHKuD+SS2RyWnj9MywqpZxS7nrA7pT5W7HNevaMbmQwbnJRMWP5RQR3AUCF2g4zz2n99yrt4mJt3FxsxQyn2dIZu2VO5+UpCeVdb9aBnA2hakcvDSylvXuIrgLBC7QaNUyfB1eiaq3iSmT1xKYxuRmJaRyMhiDrece8pttYnLfcwfcUe6bulsQDXprWu8ugrtA4AKNVufO19ES9FZtElMym0N7WKlocbMUUmsaZqfc/Z6ihCpjDJlc3jXPXf84Tj13OySJsHNtbZuIieAuELhAYYdqY1TLcOUe8Xu1ZK/bJDIyokEvfB5y1ZbR2v3aBNWQz4OszEqumNxS7pJEWl94p6WQ5bhmbTteH5tDXG1GVm1EcBcIXKBRlXs06NWamrlNMiMj6PMg6C1NblZCYVCHfUIVKNTX84SuW8pd/xxueO6A4rszBrx6vjYlkSK4CwQu0GieO7+SiAS8VdvElMwqu0gDPo+rtkxhUId9KSQApPjAEPUE5pZyBwpeuxu2DABsH2iFRLVLqorgLhC4AA+gjabcW9Qe49UgqbYICPokV1v+zqcbQ7nztgNuJFQBIBr0YcvqGA7UKKkqgrtA4ALVnle6WPTBXc4zyHn31XuC2zI+j6vtB+ZTinKPBq1bCRjnqHLl7pbK1j9HyMWrgV1r2/Dq+Zmq/D6MiOAuELhA49kySrDjuyWrUevObZmgT3LVc+fTj8x2enJCFsrdyXBsp2jB3SXPHVB897l0Dicuzbn2mFaI4C4QuEChA2NjVMtw5c4HSFTDmklmZIR8HoRc9tzn01y5229i4msAqqTctYRq2f6KjqnlZiYR3AUCF2i49gO6Onf9927Bd5GG/F4EfR5XSyHjKWXnq/2wDuU2Lbhnq5dQdctzB4C1HWHEgl4cHYm79phWiOAuELiA3papZXMoK9Ka564EJrdtGf74Yb8HAa/Lyj2VQ4vfC8lkODaHT0pKVDGhGnI5oQoozcU6WwKYTVa/1l0Ed4HABbgyZkzpJFhv+BUEnw7ktnLn5Yohn/vVMnOprHbFYQW3SmpRCslPJG4RC/kQT+XK37FCRHAXCFwgKzN4VKXZCOWQBc+9OsqdV6mE/FWolknnbP12QF8towTJqpRCulznzokFvUK5NwI5OV/z2YeC5QUfacerOxqhYiaTy8MrkdYl0e018c8EV+7uVsvkbCtlgEKdO1fAVVHuVbBlAEW5zzVCcCeiLxDRJSI6ojv2SSI6TkSHiOgpImpTj68noiQRvar++9dqLr4WfOYHb+Cd//ZivZchaGD4BiYekBpFufu9klYa6PYuVa7cw373q2Xm0jnbGndAUdN9bSG8MaaUFFal/YAa1O12yi6F1pCvJv1lnJzmvgjgTsOx7wO4ijG2DcAbAP5cd9spxtgO9d/73Vlm/RieTuL8VO0a7AuWH1m5sGEIaIzmYRlZCe4+j/IRd/uEw20YbssobXrdOYE48dwBYGtvDMdGlaqTaij3trAPHolsd8ouhVjQh9lktuqJ97LvBGPsJwCmDMe+xxjjGYGXAPRXYW0NQUbOu9qrWtB8aMG9SmWHSyGTy8PvkeDXlLvLwT2jt2U8YMy9MtD5VA7RMrYMAFzRG8WZiQWksnJVNjG945p+fP33ry97FbFYYiGl34+bVztmuPFO/C6AZ3XfbyCiV4jox0R0s9UPEdEDRLSPiPaNj4+7sIzqkMnlkc7lkW+ACghBY6Lf6g80jueuKPfqJHkLtoxXC6huBau5VPmEKqAo9zwDXr84h1RORsArgci6fHKxhP1eXLOuw7XH4/ABJ9VOqlYU3InofwLIAfiKemgUwFrG2E4AHwLwVSKKmf0sY+xhxthuxtju7u7uSpZRVfgH1c1qAEFzkTEo90YI7mmjLeO2cs8WK3fAnSHZOTmPZFbWSjjtuGKNElqOjcaV+akuqvZqElOvBKrtuy/53SCi+wH8KoD3MNU8YoylGWOT6tf7AZwCcJkbC60XGRHcBWXQlLu/8WwZHvBc99x5nbuaUAXcCe5OWg9wBtrDiPg9SnDPya6XLFYLrtzjjajciehOAH8G4O2MsYTueDcRedSvBwFsBnDajYXWC654hO8usEKrlglWr4/LYsnkFCXLlbvrnnu2UC1TUO6VP8ecWtroJKEqSYTLe2M4NjqHVNa9+anVJtYotgwRPQ7gRQBbiGiIiN4H4J8ARAF831DyeAuAQ0R0EMA3ALyfMTZl+sDLBKHcBeUoqZZpgL8V7rn7q6TcueceVOvcAXeUOw/uMQfBHQC29kZx7GIcqazsahlkNeGvrdq2TNl3kDF2n8nhRyzu+wSAJypdVCPBPxQJodwFFqQNCdWGUO5yHkFfdZV7wCvBI5Grnju3ZZx47oCSVP3yS+dxenxh2Sj3gi1T3RYEy+PdqCPClqkfE/Np/Ok3Djb8DmFjKWS6yiVuTjCWQrqd5E1mZG0Hp6bcXXgO3svdiecOKMEdAN64NLdslDsvray7LbPSKdgy1W/0IyjmF2em8PV9Q9pGlUbFaMs0hHLntoynejtU+Q5O3uLADQGkKXeHwf3y1VEQKQ3blktC1e+VEPJ5GjOhupLQgnum/h/YlQYPktXe7FEp1e6dvhSUHaqesp57Ts7jk/91HJPz6UU9fjIrF2aM+nn/msqDezzlvFoGUGrR13dGALi7gana1KIFwfJ5N+pEwXMXyr3WcHvDzUEQ1YAr92gjtR9QbRmPRJDI2nN/fWwO//z8KXzv6NiiHp8PxwbgrufOg7tDzx1Qkqr6dSwHYqHqd4YUwb0MaU091v8Du9LQ3vsGz3dwPzvSQI3D0qotAyg2gJVVNL2gBJiJuUUq94yMsE95vUEXd6jOpbLwSqT5+E7Yulrx3ZeTco8FfSKhWk8YY6IUso7wksLGV+6Kn83HwjXCDtWMuh0fAHweyfKEM7mgBPVLiwzuCZ0t43a1TEvQu6g2AjypGlhGyl3YMnVGn4QSpZC1hwfJRvfcueXBW+w2gnLnXSEBRdFaK/cMAGB80co9pyVUeXB3QwA57SujZ+uaZajcQz5hy9QT/QdCKPfak8ktD0uMr9PnIfi9UoMo97xWKePzSMharGkqoQSY8SUkVHki1SMRfB5ybYeq0xp3zprWIK5cE8OW1dGKn79WtIZ8Va+WcbcLfZOhV2Cizr32LJembVy5+7wSAl5P3YN7Ts4jzwpK1u+VLBOqU5otk1rUc+jr3AFFvbtjy2QXrdyJCN/5I8sGtA1JLOjFXDqHfJ7ZDgKvBKHcbRDBvb5kloktw6/weKOuetsyGZ1NBKiee5mE6vhcelHDI5K6OndACe5uVAnNOezlvtyJhXxgTJk6VS1EcLdB/yFNNLh6bEZ4sGiEXi128L8TviO03puYeAmpVi3jkZDJmQfuKdVzT2Xz2gaicjDGkMgalbvkykmYJ1SbnVgNOkOK4G5DRi4ElUYvx2tGlpMt45UIkkQIeKW6n4xKlLvNCYcHd8B5UjWdy4MxFAd3rzu2zFISqsuRWA1aEIjgboPeOxXVMrVnOSVUeYOuRlDu+isJAAjYJlQzGOgIAXBeDqkfsccJ+jyunITnl5BQXY7EQtXvDCmCuw1FnnuDB5hmhNsyje65Z2WmjbPzeyTHjcNSWRlyFcY3clFSUO5kesJhjGF6IYMtq5QqE6fKPaHr5c4JuZBQTWVlZOT8ilDutegMKYK7DTy4RwNekVCtA8tGuat9XAAg4HOu3G//1I/xxZ+fdX89ueJh0X6PebVMPJVDLs+0EkKnwV1T7v5CEA644LkvZgrTckcbtSdsmfrAP6SxkE8o9zqwXDx3paa8oNydVMuksjKGZ5I4N7ng/nrMqmVM1sQ3MG3oaoHPQxXbMpWehOcW2TRsOdMarv4cVRHcbeAfiFYR3OsCD+6N0B/djqych0/bDeqsJJAHMqcVKouh4LkrwdcqDzCVUIJ7Z8SP7paAc1tGbaIX9htLIStU7qnFDepYzrT4vSBqAOVORF8goktEdER3rIOIvk9EJ9T/29XjRET/SEQniegQEe2q1uKrDf+QtIV9wpapA5ot0+C9ZfS7Qf0O69y5YluoZnAvKoW0Vu4dET+6owHHu1S50NF3YQx6JReU++IGdSxnJIkQDVS3M6RT5f5FAHcajn0EwHOMsc0AnlO/B4C3QhmMvRnAAwAeqnyZ9YGrHa7cF7PJQ1A5XAE3+ok1KxdXyzhRsFyxLaTdf228hNdfZofqZFFwDy7aczcq90qvbue0EXvNH9wBxZrh/eurgaPgzhj7CQDjoOu7ATyqfv0ogF/THX+MKbwEoI2Iet1YbK1J65S7nGd1L3FbaaSXi3KXWXGTLkfKvRa2jDPPvZ0rd4ctCJJm1TJ+Z577D4+P4d5/e9H0voXh2M1vywC87W/9lbsZqxhjowCg/t+jHu8DcEF3vyH1WBFE9AAR7SOifePj4xUso3rwDwTfTZYS05hqyrJpP5CTF2/LJKtnyxhLIRXlXnrVOZXIwO+VEPF70B0NYHIhg5wDAZMwS6h6lWqZcle3L56axN4zU3jm8GjJbfOqLbMSdqgCSnBvBFtmMZh1wSn5jTPGHmaM7WaM7e7u7q7CMipH89xDfgBAQsxRrSnpZVIKmZUZfF61WsapLVMDz71cP/ep+Qw6wn4QEbqjATBWvGPVikIpZCG4817q5V47t34ee/FcyW1zqRVmy1S5p3slwX2M2y3q/5fU40MABnT36wcwUsHz1I2sznMHGt/7bTaWS517Vi4kVANeDzJyeQXLN69UxZaRS5W72ZqmExm0RxTh0hMNAHC2S5XbMsZSSKB8ZdPEvHLyePXCDA4NzRTdNp/OIeCVtHU3O7GQt2E3MT0N4H716/sBfFN3/LfUqpnrAcxy+2a5oa+WAUQLgjlQWEAAACAASURBVFrDE6pZmVVlJ6db6NsPcLVcLj+jKfeM+4l6o+fOa/CN1szUQgadanDvVoO7k6RqIqPYUF5PIXzwsXjl8iPjc2m8abATYb+nRL3HUzlEV4jfDiiise62DBE9DuBFAFuIaIiI3gfgEwDuIKITAO5QvweAZwCcBnASwOcA/IHrq64RGTkPosJlYqMryGYin2fIykxL2jXye58pqnNX/i9nT3DPXc4z1/u/l5RCqv8bK2amFgrKvbvFeXBPZeWSGadcxZe7up2YT2NDdwT37OrD0wdHimyg+fTKaBrGiQWVKrxqtYh29E4yxu6zuOl2k/syAA9WsqhGgdcv8wAjlHvt0JehJjIykllZG0DdaGRyeQQ8xYG03AdWXwI3n84V1Yy7sR79WvhVRSaXRyRQuN/UQgYd6lWpptwd1LonMjmE/cW/C22Oqo1yz8l5TCUy6G4J4K5tvfjyS+fx9X0X8P5f2ghAqXNfUcFdtXvnUll0tgTK3HvxrAxza4nwCfI8cSR2qdYO7t3yfEcjK/eiOnfP4pQ74H5SlV9xeqVCkpevU7/meCqHDjXaB30exIJeXIqXL4dMZvNFZZDKz6u2jI3nPrWQAWNAVzSAy1ZFcf1gB7704jnNclM6Qq6c4M7/tqtlzYjgbkNGziPglbRLzkYOMM1GWt2Io5WhNnA5ZFZf5+5zqtwLH2i3k6r8ipNICe4+kxPOdIJvYCp43E53qSYzpVcaQW/5zwhP1nIL6D171mF4JolfnFW20MytsOBeaPtbnaSqCO428A9JSNgyNWc5Kfeifu5qP5eywT2Z1V6b27tU+RUnJ2Ci3Pl4Pe65A2pwd5hQNSr3gAMBNKGeOLqjynPednkPAl4J3z1yEQD33FdOQrXaAztEcLcho35Iwj7lDCtKIWsHV5nLIrjL+aI6dwBlm4fFUzn0tgYBVMeWCeiCu+a564L7lK71AKcnGnRcChmytGWsX/e4ptyV1x0JePHmLd149sgo8nmG+Arz3FurPGpPBHcbuCIL+pW3SXjutSNTEtytlfDPTk5onQprDWMMWbmQUA04Tagms1jTpkxAqpYtw+FfZ3VzVM2Cu1PlnszIRTXuAHTWpfXr5jXuXdHCc771ql6MxdM4cH565VXLhKrb9lcEdxuUIQwS/B4JHomEcq8hXPmWU+5j8RTe8/m9eOLAcM3WpieXZ2AMRY3DAPvgnsrKSOfy1VPuBlvGp9XeF95D3u63I1wc3BMZuex6Jhcy2u+FE3Rgy4zPpRHxe4oqbW7b2gO/R8ITB4bA2MrZnQqIhGpd4R8SIkLI5xGeew3R+vqoSs7qqun8VAKA8ylCbsN97MXUufNt9lVV7t5S5Z7RKXd90zAOT3TaWTOX4imMz6VxxZpY0XEnwX1iPo2uaHHJXyzow82bu/Ctg8o+x5XkuQdU4VitXaoiuNugv7wN+d0ZACxwhua5h+2V+/B0EkAhWNWakt2gDoI7vwwvKHd3/674FSfHr+YDjJ57NOjVrjgAoCdWfiPTayNxAMBVfa1Fxws7VK1f9/hcWjuB6LnzqtXaCW6lNA0DACJSWhAIW6b2pHUfkpDPg2SdfN2VSElC1SJoDE0ryp2X9tWajKVytw7YPIHWHvEj4JWw4PLfVannrqjqbK44uOv9dsBZC4LDw7MgArb2GpS7g1LIifk0ukyC+x1XrNJq8leS5w4ovnu1bJmV9U4ukkyuUHUQFsq9phgTqmkr5T6jKPeZRPV6dNjB+7UEdI3DAHvPPa7rW94S8FbdlvGZKPfpRGlwXxVVriQuqCdMM44Mz2JDV6TEG5ckgt8j2X5GxufTuH6ws+R4W9iPGzZ14SdvjCO6gjx3APiHe3dW7YQmlLsNmZysfUiCwnOvKcaEqlUye0i1ZZy0qq0GPIgbSyHtGodx5d4a8iIS8LqeUFWuOAvVLFq1jMGW0SdTAeVKYrA7ghdPTVo+9msjcVy1ptX0tqBPsuwKmcnlMZPImip3APjVbco8n+6o+9vwG5mr+1uxvitSlccWwd2GjK6Va8iF6e4C5/CgGQl44ZHIsmcJ99xn6mTLaAlVY/sBm5JA7rHGgr6qBHejLaPvLcPRNw3Tc9OmLrx8ZsrUVppayGB4Jomr+mIltwGKAOKfkXyeFQ3+mFzgG5jMg/dv7OrHtz5wE9Z1VifQrUREcLdBf3krbJnaktYlKvmUHyOMMc2Wma6TLWNMqGrtB2yVu2rLhHxoCXiqYMvIRZuYjG2IGWNF7X713LSpC8msjFfOz5Tc9trILIDSZCqHB3fGGB786gHc89DPtdsm5tQa95bS5wQUW+fqfvPHFSwNEdxt0Af3oF/YMrWEK8eAz2M5fHl8Po10Lo9VsQCSWbkuV1bGhKpT5e73SAh4JVW5V7daxqdtYlLWlFTr7M2U+/UbO+GRCC+cmCi57fCwEtyvtLFlUtk8vvjzs3j2yEUcGprVrqjG55WGZCvNdqknIrjboFzeKt5l2OdBSgT3mqEfFRe0sMS4JXO1qiTrUTGTNSh3r7rhTb9hyEg8mUUs5AUR1caWMSj3yfnS3amcWNCH7f2teOFkaXB/bTiOtR3hkg1MnKDPg6OjcXz8meNY1xkGABwaUk4IBeUugnutEMHdBr0CCvk9SAhbpmakc4W2tVaJOm7JcCXJm2HVEuNIO0AJ9OWqZXjTqBZ/9atlCglVpbKHJ5/bw+YWyU2bunBoaKakRO/IyKyl3w4owf38VAIdET++9Lt7QAQcvKDYO+Pz9p67wH1EcLeATwLSB3fRfqB28DJUIrJU7rxShnvA9UiqGhOqQPkh2fFkFlFV/VZNuRfZMkolD18T34G6KmYeaG/c1IU8Q1HVzGwyi3OTCUtLBlCKDiQC/uFdO7C2M4zBrggOqnNSx+fSiAa9rg4lEdiz5OBORFuI6FXdvzgR/TERfZSIhnXH3+bmgmsFV2QB3SamdC7f0LM8m4l0rrhSycxzH55OIhb0Ym2HYgFM1SG48y39ehsk4C2n3LNaW4WWgAcLGRl5F/+u0rnirpBESg06PxFdnFVOiqvVHbJGdq5tR9jvwc901sxRi52pen7/lwbx2ft2YY9ay759oA2vXpgFYwzj8+a7UwXVY8nBnTH2OmNsB2NsB4BrACQAPKXe/Bl+G2PsGTcWWmu0y21PoVoGaOzWs81EOidrPcItPfeZJPrbw2hXWxTUo2KmYMuQdsxfLrgns1pHQD460C3LL59nyOVZkXIHFPXO1zQ6m4JXInRFzIOt3ythz4aOouCuVcqssbZlbtjYhbvUenUA2DHQhon5NEZnU5iYK+0rI6gubtkytwM4xRg7V/aeywTjHEre0lRUzNQGvfrkVRhGhqYT6GsPoU31jmfqsJGJJ1QXZcvoPHce3CuxZn5+agJ/+c0jAMxzAPz7gnJPYVUsCEkiWHHT5m6cnljQ8hqHh2expjW4qFmf2/rbACi+u1Dutcet4P4uAI/rvv8AER0ioi8QUbvZDxDRA0S0j4j2jY+Pu7QM9ygJ7mqbUqHca4N+mlDARLkzxjA8nURfWwh+r4SWgLfOyl1vy3jKeu58xBrfxl9JUvWxn5/Doy+ew8XZVNH+AD0+XZJ3dDZlaclwbtrUBQB44LF9eO8je/HD45dwpY0lY8bW3ih8HsKrQzOYmEuLZGqNqTi4E5EfwNsB/Kd66CEAGwHsADAK4FNmP8cYe5gxtpsxtru7u7vSZbiOcXOKUO61RUmoKu+52e7g2WQWCxkZ/e1K29y2sK8+pZCWCVXzvxPey92pcv/mq8P40otnLZ+fMYaX1Rmkr16YKSoh1eP3StqJ6GK8fHC/bFUL7tnZB79Xwnw6h809Lbh394DtzxgJeD24ojeGX5yZQjyVs9zAJKgObnSseSuAA4yxMQDg/wMAEX0OwLddeI6aY1RkITGNqabolXvQJ5V0heSVMjy4t4f9dQnuxis8wD6hynu5Fzx35QRmpdwf+tEpHL84h3gqhwdv3VRy+8lL81pp46GhGW2XZ4ktoyp3xhhGZ5O4/fIe29dFRPj0vTts7+OEbf1t+PJexa0Vyr22uGHL3AedJUNEvbrbfh3AEReeo+aUeu5ijmotSWcLW+iD3lLlzoN7X5tSKdMe8dfXljFWy1i0Hyj0lSm2Zcx2qebzDGcnFxANePHJ/3od//bjUyX3eemMotq7Wvw4ODRjerLh32flPOLJHFLZfFnl7hbbB9rA1EIgsYGptlQU3IkoDOAOAE/qDv8dER0mokMAbgXwJ5U8R71Il3juisJKZkVP91qgH/LMB6UwVigX5Im+gnL31WVgB59LWmTLeKy7I/KOkMZqGTNbZmwuhVQ2j/9+5xb8t+1r8PFnj+PRn58tus/e05NYFQvgLVeuxqELs9pJkO+s5nDPfTSuvG+9raHFvtQlsWOg4NML5V5bKgrujLEEY6yTMTarO/ZextjVjLFtjLG3M8ZGK19m7dG8S0MpZDJjP/hYYM3ZiYWi8jo70ll9tYwHjBU34xqaTiDs96BNLYOsly2TlfPwSASPrvIk4JOKmnQ9eWBIU+z6Xu6AfUL1zPgCAGBTdws+887tuHVLNz7x7HFt5yhjDC+fmcKeDZ3Y0d+GuXQOb4zNAbBS7gyjs0qPl1op98GuFu01CuVeW8QOVQtKPHctoSqUO+cXZ6fw0xPjRa1d7fjsD0/iDx9/xdF90zlZS6jyIK8vh+SVMkRKUG0L+zCXyhX1LK8FGTmv7QDl+D2FhOrrY3P40NcP4ksvKr6zvpc7YK/cT08owX1DdwRej4QPv2ULklkZ39g/BAA4O5nApbk09gx2YPuAUnb4smrTWNW5X1SDe2+Ngrskkdb7p1MkVGuKCO4WZC1sGVEKWeDBrxzAex95Gdf9zXP4H08dxqnxedv7j8wkMbWQcfQeGvv6AMXv/fBMEn3tBWuBN8Gq9USmTC5fZMkASpUIv/I7rDbO4l0W9b3cAaUhHQAsmORyzk4sIOiTtAlJV/W1YtfaNnz5pXPI5xn2nlbaA+zZ0IlNPS0I+z34hVo5YyyF9Hs9yMh5jM6mIFFtLZLbt/bgyjUx7WQtqA0iuFtgrdxFcOfMJLP4pcu6ceOmLnxj3xD+7rvHbe8/Nqeoxktx6xmdnCJbxmQ+59B0UvPbARQ2MtXYmtHnBjj6Hap8oPT+c9NIZuSiXu6Aomwjfo+pcj8zsYD1nZGizUb337AeZyYW8MLJCew9M4WuFj82dkfgURXyG2Pz2hqK1qQp9yS6WgIlJ6Rq8ns3D+I7f3RzzZ5PoCCCuwVWde6iFFIhnZORyeVx3YYOfPa+ndgz2KFd8lsxpt5+MW5/P6BYufNmU9yWmU/nMJvMapUyAOrWgiBrotz1O1QPD88iqHrwL5+dKurlzrFqHnZmcgGD3cWTie68ajW6Wvx47MWzePnMFK7b0KFZUztUawYwr3PPqsq9VpaMoL6I4G6BsaRMkggBryRKIVXm1cQgT5b1RINat0Ez5lJZzXpwEtyLE6rcc1d+fkStlFnTVghSvH1trZOqWcNgDKBQ5y7nGY6OxPFrO/rg90j42cmJol7uHLMh2Tk5j/OTCaw3jJ0LeD2477q1+MGxSxieSWLPhsLAab7dHzDz3JUTzEUHu1MFzYEI7hakTbaVh8SoPQ0ejLTgHgtgfC5t2d1wTBfQx8oofKA4oWq8ahrVkoIFW4ZPFap1OaSSUC1Vybk8w8lL80hmZVy7vgPXrGvHT09MFPWV4Zgp96HpJHJ5hg0mw5PfvWetVp1z3YYO7fh2XdlhiefukZBVE6q1KoMU1BcR3C0olEIWkkBhnxi1x+E7LaNBrtwDyOWZpXK+OFtQ9eWUe07OI88KJ9aAr9hz5yeH1TG9cq+PLZPJMdPgDgAHzk8DUBKhN23uwrHROM5MzGu93DmRgKdkE9OZSbVSxiS497aGcOeVq9HVEsCWVVHteF9bSNviX6LcvRKmE1nMpXNCua8QRHC3wGynX1Aodw1NuQcLtgwAS2uGB3S/Vyob3NOG/igFW0Y5zq8CenTDJkI+D/xeqeYJVXNbRjkZHTg3jYBXwsbuCG5UG3EdGY5ru1M5ZrYMr3E3C+4A8Il3XI2n/uCGomQrEWG7as2YtR/gf7vCc18ZiOBugVlwD/vFHFWOptwDigrlgdYquPOAfOWaWFlbxqrdMlfuF+MptId9RVN9iAgdYb/WZ6VWKPNKDXXu6rr3n5/G1t4YvB4JV/e1akE9VqLcvVgw7J84M7GAWNBrOucUAKJBHwY6wiXHd61rh0cibdOdcU1A8RWPoHkRwd2CjCyX7DwMCVtGYz6t2B9cufNa7EsWqvzibAqtIR/WdYQXodwLwzoAnS0TV/qRG1E6Q9a4WsbEc+dXHKfHF7SZox6JcMNGRb078dzPTi5gQ1ekKPHqhN+9cQP+8/1vQthffHWg9+CFLbMyEMHdAuMEeUDp6S5sGYWSapkyyv1iPIVVsQBWtQZxKZ4u6hNjhO/uDJSUQhaUu1mAag/761LnblYtw7lKN3P0ps1qcA+Vt2VOjy9YWjJ2hPwe7FpbOkJBfwIyOzEKmg8R3C0wDhkGgJBPlEJy5tLFCdWgz4No0Gup3C+pant1LIiMnLdV2EZbRvPcc3ySUNrUWuiI1L6/jNkOVb0o0M8c5QMw2kLFVkvE70Uqm9faOKSyMkZmk9jQ1eLaOvl72RHxiyHVKwQR3C0wU2ThJlDuR4Znceff/0TbBl+OTC6Pdzz0c/z0RPG0rPlUDj4PFanUnmjAVrmvVoM7ANsNTyUJVS9v2iYjK+cxuZBGTwPZMiXKXT0Z+TyEy3TVLOu7Ivind+/Eb1zTX3R/3tOd7wM4P5UAY8D6rlJPfanw/jfCb185iOBuQdrElgk2gef+/aNjOH5xTqvGKMeF6QT2n5vGvrPTRcfnUjm0BIo341htZMrJeYzPpbG6NYhVqp0yZuO7G9stSxLB75WQysm4NJcGY+ZBitsyVrX21SAjl/6d8Ha7W1ZHSwL/r25bU9LXpcXQPOy0+rsZdFG58xOlqJRZOSzr4P7ayCze/Mnn8ZLaQMlNMrnSniFhvwfJZd4Vkk+xn3JoX5yfSgCA1maWM5/OaclUTk8sgEtzpUF7Yj6DPINmywD2te4Fz71gHwS9So/0i1rL2tLGV21hH/KsUMmzFFJZGc8eHsWDXzmAD37tFdvcAKD0cy/pCqn+3ej9djuMnSHPqjXu7ip3ZU0imbpycGPMXt3wShLOTiYwMV++EdViMfPcOyJ+LGRkJDOy1qlwuXFkWGlkNTXvLLhfUIO7MVGpKPfiqo+eaEBLluoVPQ/kq2NBdEcDIHJoy/gMu4MzsubpmyUFedngVCKD1rCv5PZyfGP/EP7ym0ewkFGmQKVzefw/b96Iy1fHLH/Gqv0AAMcDpY093c+ML6CrJYBocPGvwQq/UO4rjmWt3FvVemGjqnQDs23l/IMxOpt0/flqwfhcWgu0ThOP5yfV4F6i3LNaMpXTEw0inctrAyk4F3UDInweCZ2RgK0tY2zaBiiWWConF50ojFTSXyafZ/jU917H+q4IvvJ7e/CTP70VRMCzhy/a/pxZQvXy3ih++4b1eNtVqx09d8Qwau/U+DwGl1ApYwdfo6iUWTlUHNyJ6Kw6Vu9VItqnHusgou8T0Qn1/9LaLBfgU3iqEtxNlDvvyTHqoDdKI3JkRBuY5XizzzlNuZfaMtFAqS0DAOMGa2bMoLZXtwbK2DJKcA/qlDufo3oxnoLfI5lu7uF/DzOJDKYWMvjf3zrq2LJ76fQkRmdTeP8vbcSNm7qwKhbEtes78OwR+0FiZp57wOvBR99+JTodTh7SD8k+MTaH/eencf1gR5mfWhwF5S76yqwU3FLutzLGdjDGdqvffwTAc4yxzQCeU793naC65bxqwd3woeVdCHlXwuXGa8NKcG8JeB0Hd27LxA3v8Vyq1HPniUJjv/aL8RR8HkKnGpBXx4K2tkxBues8d5+EVDaPsdkUemIB0809POB/++Ao3vKZn+ALPzuDh35UOlT6G/uH8PTBkaJjTxwYRjTgxR1XrNKOve2q1XhjbB4nL5kPIWGMmVZVLRZ9QvWfnj+JkM+D375xQ0WPaWTn2jbcta23qLmYoLmpli1zN4BH1a8fBfBrVXoetIZ8JYHHDcw+tKs1W2aZKvfhODZ0RdDXFnIU3BljWkK1xJZRq2X08P4yY0blPptCTzSo9UFZFQuWqZZRE6q+YlsmqSp3q3I+PrDjyVeG0dXix+2X9+DlM1Pa4wFK5c7Hvn0U//0/D2JoWnltiUwOzx4ZxV3beotqwO+8qhcA8F0L9S7nGRhDxYMvuC1zeHgW3zo4gve+aZ1l24Gl0hMN4p/fvctVH1/Q2LgR3BmA7xHRfiJ6QD22ig/GVv/vMf4QET1ARPuIaN/4+LjxZse0hnw1s2UCXg+6WvzL1nM/MjKLK9fEHG/2mVzIIJGREQ16S0oM5yyqZYBS5T42p+xO5ayOBTGdyBYFXT3pbHGdO6AE93RWxlg8rZVTGokFvbjr6l784W2b8PQHbsK7rluLZFbGgXMz2n1euTCD2WQW6Vwef/PMMQDAd49cRCIj455dxfXnq1uD2LW2Dc8eKfju5yYX8JW958AYQ1ZW3g+3lPtX956H3yvh/755sKLHEwgAd4L7jYyxXQDeCuBBIrrFyQ8xxh5mjO1mjO3u7u5e8pPXMrgDimc5MrP8lPtMIoOh6SSu6mtFR8SPSQfKnav2q/takWfAvFoGyqcwGXukRANehHyeklp344AIHpytxu0ZRxwCii2TzMq4OJvS+tgYISL883t24cNv2QK/V8KewQ54JMLPTk5o93nu2CV4JcIDtwzimcMX8bOTE3jywDAGOkLYva40NfS2q3vx2kgc5ycTODOxgN/81xfxP586giPDcc0+qlS5B7wSPBIhI+fxnj3r0OXQqxcI7Kg4uDPGRtT/LwF4CsB1AMaIqBcA1P8vVfo8VrSGfFUZipzO5REw+dCubg0uS+XOSyCvWqMEdydDLXilzNX9ik87q77PvKrDaMsQkVrrblDu8bRm2QCFxKpVUpUrd33OI+TzYHwujWRWNq1xNyMW9GHHQBt+qgvuzx+/hOs2dOBDd1yGgY4Q/vzJw/jZqQncs7O/qH0u51euVCpeHv7pKbz7cy9pJ54XTk4UTkKexTX3MkKkzFH1eyX8/i1CtQvcoaLgTkQRIoryrwG8BcARAE8DuF+92/0AvlnJ89hRNeVukShb0xrEqAvK/QdHxzDnsAWAG/BKmSvXxNAe8WMmmYVcZicnV+58Mw4/ifJ1G4M7wGvdC+/PfDqHecOAiHItCHhHTq+hFJK3FlhMOd+Nm7pweGgGs4kshqYTeH1sDrdd3oOgz4O/uOsKbav/Pbv6TH9+oCOMbf2t+PJL55HMyvjq712Py1dH8cLJcWRNrjCWyvaBNjxw86BpWwWBYClU+le5CsALRHQQwMsAvsMY+y6ATwC4g4hOALhD/b4q1NyWaQthLp2rKDC/cn4av/fYPnx931AlS1wUR4Zn0d8eQnvEj46wD4yVbkwycn4qgVWxgBaYZ5LK/fkOUKPnDiiJu3Gdcr9oMjWJf22VVNXPT+XoE52L6Y9y06Yu5Bnw4ukJPH9cuYC89XIlBfSWK1bhl7euwi2XdWNdp3Vd+Xv2rEVPNIAvv28PrlgTw02buvCLs9Pa+1CpLQMAX3rfHvy/v7Kl4scRCDgV7VBljJ0GsN3k+CSA2yt5bKfEQj7MpXKQ86yo93qlmJVCAvqNTKklVx48eWAYACxL7KrBayNxTYF3qJ7udCJjW4t9fiqBtR1htIV4/bhyQuM7KY117oBSDvnjNwrB3VjjDigtb4M+yVK5p01aP+grZxazhX7n2jZE/B68cHICw9NJrOsMaxuEiAgPv/calGuZfu+1a/HO3QNa+eWNm7vw+RfO4OenFLvHjeAuELjNsv+r5LtU3bY4LG2ZNmUTyFJr3dM5Gd86pNRYnxqvTXCfS2VxZqIwOKJDLRmcWrB/zy5MJTDQEda28vNyyHltfmrpya0nFsB8OoeEmnzV707lEJFS626h3M3bLReU+2JsGZ9Hwp7BTjx/fBw/PzWJ2y7vKaqRlyRyNBBDf589Gzrg8xB+qF4JuGHLCARus+z/Ktuq0IJAzjPIeWZRLVNZrfvzx8cxk8iivz2E0zUK7kdHlGQq73XSHlHes6kF6548fDfouo5Ioc2DauMY56fq0WapqpUwVu0C7Grd0zm5qGkYULBl2gzj9Zxw06YuDM8kkc7lcdvlJVW5iybs92LX2nbsPT0FAKZXeAJBvVn2f5XV6C9jNj+VsyoWBBEw6lC5j8wkizoLPnlgCF0tAbx7z1pMzGe0CpRqcmxUDe69inLvjChWjJ1yH5pOgjFgbWcIAa8HYb/HcUIVKNgxY/EUYkFvSaO1VXbK3eSqKah+v5R+5HwCUtjvwXUb3NnWf9OmLtOSTYGgUVj2f5WtVegvY9a4iuPzSOiJBhwp98NDs7jhEz/EJ//rdTDGMLWQwfOvX8Kv7ViDLeoQh1MT1VfvwzNJBLyS1h6A92Cx28jE2w6sVYcwt+kS18YpTHr04/bSORkvnZ5Ef3tp69re1iDGZtOmFTtmCVV+clhKNcnmnhb0tYVw65aekiuCpcJPGIDw3AWNybJu+QtUR7mn5eIZnkZ6W0OOgjv31P/lR6eUbogtfmRlhnt29WvB6tSledOZl24yMpNCX1tI842DPg8ifg8mbdr+8jLIATW4t4b9RZ67VyLT90ezZebS+Pgzx/HG2Dw+/1u7S+432B1BRs5jaDpRUqlillDlVszq2OI3+BARvv7+N6HF796f+9V9rYgGvZhTJ1IJBI2GCO4m2NkygNJA7PjFubKPM6Judrp7xxr8w3MnEA14cfnqKK5YE0NOzsPnIZyecDYRqRKGZpJaIpjT0WLfguD8VAIhnwfdiQHB/gAAEyBJREFUajVNW8inWUjz6RyiQa9pIrI97IPPQ3jqlSEcGY7jd2/cgF/WNePibOpRrlxOjM2XBHer1g/A0sfE9bW52w3R65Fww8ZO/NdrY8KWETQky/6vshrBvVzPkN7WEEZnUmWn9IzOpNAa8uHT79yBe3b2YS6dwzvU/iVej4R1nRGcqkE55MhMsiS4dYT9ts3Dzk0qZZA8gLeGfFqd+7xJR0gOEaEnGsSR4Tiu6ovhz95qXru9qUcZIXfC5PWbJVT5lY5VX5l6cNNmpW1GxMUrAoHALZb9X6XW9tfFxKRZy1k9va1BJLMyZpNZrROhGaOzSfS2BuGRCJ/8ze1429W9uOWyQh+djd0RnHI4y3SppHMyxufSpco94seEjS3DyyA5beFCm4e4yRQmPatiAcwkMvjsfbssPe7WkA+rYgHTWv+0iXLnyVvj66gn9+4eQG8siPUuD9YQCNxg2Qd3wP1dquVtGV7rnrIN7iMzKe2+HolK7ImN3S344fFLyJpMfXILXmfOe9Fz2iN+vDFm3af8/FQCN24qJA1bwz7MJLNgjClTmEwqZTgfffuVkPMMG8oEvc09UZy8VGpvmc2v3TnQhn941w7csnnpTebcxu+VTC0ngaARWPa2DFCF4K4mVK1tGWfj9rhyt2KwuwVZmWmVKdVgWC3ZNLNlrDz3ifkMklkZAx2Fn2kL+ZHJ5ZHK5k2HY+vZ1t+GnQ6SxJt6WnDi0nyJvaUkVIsVvyQR7t7R5+ouZIGgmWmK4N7mcnBP25RCAjrlblMxk8zImE5kbW2Ejd2Ksj3twJr5q28fxV99+2jZ+xnh7YnNEqqJjIxUtrSn+oXp4jJIQDfCLpnBfCpnWga5WDb1tCCRkUveRzNbRiAQLI6m+ATV2pbpagnAK5HtRiau6sspd6B8GwI5z/D4y+fx+RfO4KlXFtdsjLdJMPZjKbQgKFXvFwxlkACK+svMmUxhWgqbeVJ1rNiaURKqTfGnKRDUjab4BNkFd8YYPvLEIfyPpw7j56cmyra5BQrB3SrAeCTCqljQttad32Y3kLg15ENXS6BscH/94hwW1IlIf/HUEZxZRPnkyEwSXS2Bki377RHr4D40rZwQ+tsLa9f6yySyplOYlsJmdSOXMalqVucuEAgWR1N8gmI2wX1qIYOv/eICvrr3PN79ub24/uPP4Sdv2I/1c7KtfE1b0LZ5GL/NmMg0MtgdKWvL7D8/DQB45P5r4fNK+MPHD1iOqDMyPJNEn8kaOm2C+4WpBDojfoR1JX5tIeX+4/NpZHJ524SqUzoifnRG/EXBnTFmmlAVCASLoyk+Qa26tr9G+Cahh96zC//87l0gAI+9eNb28ezaD3DK7VIdNemGaMbG7payyv3AuWn0RAO4dn07Pvkb23FkOI5Pf+8N25/hDJtsYAIKyt0sqTo0nUR/R3HLAO65D6uq3g1bBgA2qklVDj+xBhbZHEwgEBTTNMEdMG/7yzcJXdXXiru29eL6wU4cG7XfXVrOcwcUL/3ibKpoaLSe0dkkulr8ZXuZbOyOYDqRtd1QtP/cNK5Z1w4iwh1XrMKdV67GN18dMb3fF144o33PGMOIRXDnnrtZC4IL0wkMtBf/DA/uQ2qydam97I1s7mnBibE5rWLGyYlVIBCUpyk+QXa7VE+Nz8PvlbQAt7U3huGZpO2mJye2zMaeFmTkvOkOS0CpUnEyVGKjmlS1av97aS6F81OJov4zewY7cDGeKhl28W8/PoWPfeeo9j5MJ7JIZfOmwb015INEpcpdzisnhAGDcg/5PPB7JFzgyt0Fzx1Qgns8ldOmN/FKJf1wDoFAsHiW/AkiogEiep6IjhHRa0T0QfX4R4lomIheVf+9zb3lmmMX3E+PL2CwK6LVR2/tVZJ4xy7GLR/PiXLnG3x+esLcv1dq3MvvptxYpmLmwLkZAMCudYXgvq2/DQBwcGhGO8YYw4Hz02AM2H9O6TM+YlHjDih14+0mLQguxlPIyqwomQoobQVaw76CcnfJluFJVX6SLFeGKhAInFHJJygH4MOMsa0ArgfwIBFdod72GcbYDvXfMxWvsgz6Sg4jp8bntQAKAFeoPc15j3MznASYvrYQBrsi+NnJCdPbR2dSWONAufe1h9AR8WtTfYwcOD8Nv0fSpigBypBrr0Q4eKEQ3M9PJbR2AnyIhNUGJk5HpDS4D/EySJM2va0hX8Fzd1G5A4WKmYxQ7gKBKyz5E8QYG2WMHVC/ngNwDID5CPkqYzWNKZ2TcWE6icHuwjb47mgAnRG/bXB36vveuKkLe89MaffnzKWUcsFeB31QPBLh3msH8P2jY1ow1rP/3DSu7m8t8u6DPg8u743i0NCsduyAWlHT1RLAS2eKlbtVxU67SXDntovRlgGU95mf+NxKqHZHA4gGvTihtiHgVUBu9V0XCFYqrsgjIloPYCeAveqhDxDRISL6AhGZ7kMnogeIaB8R7Rsfty9NLIeVLXN+MgE5z4qUOxFha2/MNqmaUdvxSmW2ut+0uQuJjIxX1MDKuajVuDvrYPiePWsBAF/de67oeDon4/DQLK5ZV/oWbu9vw8GhGS2hu//cNKIBL35zdz+ODM9iPp3DiDqkoyNi3v/GrAXBhakEiMxPCDypCriXUCUiNalarNyFLSMQVEbFnyAiagHwBIA/ZozFATwEYCOAHQBGAXzK7OcYYw8zxnYzxnZ3d1fWDCpmEdy5j61X7oDiu78+NoecXKy4OZlc3lFwuX6wExKhxJoZmTXf8m9Ff3sYt29dha+9fKGofv21kTgyct50mMf2/jbMpXI4M6mUeu4/N4Mda9vwpsFOyHmGA+emS4Z0GOloMVPuCayKBk2Vc2uocJJwo/0AR2kgVuy5C1tGIKiMij5BROSDEti/whh7EgAYY2OMMZkxlgfwOQDXVb5Me4I+DwJeCfGS4K4EvkGdcgeUiplMLm+509NsWIQZrSEftvW34aeG4M7bEjhV7gDwW29ah8mFDJ45PKodO3BOuSLYta6t5P7bB9Sk6oUZzKWyeP1iHNesa8c169rhkQh7z0xa1rhzFOWeLSrnHJpOFjUM08OVu9UUpqVydX+r9trTWaHcBQI3qKRahgA8AuAYY+zTuuO9urv9OoAjS1+ec8xaEJwan8fqWLDEH96qJlWPWvjuToM7ANy8uQsHL8wgrquxH5lNgUgZAu2UGzd2YbA7gsdeVKwZxhheOj2FtR1hbXSdnk09LQj7PTg0NIuDF2aRZ8A169oRCXhxdV8r9p6eUoO79RraI37IeYa5VE47NjSVME2mAoXcRovFFKalcu+1A9gx0IY/e+IQTqszZcUmJoGgMiqRRzcCeC+A2wxlj39HRIeJ6BCAWwH8iRsLLYdZcD89vlBiyQBK+aHPQ5a+e0Z2Htxv3NSFPANeOjWpHRudSaInGlhUj3ZJIrz3+nV45fwM/vzJw7jlk8/jB8fGigYx6/FIhKv6WvHqhRnsPzcNImCHqub3DHbg4NCM6ZAOPbwFweSCUmOeyeUxGk+VlEFyuHJ3K5nK8XkkfPa+nQADPv7McQDWfX0EAoEzKqmWeYExRoyxbfqyR8bYexljV6vH384YGy3/aJVjDO6MsZIySI7fK2FTT7SoYiaRyeHw0CwOD81iYj7t2BbYtbYdIZ8HL+ismdHZlKMadyPvuKYf0YAXX993ARu6WvB3v7EN/+uuKyzvv2OgDUdH4njp9CS2rIpqSc7rN3RqowLtgnt3VJmPelb17UdmkmAMJa0HOK3qrla3kql6BjrC+MQ7tiGZte+lLxAInNEUk5gAJbhfjBd2bI7PpzGXypkqd0BJqv70hBKQ59M5vP2zLxQNq965ttTnNsPvlbBnsKMouI/MJnH56uiiX0Ms6MN3/+QWBL0SOtXB1HZs729DRs7jxdOTeLdacQMA16xvh0RAntkPht69vh2xoBdPvzqC2y5fpXWDLGfLuLWBychd23rxwsm1ePzl84hV4QQiEKwkmiq4v67rC847LZopd0DZzPTkgWFMzKfxsW8fxdnJBXz8nqvRrQbVLYsIzjdv7sbHvn0U3z40gruu7sXoTAq3bulZ0uuwC8ZGtg+0al9fo6uoiQV9uGJNDEeG47bKPeD14L9tX4MnDgxhPp3ThnSUS6i6tYHJjI/dfSXee/067apCIBAsjaYJ7sa2v7wMcmOPeXDnSdW//s4xfPPVEXz4jstw33VrTe9bjvuuG8B3j4zig197FQvpHJJZeVGVMkulry2EzogfkwuZklr4GzZ24fWLc2XXcc+ufnxl73k8e3gUF6YS8EiE1RaJYN72123PXY/XI+GKNbHydxQIBLY0jbHZFi5u+3t6fAFBn4Rei0DFg/tTrwzjTYOd+INbNy35ucN+L/79d67Dtv5W/NkThwE4r3GvBCLCzrVt6GrxY11nsZXy4K2b8LUH3lQypMPIrrVtWN8ZxpMHhnFhWqmu8VrkG1pD1VfuAoHAHZomuPPAw2vdT43PY7CrxXKXaUfEj9WxIDoifvz9u3ZUPHi5JeDFo797Hbb3K1ZJLYI7APx/v3olHrn/2pLSxNaQz3RnqxEiwj27+vHi6UkcODdt6bcDysYln4c0710gEDQuTSPB9C0I2iN+nBqfx/Z++6ToZ+7dgVjIu6h6dDtiQR8ee98e/NeRi9jW11r+B1xgbWcYazutA7ITfn1nHz79/TcwPJPEjZs6Le8nSYTP/dZu7apHIBA0Lk2n3GeTWfz4jXEMTSctk6mcN23sxJVr3A3CrSEf3nntQNm+NI3EQEcY123oUL62Ue4A8OYtPa6dDAUCQfVoOuX+sW8fxb5z09jc04J3XjtQ51UtH96xqw8vn5ky7QYpEAiWH00X3Pefn8bv3zKIP7njsrLJREGBu3f0YXgmhdu2Lq2EUyAQNBZNE9wHu1vwR7dtws2XdePa9R31Xs6yI+jz4EN3XFbvZQgEApdomuDukQgfesuWei9DIBAIGoKmSagKBAKBoIAI7gKBQNCEiOAuEAgETYgI7gKBQNCEiOAuEAgETYgI7gKBQNCEiOAuEAgETYgI7gKBQNCEEGOs3msAEY0DOFfBQ3QBmCh7r+ZiJb5mYGW+bvGaVw6Lfd3rGGPdZjc0RHCvFCLaxxjbXe911JKV+JqBlfm6xWteObj5uoUtIxAIBE2ICO4CgUDQhDRLcH+43guoAyvxNQMr83WL17xycO11N4XnLhAIBIJimkW5CwQCgUCHCO4CgUDQhCzr4E5EdxLR60R0kog+Uu/1VAMiGiCi54noGBG9RkQfVI93ENH3ieiE+n97vddaDYjIQ0SvENG31e83ENFe9XX/BxH5671GNyGiNiL6BhEdV3/nb1oJv2si+hP17/sIET1ORMFm/F0T0ReI6BIRHdEdM/39ksI/qvHtEBHtWsxzLdvgTkQeAP8M4K0ArgBwHxFdUd9VVYUcgA8zxrYCuB7Ag+rr/AiA5xhjmwE8p37fjHwQwDHd938L4DPq654G8L66rKp6/AOA7zLGLgewHcprb+rfNRH1AfgjALsZY1cB8AB4F5rzd/1FAHcajln9ft8KYLP67wEADy3miZZtcAdwHYCTjLHTjLEMgK8BuLvOa3IdxtgoY+yA+vUclA97H5TX+qh6t0cB/Fp9Vlg9iKgfwF0APq9+TwBuA/AN9S5N9bqJKAbgFgCPAABjLMMYm8EK+F1DGfkZIiIvgDCAUTTh75ox9hMAU4bDVr/fuwE8xhReAtBGRL1On2s5B/c+ABd03w+px5oWIloPYCeAvQBWMcZGAeUEAKCnfiurGn8P4E8B5NXvOwHMMMZy6vfN9jsfBDAO4N9VK+rzRBRBk/+uGWPDAP4PgPNQgvosgP1o7t+1Hqvfb0UxbjkHdzI51rR1nUTUAuAJAH/MGIvXez3Vhoh+FcAlxth+/WGTuzbT79wLYBeAhxhjOwEsoMksGDNUj/luABsArAEQgWJJGGmm37UTKvp7X87BfQjAgO77fgAjdVpLVSEiH5TA/hXG2JPq4TF+iab+f6le66sSNwJ4OxGdhWK53QZFybepl+5A8/3OhwAMMcb2qt9/A0qwb/bf9S8DOMMYG2eMZQE8CeAGNPfvWo/V77eiGLecg/svAGxWM+p+KAmYp+u8JtdRfeZHABxjjH1ad9PTAO5Xv74fwDdrvbZqwhj7c8ZYP2NsPZTf7Q8ZY+8B8Dzw/7dzxyoNQ1EAhv87FbrpI7i4OnZwENw6O3fxMZx8CJ/AwcFFpKsvIA6iUkTbqYurs8N1uKeLEOjQEnr8P7gkhEDuyQmH5CSEs9gtVdy11i9gWUo5jE2nwIzkuaa1Y0allGFc76u40+b6j6783gOT+GpmBHyv2jdrqbXu7ADGwAewAC76ns+WYjymPYq9AM8xxrT+8wPwGcv9vue6xXNwAkxj/QB4BObALTDoe34bjvUIeIp83wF7/yHXwCXwDrwB18AgY66BG9p7hR/anfl5V35pbZmrqG+vtK+J1j6Wvx+QpIR2uS0jSepgcZekhCzukpSQxV2SErK4S1JCFndJSsjiLkkJ/QKtFug5FEdaRQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_episodes = 1000\n",
    "durations_list = []\n",
    "render_flag = False\n",
    "lr = 3e-2\n",
    "num_batch = 5\n",
    "\n",
    "def init_buff():\n",
    "    global state_buff, action_buff, reward_buff, value_buff\n",
    "    state_buff = []\n",
    "    action_buff = []\n",
    "    reward_buff = []\n",
    "    value_buff = []\n",
    "    \n",
    "def run(env):\n",
    "    actor = ActorNet(env.observation_space.shape[0], env.action_space.n)    \n",
    "    critic = CriticNet(env.observation_space.shape[0])\n",
    "    optimizer_actor = optim.Adam(actor.parameters(), lr=lr)\n",
    "    optimizer_critic = optim.Adam(critic.parameters(), lr=lr)\n",
    "    \n",
    "    init_buff()\n",
    "    for ep in range(num_episodes):\n",
    "        state_numpy = env.reset()\n",
    "        for t in count():\n",
    "            state_torch = torch.FloatTensor(state_numpy)\n",
    "            policy_torch, value_torch = actor(state_torch), critic(state_torch)\n",
    "            action_torch = policy_torch.sample()\n",
    "            action_int = action_torch.numpy() #.astype(int)[0]\n",
    "            next_state_numpy, reward, done, _ = env.step(action_int)\n",
    "            if render_flag:\n",
    "                env.render()            \n",
    "            if done:\n",
    "                reward = 0\n",
    "                \n",
    "            state_buff.append(state_torch)\n",
    "            action_buff.append(action_torch)\n",
    "            reward_buff.append(reward)\n",
    "            value_buff.append(value_torch)\n",
    "            \n",
    "            if done:\n",
    "                durations_list.append(t+1)\n",
    "                break\n",
    "            state_numpy = next_state_numpy\n",
    "            \n",
    "        if ep % num_batch == num_batch - 1:\n",
    "            calc_discount_reward(reward_buff)\n",
    "            optimizer_actor.zero_grad()\n",
    "            optimizer_critic.zero_grad()\n",
    "            for state_torch, action_torch, reward, value_torch in zip(state_buff, action_buff, reward_buff, value_buff):\n",
    "                advantage_torch = reward - value_torch\n",
    "                critic_loss = advantage_torch.pow(2)\n",
    "                policy = actor(Variable(state_torch))\n",
    "                actor_loss = -policy.log_prob(Variable(action_torch)) * advantage_torch  \n",
    "                loss = actor_loss + critic_loss\n",
    "                loss.backward()\n",
    "                #critic_loss.backward()\n",
    "            optimizer_actor.step()        \n",
    "            optimizer_critic.step()\n",
    "            init_buff()\n",
    "        \n",
    "        if ep % 100 == 99:\n",
    "            clear_output()\n",
    "            print(f'Episode:{ep}')\n",
    "            plt.plot(durations_list)\n",
    "            plt.show()\n",
    "            \n",
    "    print('Final results')\n",
    "    plt.plot(durations_list)\n",
    "    plt.show()    \n",
    "\n",
    "env = gym.make('CartPole-v0')\n",
    "run(env)\n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}