Actor-Critic implemented by PyTorch, separated loss formulations are used for actor and critic agents.

Actor-Crtic_detached_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from IPython.display import clear_output\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "from torch.distributions import Categorical, Bernoulli\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "\n",
    "import gym\n",
    "\n",
    "from itertools import count\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ActorNet(nn.Module):\n",
    "    def __init__(self, state_size: int, action_size: int):\n",
    "        super(ActorNet, self).__init__()\n",
    "        self.fc1 = nn.Linear(state_size, 24)\n",
    "        self.fc2 = nn.Linear(24, 36)\n",
    "        self.fc3 = nn.Linear(36, action_size)\n",
    "        #self.fc3 = nn.Linear(36, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        \n",
    "        prob = F.softmax(self.fc3(x), dim=-1)\n",
    "        policy = Categorical(prob)\n",
    "        #prob = torch.sigmoid(self.fc3(x))\n",
    "        #policy = Bernoulli(prob)        \n",
    "        return policy\n",
    "    \n",
    "class CriticNet(nn.Module):\n",
    "    def __init__(self, state_size: int):\n",
    "        super(CriticNet, self).__init__()\n",
    "        #self.state_size = state_size\n",
    "        self.fc1 = nn.Linear(state_size, 24)\n",
    "        self.fc2 = nn.Linear(24, 36)\n",
    "        self.fc3 = nn.Linear(36, 1)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        value = self.fc3(x)\n",
    "        return value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_discount_reward(reward_buff, gamma=0.99, normalize=True, done_flag=True):\n",
    "    prev_dr = 0\n",
    "    for ii in reversed(range(len(reward_buff))):\n",
    "        if done_flag and reward_buff[ii] == 0:\n",
    "            prev_dr = 0            \n",
    "        else:\n",
    "            reward_buff[ii] += prev_dr * gamma\n",
    "            prev_dr = reward_buff[ii]\n",
    "    \n",
    "    if normalize:\n",
    "        mean, std = np.mean(reward_buff), np.std(reward_buff)\n",
    "        for ii in range(len(reward_buff)):\n",
    "            reward_buff[ii] = (reward_buff[ii] - mean) / std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode:499\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO2deZxcZZnvv08tvaaz7xshEJYQIIEQQGSRRYFwdUAQcEZRcRivOO4zA654vSiDGzLj4KAyghcRFRlQGCRGEJCwJBCSQIAsZA9JZ+1OL9W1vPePs9SpU6dOV3dVd1dVP9/Ppz916j1vVb+nu+pXTz3vs4gxBkVRFKW2iAz1AhRFUZTyo+KuKIpSg6i4K4qi1CAq7oqiKDWIiruiKEoNEhvqBQCMHz/ezJo1a6iXoSiKUlWsWLFijzFmQtC5ihD3WbNmsXz58qFehqIoSlUhIpsLnVO3jKIoSg2i4q4oilKDqLgriqLUICruiqIoNYiKu6IoSg3Sq7iLyAwReUJE1orIqyLyGXt8rIgsEZF19u0Ye1xE5HYRWS8iq0TkpIG+CEVRFCWXYiz3FPAFY8yxwGnA9SIyF7gBWGqMmQMste8DXATMsX+uA+4o+6oVRVGUUHqNczfG7AR22sftIrIWmAa8DzjHnnY38CTwL/b4PcaqJfyciIwWkSn289Qs+zp6WLZhL3OnjmTngS7eceT4wHmvv93Goe4UC2eNDX2+RCrNwyt3cPnJ00mkMvz82U10JlIDsfThhQjvmz+V5zbupSEW5f0nTx/qFSlVxJ9e20VHT4odB7qZNLKeTXs6OGxcM1v3d5LJ9K98+lGTW7jkhKllXmkfk5hEZBawAHgemOQItjFmp4hMtKdNA7Z6HrbNHssRdxG5DsuyZ+bMmf1YemXx0MrtfOP3r7n3N92yOHDehbc9HXre4ftL3uQ//7KRkY1xmuti3PI/rwMgUqYFD1OMgfW723l09dsAvHf+VOJR3XpSescYw8fvKZxs2d/35iUnTB1acReREcADwGeNMW1S+EqCTuR9pBlj7gTuBFi4cGHVdwxJpct7Ca3tCQDaupLu2COffifHTR1V1t8z3Dj920vZ35H9m6Yzhnh0CBekVA2JVCbn/pETR3DSzNH8evk2po1u5K83nDtEKwumKJNFROJYwn6vMeZ39vAuEZlin58C7LbHtwEzPA+fDuwoz3Irl8wAdrTqTqYBaFQVKpl4NEJHT9a9NZD/N6W26OpJu8dfuOAolnzuLI6a1AJAS0NFVHLJoZhoGQF+Bqw1xnzfc+ph4Br7+BrgIc/4h+2omdOAg7Xub4eAryZlQkTcF1VjnYp7qdTFIhzy7F2k++knVYYfXqNg6uhGRIRpoxuHcEXhFPNxcwbwIWC1iKy0x74E3AL8WkSuBbYAV9jnHgUuBtYDncBHy7riCmUgDcAutdzLRl00wp5DCfe+artSLF7LfeyIOgAmjWoYquX0SjHRMs8Q7EcHOC9gvgGuL3FdVcdAfb03xrji3qDiXjJ+y72/EQ7K8GHrvk4mjqynwyPuh49rBmD6GMtyv+SEKUOytjAqz1Gk5NHdk0YE6mMa1VEqdbEInZ43aVp97koI+zp6OPPWJ7jm9MN4z7zJAPzogycxa7wl7hNbGljxlfMZ01Q3lMsMRNWiTJgBEgkRoSuZpjEeJSRCSSmSOl/Yo1ruShhPvdkKwMtbD9CZsIyCGWNz/ezjRtQTiVTee1PFvUz0phHdyXS/PgAct4z628tDne/bj2q7EsazG/YAMGlkA522e7SprjocHiruZSJMt3ce7OKYrz7GPcsKNk3JQzzbHF09GfW3lwm/5a5uGSWMHQe6AdhzKOFmiDdVSdSainuZMCHBkFv2dgLwyKriI0K9z9edTGsYZJnIs9zVdFdC2N1uifvutoS7odqslvvwIkwjHF95fyJqvD53pXT84q5x7koYu+1M8db2rOVeLYaWinu5CBFuZ6+lPzJijKGrR8W9XOT73FXclWASqTQHOpOMaozTk86ws62beFTyXkOVSnWssgoIkwgnyKUvQpLjc0+maagSa6HSyYuWUXFXCuDUd5o9wQp73NOeqCojS8W9TIRrhBQxpzDdyTSNcf1XlYN8t8wQLUSpeBxxnzGmCYDtB7oYN6J+KJfUJ1QxykSYBViKWwZsy72KLIZKRi13pVgO2hVZJ9slBrbs62RCi4r7sCPcLeNY7tlZfYl5706maYipuJcD3VBVisUpUzHeriPT3p1ioor78CNMqyVgTl80pSeVqZpNnEpHN1SVYulwxT0r6BNbKrdQmB9VjDIRZolHHMvdY98XazEaVNzLib/rklruSiHauy1x9/rZJ41Uy33YUVS0jGfzri8WY09axb1caPkBpVgct8y45mxRsIkq7sOPMMtdAjZUixZ3A8m0ydsIVPpHve/v+NX/XkNnjzYeV/LpSKRojEdzyg1UYvXHQqhilInQDFXyN1SLdQf02LF6armXB//f8bWdbfzs6beGaDVKJXMokWJEQywnUq2lIT6EK+obqhhlInRDNaAaaKbI+GqnKa/Wci8PQR+S3iYMiuLQ3p2ipT5X3EdWYK/UQhTTQ/UuEdktIms8Y/eLyEr7Z5PTfk9EZolIl+fcjwdy8ZVEWOEwR/i9HwDFViPsUXEvK0dPbuHw8c2cOGO0O+Y0IFcULx2JFM31MRo8CYTVZLkX8zH0c+DfgXucAWPMlc6xiHwPOOiZv8EYM79cC6wWwrTaEf5Mf9wyKXXLlJMjJozgiS+ew4rN+3j/HcsAFXclmEOJFCPqY9THvG6ZGrLcjTFPAfuCzomVnfMB4L4yr6vq8G+o5iYs2bch8wvRk7aER8W9vEQ8vjIVd8XPo6t3sm1/FyMaYkQ9XZaqpZY7lO5zPxPYZYxZ5xk7XEReFpG/iMiZhR4oIteJyHIRWd7a2lriMoYev1QHaXeO5d6LuDvak0jalnu0el5U1YD3Ddud1AIzisXyTft44a19fPLel9h5sJtmn5hXU6vLUr9jXE2u1b4TmGmM2SsiJwP/LSLHGWPa/A80xtwJ3AmwcOHCqo829mu1CTrn9bn34pZxHuNEy6jPvbx4LfcutdwVm8t/vCznfrXUbg+i34ohIjHgMuB+Z8wYkzDG7LWPVwAbgKNKXWQ14I9b9953fO45ce7FRssk1ec+EKhbRimG+iqu6VSKYpwPvG6M2eYMiMgEEYnax7OBOcDG0pZYHYS5ZZzjvrhlHLpT6nMfCLxumZVbD2gikxJINVdjLSYU8j5gGXC0iGwTkWvtU1eRv5F6FrBKRF4Bfgt8whgTuBlba+S7ZbyWe/6c3twyjmHpWJUq7uXFm6iaSGW45X9eH7rFKBVBMqC4v7c5x7TRjYO5nJLp1edujLm6wPhHAsYeAB4ofVnVR360TP65HMEv1nJ3N1RV3MuJf2PMacygDF+C9l6cGPfXv3lhYDJiJVM9QZsVTnicu4XXz16sWyZhu2V0Q7W8RH3v1Jljm4ZoJUqlELT34rhlqtE9o4pRJvwZqkE+dy/FJjE5lns1b+xUIl6fO+imqgLdPeFumWpDxb1M+LU6V+zzM1SLjZZRn/vAEPGI+/gR9RrrXsUk0xlSZWiG6wQveKmv4t7F1bvyCsNvnWcCLPf+1JZRcR8YvIZ7c31UY92rmGO++hjvvu2pkp8n6NubWu5KgFsmIFqmH52YElpbZkDw+twb41F1y1Qx6YxhY2tHyc/TFVAdtBp97Q6qGOWiiAzVTEAETW84FqVGy5QXr1umPh6lO6VumeFMa3uCK+98Lm+8msVdo2XKhD9DNTAUsg9x7g4HOpM010XVci8zkRzLPUK31nQfdmzd18mZtz4BwMmHjQmco24ZJb+auwk67HuGKsDYEdXT2qta8LplGuLRwM00pbZZ8tou93jF5v2Bcxp0Q1UJzVANcMsUGy0DMLa5epryVgsRzyu/IRYN9Lcq1UWxrk6HXW3dgeOfPOcIRtRbTo1qdsuouJeJ/MJh2WO3cJivtsx/PLme6+5Z3utzj29Wy73ceOPcG+vUcq8F+hrO+nYBcf/4mbPd14eGQioBhcPy/TI5VSGN4dbH3uBxz1fDQoxVcS87kRy3TETj3GuA9u5kr3P2HErwdz99nrcPdhe03BviEUY2WpZ7PFK9EqkbquUiJFrGseJNjluml8JhnuNxI9QtU24ifp+7umWqnrbuFBNHhs95Zt0enlm/h9XbD/L2wQLiHovyi4+dymOvvs2YKjasqvdjqcIIjZYJcsv01qzDczyuil9glYrXLaMbqrWB13Lf1dbNZ3/1cl7+wprtVrvnjkSKHR5xr4tF+OgZs/in9xxNJCLMGt/MJ84+YnAWPkCo5V4mitlQzbHc+7D501RfvZs6lYo3Q7UhFiWZNjz48jYuXTB96BallER7d7Ym/82PrOXhV3Zw9tETcv6na3ZY4r51X6fbfB7grmtO4Z1zxg/eYgcBtdzLhD9DNSgU0jujL6UwqtnvV6l4S/4e6OoB4HP3vzJUy1H6iffb8JZ9ne6xYzyJx8GZyRhe3W51/Fy3+1DO8zTW1d57rPauaIgI76GaXzistzh379xIpMoKSVcZly6Y5h6/taf0NHZl8PC6Nx2XC2Tff85nuDGGL/7mFdoTlnW/3i/u8dpzYqi4lwm/Cz0TVFumD+UHvKdjKu4DygnTR/PQ9WcAsMH3plcqm5TnjferF7fyxtvt1h17WEQ4lEhx08Ov8ruXt7tzN+/N/RBvquJG2IUops3eXSKyW0TWeMZuEpHtIrLS/rnYc+5GEVkvIm+IyHsGauGVR+EN1WwoZPEbqmq5Dy5OPHNQqzWlcnHE/ciJIwD43UtWS2fve+1bj67l7mWb3fsj6mN0+KKjhqW4Az8HLgwY/4ExZr798yiAiMzF6q16nP2Y/3AaZtc6oW4Zt557dqzXaBm13AeVuF2YrUfFvapIp603ygcXzWRccx0dPSm27uvk8Vet/JGeVIa/rt/jzr/7Y4sY2WC5YLz1mhqHo7gbY54Cim1y/T7gV8aYhDHmLWA9sKiE9VUN+aGQ+dEyXsUPipbp6knz+1d25J2PVFvzxirEqbrZo9Uhq4qkXccjFhWa6qN0JNK899+fcS367mQ6Z6N1TFOcJru0gCPyUN0FwgpRyi7Cp0Tkw8By4AvGmP3ANMBbN3ObPZaHiFwHXAcwc+bMEpZRGeRnqOYf57pl8p/jpodf5f7lW5k2plEt90HGseLUch84Xn+7jagIcya1lPxcmYzhsVffpsPeII1FIjTXxehIpNjfmY13b+9O5byXxjTVuS6YloY4ew5ZkVKxGiyp3d8rugM4ApgP7AS+Z48HqVCg/8EYc6cxZqExZuGECRP6uYzKoagG2QU6MS3bsBeAHQe7ACvBwvtBEI2quA80juWeVMt9wLjwtqe54AfFd0za3d7NK1sPuPdf2rLfzSr97Uvb+OS9L/FPv10FWAZQc32Mjp5UznN4rXaAUU1xj7jH+Mg7ZvXnUqqCfom7MWaXMSZtjMkAPyHretkGzPBMnQ7sKG2J1UF+4TCvWyY/Q9V7fPVPnmOr50UoSE7VyKi6ZQacuM9yX75pX9E195XS2NXW7YagtrYnWL/binj5u58+z/t+9FdS6QwbWw9x2X88y+d/vRKAA509Oc8Ri9rinkjjfbts8oW2ttTHaK6zHBYtDTFueu9xbLpl8UBd2pDSL3EXkSmeu5cCTiTNw8BVIlIvIocDc4AXSltidRJQNyzHck+lc4WjoydVMINV3TIDj9fnvmLzfi7/8TJ+uHTdEK9qeHDqt5byru8+CcA533mC879vWfdv7rLCUje0dnDv81sA2NdhiXpjXa5HORoRmuuibGg9lPM+8oc8ioi7eeqU9a1Ver06EbkPOAcYLyLbgK8D54jIfCzd2gT8A4Ax5lUR+TXwGpACrjfGDIuiHeFJTPnz/Vah+Dxa3tMaCjnwxG3XV0/auF/91+1qH8olDUu8IYrTRjey/UAXa7YfZLWdoORsePsL78UiEZrrYzklCAC3fszUUQ3u8Zgmq1ZTU90wF3djzNUBwz8LmX8zcHMpi6pGwqJlvFIfEUu4kyHdOqyvlWq5DwZOfLSIUBeNkEil2bq/0x4bypXVFn3NH0imM4wfUcf2A12s3dnGazussgHbD3RhjOFQIlfEY1HLcg+iqS7KE/90jmtQnX7EOH7x3Oa8LNVao7Y/ugaRYi33WCRCTzqT55bxC4la7gPPK19/N/WeWOe6WITfr9zhWnii6l42/Bb17rZuJo5syBnb35H1ox/sSrqlApZv3s+hRIpjJrfw+tvt7OvoyRf3iFBfIJxx8sgG6mPZc2faBcJOnDGq/xdUBdRe/M8Q4S8cFuRzh6yIp3yWjPieQ33uA8+oxnhOG7V4VHLKwGp+QWFS6QydvsiUMHbakWAAP316I4u+tZSNrYdyvuEu+OYS97itK0lblxXSuNKOmDnrKCuqbseBbg51+y33SMFmHZN8HyItDXGeveFcvrJ4btHrr0ZU3MtEvl89IInJM5oKicRIpjM5j4mquA8K3oxFCI7rVSz+/p7lzP3aH4uam0xnWHz7M+797z3+JmC1uess0CTlYFeStq5cAT9t9lgA1re2s/NgV04SUiwiHOwKFvcpoxryxqaObqzq/qjFoG6ZMuEX96AeqtY869gv7l4j8SP/9WLOORX3wSHuS2RRw70wT7zRWvTcdbtyfdtddgONtq4U+30hjQ672hI5CWUtDTGOmWy1WXJKMx8+vpk224KPRoTDxjW780+bPZb9HUne2NXOpABxHw6o5V4mQt0yAcdBG0yFEqHULTM4+C33iAh7DiWYdcMj3P/iliFaVWWwatsBZt3wCC9uyq1Ecu53n2TWDY8w64ZHCj7WaZDh50BnDwc6c63txcdbUdbb7E3taaMbAcv6Hu9rN7ltf6fbdCUeFT53/lFceNxkAA4b28z4FisqZqr9HMMNFfcyEdqJKWfcwr+hGpbhqr7fwaHOb7kDm/daInPfC1uHYEWVw1/XW1nUS9fuzhnf6EkSKhQR8+r2YHH/3pI3ueTfnskZmzbGEmIn9HHGWOv+pQum5334JtPGLRsQjUSoi0V41zGWXz6VMXzjvfP49mXH59TrH06oW6ZMhPZQDchM9btlwpIhY9qJaVDI87mLZJs9DMF6KgnHQg7rQ3DNXS/wrF1KY93NF7lurjU72pg3bSTzZ4zm/z23hVGNcbqSaVrbE3nP4VjqD63cwbjmOm67cgHLN+/j4nlT8uaC9a22h+y3W+d3pjIZjpw4wg11HY6oapSJYgqHeef5o2UyxhS03lXbBwe/z/2Bl7bxm+VWffDQr1bDAOfbY1jvX0fYwSopAFay3ms72jhl1ljXgp40sp4xTfHA5xjTXEeLnTl65pzxTB7VwCUnTM0LB77v70/jT58/y92PikWdW0fch/f/C9RyLxvhbpn8yJl8y73wi1Et98HB75YBeOrN4jcOaxnnG0yxmvmN37/GM+v2EI0IXck0x00d5VZCPXrySNbtamdXW77l3lIfY8LIetpbU3lx8AAPf+oMDnWnOP2IcUDWYvffptMq7qoaZcL/dbWQ5e7g90+GGYYaLTM4xGP5bwcnPns4ScXBziSX/cdfWbP9IImUVR/9hbesjdQwI8TLktd20ZVMu8lGpx4+loWHjeEri4/l5kvnMaoxa7l/+7Lj3ePm+hgNdsLRxJbcDVSwWiK+48jx7n3HUncMIKduTC12VuorarmXiWLdMg7+2jIZY/IibhxU3AeHIMt9OH67f+Clbby05QD3LNvEh06bxaptB1mFtcHpr+lSLDPGNgHw8TNnA7jiPm/aSE6ZNcad5y3mNSFA3P3E7feG8x45e84Evvjuo/i70w7r1zprCRX3MlFstIxDMu0X98LPreI+ONQHWO4Ow8nl7vjOj5rUwp6OXNdJuh9/iLs+sjBvrKXBEvemeCwnmcgr7hNbeo9Pd3odON8oIhHhU+fO6fMaaxEV9zIRHuee/4ZIZfI3VAuh4j44xEOaohT6VlWLtLZbm6GpjKHV5xfvq+H+2GfPdJOPvLTY2aWNddGcFncjGmJus/JiSvLW6QZqQVTcy4S/yGNwTcgs/jj35zbu5bmNwa1qNYlpcKiLRWiqixZMiX9o5Xb2d/TwkTMOH+SVDS6Odd6TyrD9QFfOOX+UVxgXHz85UNgh27+0IR5hTFMdV5w8HREY3Rjn+x+Yz0+f3sixU3pvx/eTDy/knmWbOdyTnapYqLiXiXyfe4HKYTZ+y/3Wx94o+NyaxDQ4XDhvMpNHNXJ7QJOO3W0JPvMrqwvQ/s4knz1/Tk1Wjfzp0xtZs90qr5tIpdnts9y7kxn+++XtRT1XWO0Wxy1jjOVK+c4VJ7rnDh/fzM2XHl/ooTnMnjCCm957XFFzhxsaLVMm8qJlco4D3DJ9CNVSy31wOPeYSXz+gqMCz+32JNz8cOk6tyNQLdGdTPN/H1nr3u9JZVjfml8X5rP3ryzq+RpDxd2yK9WdMnCo5V4mjLFigR2Nz81KzZ+f7MOLWuu5Vx7dNdRIe+fBLu54cgMz7YgWh86eNGt3tuWMdScLN1a7bME0fuex6sPF3bLcVdwHjl4tdxG5S0R2i8gaz9h3ROR1EVklIg+KyGh7fJaIdInISvvnxwO5+ErCYIh7ko168cr0yXepVB4dieJrmVc6v1i2mXuWbc6x2gHW7myjO5n7OvU33XA4c854PnR6bvhhY0isuWO5p0M6kimlUYxb5ufAhb6xJcA8Y8wJwJvAjZ5zG4wx8+2fT5RnmZWPMdkUaOh7D1Wluqglcf/z67sDx994O7+HrLfpBsAZR45zj73djiDc5+7U8emLe1LpG72KuzHmKWCfb+xxY4zz6n4OmD4Aa6sqMsbk1CbJtdzzX8B97SmpVBYdieru+/7o6p3sOZSgI5Hi9QARh2yzam82qbdkwEfeMYvbr1oAWMlJ/v3lsLwBZx9J3TIDRzk2VD8G/I/n/uEi8rKI/EVEziz0IBG5TkSWi8jy1tbqr99hyI2T7s3nri/qyuXE6dnemoVirf09PKuJjkSKT977Eotu/hNv2SV7g7oVORTKFF18whTGjahn0y2LOfuoCUy3y/WOba7rdQ1OtcaPnjGrj6tXiqUkcReRLwMp4F57aCcw0xizAPg88EsRCQx0NcbcaYxZaIxZOGHChFKWURmY3AJffY1zVyqHhz71Tvf41MPHBs55a09H3mZjNZDJGB5b87Z1bLL9SU+cPjpwvkhWrE+fnXXBPP65szhlVu7fpqUhzqZbFvP+k6zqj2Gux9FNdWy6ZTGXnDC1/xejhNJvcReRa4BLgL81tplqjEkYY/baxyuADUBwbFmNkTGGqaOz1o/pxenuj3NXKpPvXHEi37n8BPf+i18+H4B/fex1Lvrh00O1rEDWbD+YtxewYvP+HBfgL1/Ywhd+84p7/9Ud1gfU8Z5vK15a6rPlAerjEb6y+FgAJodY+k50l345HVr6Je4iciHwL8B7jTGdnvEJIhK1j2cDc4CN5VhopWOwiiPd9/enWfdtQd+0p4P2gK/warlXB/WxiBu2B9Bc33u1wc17Owo2ax4ounrSXPJvz/Dp+152x17esp/33/EsP3pivTv2lqdzElgfCFNHNTA5oLwuwMjGuJviXx+L8PEzZ7Pu5osY2RBcjx2Kq/2uDDzFhELeBywDjhaRbSJyLfDvQAuwxBfyeBawSkReAX4LfMIYE5xTX2MYY7VlczvW2OPnfPfJwOxT3VCtDupjETdVHqzY7d4SU8/+zpNcPMhW/fLN++zb/UCu+8Ur6H5XyertB5k7dRQjGnL3FpzX8ajGuLsx6ljw/qYmfhxX1oIZwa4eZXDoNYnJGHN1wPDPCsx9AHig1EVVIwZDRMRNSQ8zWuqiEd1QrRJi0VzLXURoiEXpCknmAfJqsgwEhxIpkqkML27ax3W/WAFkN/Vv/N1q7l9u9X11rOygWjFgld31l/KNRyMkUpkccR/T1PtGKcA5R09k5dcuYHSR85WBQcsPlIlMBhA8PTcLi3c8KvTUUIZjLbLIs1nY4rNqvcJujKEnlaGt23LDlOsb2aFEKjQbFODyO55lwTeX8Mq2A+5YLBLBGOMKO0CT7Ur68oOrWfLarrznOXbKSLfeuoNjnXvDIMcVEQXjoMI+9Ki4lxFBcL6xh1nu8Zha7pXO3R9bxHM3ngfki/tUz2ZiMm249u4XOeGmx9m6r5P9npozTgPooLDJnlSGRKqweM/7+h+54Ad/CV2jE5++YXfW7RKLCvs7c/393T1pEqk0D72yI/B5xjXXMW/aKJ744jn884VHA3DYOEvsRzfF3fWPG9F78wylclBxLxPGGETIumUo3Cm+N5+lMvQ01kXdiJAW3+bhfdedxtlHWeG7PekMT6/bA8CZtz7BVXc+58776M9f4NkNe5j39T/y7IY9Oc9x9neeYNHNS0PXsHVfuGvHaTL92Ktvu2PxaITddj12h/buFKf83z8V/LboXN/h45v5xFlH8Nhnz3TF/ahJLW7JgWLi15XKQVWmTGSMtQmVbSRsCsb5BrVzUyqXOl+m5WHjmnnX0ba4+wRzo2fz8kBnkhffsjY4l9ndjRx2HuzmYFeyoAHQG8aYvA8dsDI/nTK9t105n5aGGH9YtZO2AjVhIPebSSQiHDN5JG/usqpBHjd1lGu5jx+h4l5NqMqUCYPJcct89L9epPVQfnd3CO/4o1QuXpGvs+uonPTNJYFzZ45toq0r6dYb8rdVdDj8xke56eFX+7SOH/5pHYff+Ghg8blkOuOWJ54/YzRHTWqhp5d9AL/bCWDSSMsFM3fqSFfc1Y9eXai4lwmn5K+3gcPG1o7AuX1xyzz+ubNKXptSOg9/6gz+8k/nuPf91ryfKaMaaE+k3BaJYVVAf/7sJsCKOZ91wyOs2X7QPfeHVTuYdcMjOfXjf/qMlTqy42Cu+wUs/77jlpk4sr6oVnXNdflz/u3qk7j/utMYUR9zywqMbioc265UHiruZSJjLGH32uSFLKa+iPtRk3pvNaYMPCdMH82UUY3u/d7EferoRozBbdnnj6Lx+q+d53L88r98YYt77p5lmwF43ZWov5oAABrXSURBVFPqYMaY3MgWgPOPncjJh41hz6Eebn3sDeqiEZrqYnTZv9/xoQcR1C9gbHMdp9rlBm6/agH/9ZFTGK8bqlWFinvZcDZUsyOFslDjvQiDUvmEVTyEbCGu3W2WFe1vzuKV03hEOOXmP7lRL95Suy225e2NuBkfUMhr7pSRnDUnW6PpqkUzANi013rOdx45PnS9YYxpruNdx0zs9+OVoUFVpkw4Gape273QV/G6InzuP/nwQpZ+4exyLU8pM8VY7mBtnIIVjugl4dmI7ehJ09qecGPTveLebIv7AU94Y2dAaGVTfSwny/T9J1lVuJ1viWfY4r5gpmaNDhdU3MuEATtDNTtWilvmgrmTOGLCiDKtTik39b38D6e54m6FMx7sSrJlbydX/PhZNu/tIJFKc8HcSTmPabI7F3mtdGdsd3s3n77vZf5n9c7AujXN9bGchCNnk/Tujy3iu1ecyCS7dow2iRk+aA/VMpExfXDLaChk1dOb5T7RjjZxLPe27iT/+94VvLqjjafX7SGZNsybOoqRDXEeeGkbkPXPe3EaZry2s41HV7/Nw6/sYGKAW6a5LppTzMsJkzxy4giOnDjC3aQN646k1BYq7kXypQdXc8nxU3hHAd9lkFumUCq6inv1U0jc//CP7+SZ9XvcsEEnAWjvoR43NNbJXK2PRwLDEL3stR/jbYXnlDrw0lwfY0xzvuXuMHfKSD597pFcuWgmm/d08HZbNxERxmnses2i4l4kv3x+C798fgubblkceN7KUM11y/g30RzqYhrnXu0UEvd500Yxb9ooN0rFwZvctHW/VSXbimgJt6T3HrJCIL2Nqv1Nq8EKZ/TGofst9EhE+Py7rdICjstIqW3UhCyCYrIIs3Hu2bFC6d5quVc/vWUZN9ZFabQF9iTfJuY2u6xAfTzSqw98T4FEOD/N9dGiqzYqwwO13IugmE0og1M4LKvuhar6qbhXP17L/ZbLjmf97kOc6wsXHNtcx/YDXYxsjCOSLSb3wiar9np9LBroZ/ey15O85OVrl8xlxeb9/GntLhKpTN6GqqKouBdBuijL3dpQ9eaDJFTca5b6WNbtcdWimYFzxo+wxL25LsaYprqcLFPrOSK9iruXSSPr2WXXjZk3bRQfe+fhnPTNJSRSPTTXx9xsWEUBdcsURTHtTv2FwwC6C7hliolzVyqb3qJlIFuLZdyIOv7zQydz1lETuNXTj7U+FuGT7zqCedNGssjuXhRWeXHWuGb32ImYcVrZNWoUjOKjKHEXkbtEZLeIrPGMjRWRJSKyzr4dY4+LiNwuIutFZJWInDRQix8sirLcMXZdGXXLDAecDNUwY9mJV583dRSnzBrLPR9bxAcWzsg+RzzKERNG8Id/PJPp9iZnoV6mYPUzdXBCLW+57ARmT2h2WwFetmAa//ucI/p3UUpNUazK/By40Dd2A7DUGDMHWGrfB7gIqzH2HOA64I7Slzm0FOVzd0Ihc9wyBTZUtfxA1VMXjTCuuY5vX3Z8wTk77ZZ2c6eOzBl3rGxvCQOn4NyRE7OJa049GKdCY9Tz4mqyi31dOG8yf/7COcRsg+H7V87nXy48pn8XpdQURamMMeYpwN/o+n3A3fbx3cDfeMbvMRbPAaNFZEo5FjtUOOIe1hjZkF84rLtApx213KufSERY8dULuPKUYH87wBffczSN8ShzJuVmGjtlArzivspulXfpgmlMHtnA5JENXLbAKiFwwvTR9u8s6yUoNU4pG6qTjDE7AYwxO0XECRWYBmz1zNtmj+30PlhErsOy7Jk5s/AbpBJwxD0Sou7+TkyQ65aJiOWXB/W5DxcuO2k6l9k1XryMqI/R2p7I8dt/9IzD+dKDq3nHkeN47kvnueOfOX8OD63czpLXdiEivPPI8XT2FG68oSgOAxEtE6RceX4NY8ydwJ0ACxcurOiCF86mVZgkZzNUs3iTTcQTCxcrYLl/5B2z3NreSu3SbDes9jbw+OCpM/ngqcFGzvwZluV+2YJpnHfspMA5iuKnlC96uxx3i33r5EdvA2Z45k0HgjvzVgnFumUiIjnW/fNvZVur5ZR4LSDuYd8MlNrhb089DMiWBe6Nw8Y1s+mWxSrsSp8oRdwfBq6xj68BHvKMf9iOmjkNOOi4b6qVrLgXFt+gwmG5lnt2vFAYnbrihwdXL5rJ+psvcis1KspAUJRbRkTuA84BxovINuDrwC3Ar0XkWmALcIU9/VHgYmA90Al8tMxrHnT64pYphLXVaj1PIZ97UEccpTYp5JpTlHJRlLgbY64ucOo8/4CxCrFcX8qiKoV0xrD3UKLXDVWn9oy/cFgOnvFYgbAHdcsoilIu1HwI4dbHXmfRt5a63eQLaa+T4+SPlvGS43Mv5JZRcVcUpUxobZkQlqzdBWTLrha03O3b3LJhuYhYiS896UzBioKRiPDSVy8Ide8oiqIUg4p7GLZquz73gpZ79nyhDwBBqIuJJe4F6rlHJLy2iKIoSrGoWyYER9Szlnmhedatv3CYFxGI2xuphT4A1C2jKEq5UHEPwRF174Zp8DzPhmqB5xKy8e2FnkejZRRFKRcq7kWQjZYJPp9TNLKg5S6uuHuf596Pn+oeaz1uRVHKhYp7CMb1uVu3YUlMYLlbCtnuluUu9nF2zhlHjnerBKq2K4pSLlTcQ3B87mm7W0ch8fVuuIbFuWfdMrmnHLeOxrkrilIuVNxDcCz3bIGnQklM2bNF+dwLzFG3jKIo5ULFvQiS6XDL3ZX+sFBIkWxNGb/l7kbbqLgrilIeVNxDcKJkUulwt4mz4RqNRIoKhfT75Z0PB42WURSlXGgSUwiO6PbYlnsh4XbFXfKF28HrlskYw62Xn8A0u2+mg8a5K4pSLlTcQ3DcJY7lXkh6XXGPRooKhexJZ3IaJeNJglIURSkH6pYJwYliSWUcyz3cLROLFK4K6bXck6ngxtnqllEUpVyouIfgj5YpJNyO+Ecj4YXDHJ+7t72aF3XLKIpSLlTcQ3AkOOVGy/Syoeprs+fF65Zxom+yv8fesNX/hqIoZULlJISs5V7chmosGu6WcUIhe/zirqGQiqKUmX5vqIrI0cD9nqHZwNeA0cDfA632+JeMMY/2e4VDiqW6yV46MWVDIUPKD3gyVP2Wu4MmMSmKUi76bbkbY94wxsw3xswHTsbql/qgffoHzrnqFXaP5Z4Kt9xTRW2oCh9cNBOAs+ZMyP099q1a7oqilItyhUKeB2wwxmzurbhWNeHUjHHEu7dQyDBxFoHjp49i0y2L885NGFHP223dJFLp0hasKIpiUy6f+1XAfZ77nxKRVSJyl4iMCXqAiFwnIstFZHlra2vQlIohm8TUSyhkLz73Qtx21XwA5kxs6fcaFUVRvJQs7iJSB7wX+I09dAdwBDAf2Al8L+hxxpg7jTELjTELJ0yYEDRlyMmPlgmel/KWH/DI+JWeRKWwbzSnzR7HW9++mHnTRpW2YEVRFJtyWO4XAS8ZY3YBGGN2GWPSxpgM8BNgURl+x5CQyeTWlim0WepNYvJ+APzr5Sfw5y+cXdTvqiV3lqIoQ085xP1qPC4ZEZniOXcpsKYMv2NIcCx3J1qmtySmiEieSMciwTXcFUVRBpKSNlRFpAm4APgHz/CtIjIfSxs3+c5VF25tmVyf++62bloa4jTWWR2UbG23fO6+p4g6lSBV3BVFGURKEndjTCcwzjf2oZJWVEG4lrvP577oW0s5ZnILv/7E6YxsiOeWH7DnXDRvMmC5aqCwS0dRFGUg0AzVEJx67j0B9dxff7udE256HPAXDhNe+PJ5/PCqBe4YqOWuKMrgoiV/Q/BHy4hkN1ndOca40TKO+E9saXDPuz73AV6roiiKF7XcQ/DXcwdI+Mr17u3ocQU/Fs2X8KzPXeVdUZTBY9iI+6ptB5h1wyM8u2FP0Y8xbm0ZS9CNIS+LdNOejpzyA36yPndFUZTBY9iI+7INewF48o3is2H9VSENhu5kruX+1p6OnB6qftxiYKruiqIMIsNG3B2cTdKi5tq3jlsmk4HuZK7lvqutO5uhGuB6ccZU2xVFGUyGjbj3y+Xt68SUMYZun1tmd3vC9blHA3zuETtrVX3uiqIMJsNG3B36YLjn9VAF8twyu9sSoT53azyilruiKIPKsBH3/iQR+eu5Z4zJc8vsbu8m7UliCiIaUuddURRlIBg24u5QyHD/zK9e5tfLtwbOddwyVrSMz3JvT4T63MEpKKbqrijK4DFsxL03bX1o5Q7++bercsaczVcnFDLYck9ko2UCfO4QHP+uKIoykAwbcXfom8899zHG5EfL9KQyrjVfyOcejUR0Q1VRlEFl2Il7X/B/EGzc08F//XVT3rzv/PENoLDPPRbRsmGKogwuw07cTYDX3akdUwwrtx4oeK6Qz103VBVFGWyGjbiHuUX8m6QAL7y1r8+/o6DlHtJbVVEUZSAYNuIeRk+AuH/gP5f1+ji/j73QB0g0IlrPXVGUQWXYiLsjrUEbqkGWezE0xKNFzYupW0ZRlEGmZHEXkU0islpEVorIcntsrIgsEZF19u2Y0pdaHF09aX75/JY+1ZAJstyLoSFe3J8vqhmqiqIMMuWy3N9ljJlvjFlo378BWGqMmQMste8PCv/nD6/xpQdX86xdBbIY/GV8w1h8/BQumDsJgPpY8Za7mu6KogwmA+WWeR9wt318N/A3A/R78tiw+xAAf3mzlc6eVFGP6YtbJhoR19derOUe1DhbURRlICmHuBvgcRFZISLX2WOTjDE7Aezbif4Hich1IrJcRJa3thZfY7039nf2AHDnUxv5yn+vKeoxQeLeXBdslcciQsQVd/W5K4pSmZSjh+oZxpgdIjIRWCIirxfzIGPMncCdAAsXLuxD3mg4+zuT7vGWvZ3ucSbEBx/kc2+uj9HRk++uiUaEaKZv4h6NiFuiQFEUZTAo2XI3xuywb3cDDwKLgF0iMgXAvt1d6u8plgO25Q64FjbgimvQRmuQz310Uzzw+aN2fXYo3i0zbXQTU0Y3FjVXURSlHJQk7iLSLCItzjHwbmAN8DBwjT3tGuChUn5PX0h5LGRvGHo6xHIPcstERDjn6Al5495EpWI3VP/1/cfzwyvnFzVXURSlHJTqlpkEPGgn78SAXxpjHhORF4Ffi8i1wBbgihJ/T7/wCrHTLSlI4oPcMsl0hua6GBEBr0fFm7hU/IbqsEknUBSlQihJ3I0xG4ETA8b3AueV8tzlwFtDPax8TJDlnsoY4lGrDrvXX+919TQUabkriqIMNjVtUuaIu8n2QfUTZLmn0oZYNJLXZCPHcvdE1Lz01QtKXq+iKEq5qClx92+WBrllgiz4oA3VnnSGeDSSF8IYjWT/ZF7LfWxzXX+WrCiKMiDUlLj7ww0DLfeAkESv5e58QKTSGdct48XrPq8v0ueuKIoy2NSUOqUyfss9e+yIun8O5PrcnQ+IZNoQi0TyyvgWstwVRVEqiZoW9z++uovP378SyIp2bz73ZNow//88zqFEingsP7O0P9EyiqIog01NqVNQR6XfvbwdyLplgiz3pOdxPekMB+ws13gkf0M1miPuarkrilKZ1JS4J9OFE5Uct0ygz90j7t4PiFhU8DdXiqrlrihKFVBT6hRWvyVruQeHPTp4PyDiBUIhjee8oihKJVJT6pQMyVRyTgVN8T7OexyPSk7SEuRG4GilR0VRKpWaEvcgf7pDNs49w8GuJO/70V/Z2GrVfvda614XTSyS/+fR2uyKolQDNSXu6QCXi4Mj/GkDS9fu4pWtB7h96Tog11rvTGQTmuKxSF5iVEQksD6NoihKJVFT4h66oepJYnLcKc5sr7jf/ud17nE8oA67umIURakGakrcUyHins5kN1Qdv7mj28m0YfwIq3zAC2/tcx8TjQh+T49/g1VRFKUSKUcnpoohGeKWyZYfyI45LpdkOsO00Y2cf+wkfvXiVvd8xpj8ejU+cf/8BUdxzOSWUpeuKIpSVmpK3MNCITMey92uP++6ZVIZq0hYS0Pun6MnbchLaPUZ7p8+b04pS1YURRkQqt4t8+qOg/z+lR1Ab6GQ2Q1VV58dt0zKEIsKLQ25rfWSqUxeuQJ1yyiKUg1Uvbgvvv0Z/vG+l4Fwn7sj0t6IGoPhxU37eGHTvkDLPZnO5PncNW9JUZRqoN9SJSIzROQJEVkrIq+KyGfs8ZtEZLuIrLR/Li7fcsNxrPMgH3jaU8/dyVI1Bq748TIAW9x9lntaLXdFUaqTUuzQFPAFY8yxwGnA9SIy1z73A2PMfPvn0ZJXWSSOW+a7V5zIOE/zjP0dPTzxRitgWe7JlN1P1aPb8ajkWe7vOW5yns9dvBmqms6kKEqF0u8NVWPMTmCnfdwuImuBaeVaWF/JZIybqBSL5iYaXf/Ll9zjdMa4Waheqzzmc8s8dP0ZzJnUQlN9lJ7ODFE75t1fSExRFKUSKYsHWURmAQuA5+2hT4nIKhG5S0TGFHjMdSKyXESWt7a2lryGnnQmK+6R3KbW63Yfco/TGeNa+N4PgLpohJEet4zTZen02ePcx4Hllvmb+dZn2PwZo0tet6IoykBQsriLyAjgAeCzxpg24A7gCGA+lmX/vaDHGWPuNMYsNMYsnDBhQqnLIJHMuOV6Y5FIjjvFW+a3vTvlumi8c2KRXLdMnb1z+vEzZwO4SU4RgXcdM5FNtyxm1vjmktetKIoyEJQk7iISxxL2e40xvwMwxuwyxqSNMRngJ8Ci0pfZO4lU2o2WiUVzLfe053hvRw9Pvel8U/CU941FGOPx09fFrD/NyYeNYf3NF3HyYdYXENENVUVRqoBSomUE+Bmw1hjzfc/4FM+0S4E1/V9e8SRSXrdMJMfnkvaESH71krnusbeKZFQk1y3j6Y8ai0bcUsEaLaMoSjVQSobqGcCHgNUistIe+xJwtYjMx5LXTcA/lLTCIkmk0m6Io99y94r4zLFN7nFbVzJwDmQtdwenDIFuqCqKUg2UEi3zDHnJ+AAMWuijl+5kxq0KGY9EcjZLvW6ZprqsRX7AI+5OctPFx0/m0dVv0+jrj5ox2Q1VRVGUSqdmasv0pLMbqlGf5d6TymalesX9YKfHcrc/GG67cgE3XtSdZ7k7hr1qu6Io1UBVJ9N7a8kkkrmhkHkFv2ya6rKfZwcC3DJ1sQgzPK4bB7XcFUWpJqpa3LuT2a5J3miZeDQSIu5Zy91bRTKocbYX5/lU3BVFqQaqXNw9lnsqY5fzdZps5Kr74uOnsOzGc3PE3UtY0TGAw+2Y9tFN8dB5iqIolUBV+9y9lvs//GIFYNWIgWwk5BETmtnQ2sGCmaOZMqox5zFewpprA3x58bGcP3cS86aNKn3hiqIoA0zNiLtDLGJ9GXEs9+9/YD4vbtrHZSdNB6A+FvxlpTdxb4hHOfuo0jNpFUVRBoOaccs4TB7VAMDUUY0AzJs2io+fOZuxdvapN8P0e1ecyAcWWqI/Vt0tiqLUEFVtuY9pjnPlwhncvzzb99Qp9fubT5zOis37iYZkHb3/5OlcdtI05s8YwyUnTik4T1EUpdqoast9+pgmvvG+43LGnPj0qaMb+V8nTu31OUSED546M6f0gKIoSrVT1eIO2eqNDuNH1A/RShRFUSqHqnbLAEQiwg0XHcPps8fx6OqdboneMH70wZNoqg8OiVQURakFxBTK9hlEFi5caJYvXz7Uy1AURakqRGSFMWZh0Lmqd8soiqIo+ai4K4qi1CAq7oqiKDWIiruiKEoNouKuKIpSg6i4K4qi1CAq7oqiKDWIiruiKEoNUhFJTCLSCmwu4SnGA3vKtJxqQa95eKDXPDzo7zUfZowJrEVeEeJeKiKyvFCWVq2i1zw80GseHgzENatbRlEUpQZRcVcURalBakXc7xzqBQwBes3DA73m4UHZr7kmfO6KoihKLrViuSuKoigeVNwVRVFqkKoWdxG5UETeEJH1InLDUK+nXIjIXSKyW0TWeMbGisgSEVln346xx0VEbrf/BqtE5KShW3n/EZEZIvKEiKwVkVdF5DP2eM1et4g0iMgLIvKKfc3fsMcPF5Hn7Wu+X0Tq7PF6+/56+/ysoVx/KYhIVEReFpE/2Pdr+ppFZJOIrBaRlSKy3B4b0Nd21Yq7iESBHwEXAXOBq0Vk7tCuqmz8HLjQN3YDsNQYMwdYat8H6/rn2D/XAXcM0hrLTQr4gjHmWOA04Hr7/1nL150AzjXGnAjMBy4UkdOAfwV+YF/zfuBae/61wH5jzJHAD+x51cpngLWe+8Phmt9ljJnviWcf2Ne2MaYqf4DTgT967t8I3DjU6yrj9c0C1njuvwFMsY+nAG/Yx/8JXB00r5p/gIeAC4bLdQNNwEvAqViZijF73H2dA38ETrePY/Y8Geq19+Nap9tidi7wB0CGwTVvAsb7xgb0tV21ljswDdjqub/NHqtVJhljdgLYtxPt8Zr7O9hfvRcAz1Pj1227J1YCu4ElwAbggDEmZU/xXpd7zfb5g8C4wV1xWbgN+GcgY98fR+1fswEeF5EVInKdPTagr+1YCYsdaiRgbDjGddbU30FERgAPAJ81xrSJBF2eNTVgrOqu2xiTBuaLyGjgQeDYoGn2bdVfs4hcAuw2xqwQkXOc4YCpNXPNNmcYY3aIyERgiYi8HjK3LNdczZb7NmCG5/50YMcQrWUw2CUiUwDs2932eM38HUQkjiXs9xpjfmcP1/x1AxhjDgBPYu03jBYRx/DyXpd7zfb5UcC+wV1pyZwBvFdENgG/wnLN3EZtXzPGmB327W6sD/FFDPBru5rF/UVgjr3LXgdcBTw8xGsaSB4GrrGPr8HySTvjH7Z32E8DDjpf9aoJsUz0nwFrjTHf95yq2esWkQm2xY6INALnY20yPgFcbk/zX7Pzt7gc+LOxnbLVgjHmRmPMdGPMLKz37J+NMX9LDV+ziDSLSItzDLwbWMNAv7aHeqOhxE2Ki4E3sfyUXx7q9ZTxuu4DdgJJrE/xa7H8jEuBdfbtWHuuYEUNbQBWAwuHev39vOZ3Yn31XAWstH8uruXrBk4AXraveQ3wNXt8NvACsB74DVBvjzfY99fb52cP9TWUeP3nAH+o9Wu2r+0V++dVR6sG+rWt5QcURVFqkGp2yyiKoigFUHFXFEWpQVTcFUVRahAVd0VRlBpExV1RFKUGUXFXFEWpQVTcFUVRapD/D58IwleytWorAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "num_episodes = 5000\n",
    "durations_list = []\n",
    "render_flag = False\n",
    "lr = 3e-2\n",
    "num_batch = 5\n",
    "\n",
    "def init_buff():\n",
    "    global state_buff, action_buff, reward_buff, value_buff\n",
    "    state_buff = []\n",
    "    action_buff = []\n",
    "    reward_buff = []\n",
    "    value_buff = []\n",
    "    \n",
    "def run(env):\n",
    "    actor = ActorNet(env.observation_space.shape[0], env.action_space.n)    \n",
    "    critic = CriticNet(env.observation_space.shape[0])\n",
    "    optimizer_actor = optim.Adam(actor.parameters(), lr=lr)\n",
    "    optimizer_critic = optim.Adam(critic.parameters(), lr=lr)\n",
    "    \n",
    "    init_buff()\n",
    "    for ep in range(num_episodes):\n",
    "        state_numpy = env.reset()\n",
    "        for t in count():\n",
    "            state_torch = torch.FloatTensor(state_numpy)\n",
    "            policy_torch, value_torch = actor(state_torch), critic(state_torch)\n",
    "            action_torch = policy_torch.sample()\n",
    "            action_int = action_torch.numpy() #.astype(int)[0]\n",
    "            next_state_numpy, reward, done, _ = env.step(action_int)\n",
    "            if render_flag:\n",
    "                env.render()            \n",
    "            if done:\n",
    "                reward = 0\n",
    "                \n",
    "            state_buff.append(state_torch)\n",
    "            action_buff.append(action_torch)\n",
    "            reward_buff.append(reward)\n",
    "            value_buff.append(value_torch)\n",
    "            \n",
    "            if done:\n",
    "                durations_list.append(t+1)\n",
    "                break\n",
    "            state_numpy = next_state_numpy\n",
    "            \n",
    "        if ep % num_batch == num_batch - 1:\n",
    "            calc_discount_reward(reward_buff)\n",
    "            optimizer_actor.zero_grad()\n",
    "            optimizer_critic.zero_grad()\n",
    "            for state_torch, action_torch, reward, value_torch in zip(state_buff, action_buff, reward_buff, value_buff):\n",
    "                advantage_torch = reward - value_torch\n",
    "                critic_loss = advantage_torch.pow(2)\n",
    "                policy = actor(Variable(state_torch))\n",
    "                actor_loss = -policy.log_prob(Variable(action_torch)) * advantage_torch.detach()  \n",
    "                actor_loss.backward()\n",
    "                critic_loss.backward()\n",
    "            optimizer_actor.step()        \n",
    "            optimizer_critic.step()\n",
    "            init_buff()\n",
    "        \n",
    "        if ep % 100 == 99:\n",
    "            clear_output()\n",
    "            print(f'Episode:{ep}')\n",
    "            plt.plot(durations_list)\n",
    "            plt.show()\n",
    "            \n",
    "    print('Final results')\n",
    "    plt.plot(durations_list)\n",
    "    plt.show()    \n",
    "\n",
    "env = gym.make('CartPole-v0')\n",
    "run(env)\n",
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}