pilipolio/DoomBasic_new.ipynb

## DoomBasic_new.ipynb
{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gym\n",
    "import ppaquette_gym_doom\n",
    "from ppaquette_gym_doom.wrappers import SetResolution, ToDiscrete\n",
    "from gym.wrappers import SkipWrapper\n",
    "from gym import wrappers\n",
    "\n",
    "from skimage.transform import resize\n",
    "from skimage.color import rgb2gray\n",
    "import types    \n",
    "\n",
    "    \n",
    "def resize_observation_patch(env, size):\n",
    "    \"\"\" Because unfortunately SetResolution also impacts the recorded videos...\n",
    "    \"\"\"\n",
    "    def resized_obs(obs):\n",
    "        return resize(rgb2gray(obs), size).reshape(size + (-1,))\n",
    "                                                           \n",
    "    original_reset = env.reset\n",
    "    def reset_with_resized_obs(self):\n",
    "        observation = original_reset()\n",
    "        return resized_obs(observation)\n",
    "\n",
    "    original_step = env.step\n",
    "    def step_with_resized_obs(self, action):\n",
    "        observation, reward, done, end = original_step(action)\n",
    "        return resized_obs(observation), reward, done, end\n",
    "\n",
    "    env.step = types.MethodType(step_with_resized_obs, env)\n",
    "    env.reset = types.MethodType(reset_with_resized_obs, env)\n",
    "    return env\n",
    "\n",
    "WIDTH, HEIGHT, CHANNELS = 200, 150, 3\n",
    "\n",
    "# (see https://github.com/ppaquette/gym-doom/blob/master/ppaquette_gym_doom/doom_basic.py)\n",
    "def create_env(seed=None, monitor_directory=None, size=(WIDTH, HEIGHT)):\n",
    "    env_spec = gym.spec('ppaquette/DoomBasic-v0')\n",
    "    env_spec.id = 'DoomBasic-v0'\n",
    "    env = env_spec.make()\n",
    "\n",
    "    if seed is not None:\n",
    "        env.seed(seed)\n",
    "\n",
    "    if monitor_directory is not None:\n",
    "        env = wrappers.Monitor(env, monitor_directory, force=True, mode='training', \n",
    "                               video_callable=lambda episode_id: episode_id % 100 == 0)\n",
    "\n",
    "    return SetResolution('200x150')(\n",
    "        SkipWrapper(repeat_count=4)(\n",
    "        ToDiscrete('minimal')(env)))\n",
    "    #return resize_observation_patch(size=size,\n",
    "    #                                env=SkipWrapper(repeat_count=4)(ToDiscrete('minimal')(env)))\n",
    "\n",
    "env = create_env()\n",
    "\n",
    "\n",
    "NOOP, SHOOT, RIGHT, LEFT = 0, 1, 2, 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(150, 200, 3)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWQAAAEQCAYAAACHqA73AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvX+ILNl1Jvh1K9wOWWE7xgrbYSZgE5SYHFSwuaiwy7hg\nHvixfkY9+MEIpgcL1MsKVrD6Q38MrGBldpadBZs1jAY04AUvK8MY2tCGNkjQhm54gicoQQlKUM2m\nIAUpSEEIwhCC6J2QHO7aP+79zj1x42ZmZNXr7no7cf6p9yIj7q+4ce+553znOy/c3NxgkkkmmWSS\nD19e/LAbMMkkk0wyiZFpQZ5kkkkmuScyLciTTDLJJPdEpgV5kkkmmeSeyLQgTzLJJJPcE5kW5Ekm\nmWSSeyLTgjzJJJNMck9kWpAnmWSSSe6JTAvyJJNMMsk9keiom6Po5qWXXsKLL7p1/L333gMAvPji\ni3jppZcAAF3X9X7Toq/tKgcAXnrppdHl7Cvrw2qTLoe/6XLYpg+rb/exTc96vI9tk1/GXdq0q6xj\nx/tQm44Z7/vWpn3lRFEkf+/SN13Oh9mmd999t7q5ufnVwc2eHLUgv/jii/iVX/mVnb8nSQIAKIpC\n/r/ZbAAAbdsCAJqmGVVXURRSHqWqqluVo9vGcvTfQ8Jn0zRFnue9+rfb7ei26HIAIM/zXjm63DFl\n+X3TY/0sxudZvbPbjDNw9/FhWc/inT3rOa3f2TSnXVmhOX2X8Ql9G8eW9Szm9Pe///0fjrn/6AU5\njmPplC/s5Gq1AtDvTJZlAPoDvE+22+3g2dlshrIsAbid7lBZnBR8SVEUSXlxHPfu2SWhl8dJXBTF\n4GWPLUe3hRO667pR49M0jZTH+1mW/u2Q1HXd+z/7BbhJPFbquhYtQreF4zx2fDgWVVXJs7w2tl8U\ntkc/y7kzRlivbjvbdOxHqf/NtjRNM+p962fZpyRJ5Hs4dlzYryiKem05pqwkSXoLMZ8d+z3ocgAz\nZ/hv3a+x5ehvnMJy7qIU+GUc0x625fvf//6oZyYb8iSTTDLJPZFjbcjI8xxd1x3UKgGjeVKbmM1m\nAICzszOs12sATjvbtSP7Gk3XdZjP59IWwO1ah3Z1tjeOY2kLyxrTFv7ma1fz+Vx2zLFt0f3i+GjN\ncszY6vqojVJL2Ww2A03sUFu0Bupr3GPb03WdjI/WCrXGcozsO42NaQvL8I+ct2mP1vL9U8XYtugj\nNLWusW2J41jeEefvZrORtowdJ/aD7cjzXL7Hse9Zn1r4DVEr1mbFQ+KbJ9q2lXE5ti1Zlg3acozp\nhe3RbQHMezq2LXw/emzHyqQhTzLJJJPcEzlaVei6Dnmej9ZQudPQrtw0zcAWtu95XcZ2uxWNgLur\ntsGO2cnath3cp+2mh9rDtrAd6/X6qOf9snif3pnZt0P98dtC7SLLMtHKbmNLZn9YXpIko/tD4f23\n1Y7vKmPn1j4J2X5vo7FzjgJOextzItMSRdHAQdV13VF9CznN1uv1aI0/ZF/VmjFwO2cZx3TsyVu3\nRffnWE1/n7342PdTFIV8NxyLy8vL430ex9z8s5/9TBZFDsRyuQRgJslYM4Y/mMvlcvRE9T24/sJ8\nmzLiOMZisZB++PftKwNwE4plxHE82sPM+9j+xWIhL/RYs4N2XPje/LFlxHE82CTGPE/xvdj6qH3s\n5pAkiRwDb7uosg1ajl3EdBnHOAT9d5Bl2dGOUl2/dowDQ4fsMWVwrtR1PXo8/CO5RhuMLYP1hxAi\nx5Shv3fgONPCsyjDX8P4PDB+QwjJZLKYZJJJJrkn8sIxKZx+4Rd+4eY3f/M3BxoHsFubosaknSKH\nnt9Xhl+eFh9Pedcy/B1bt/1QGYfasE98k4HfhruWcUh8ze6uZdzmeT57lzbodgCHzWLv17zc14ax\nc+qu39ezKON5+772jYU2T/ht2FVvqB9jy/jOd77z3Zubm9O9hWLSkCeZZJJJ7o0cpSH/0i/90s1v\n/dZvBX/Tzg4dFDBW/F1lrCNpXxm3kbuUcdcxYBk+NOkuZdxGntUYAGYc7vIe7zIGLOO2cpdxnL6H\n+/E93Jcx+Pa3vz1KQ76VU087NnSMuI+80J5HekC140l7wlkmr2kHBMWv2y/DlziOcXJy8qHUz0mj\n66cTxq9r1zXtMAg57g71XdevHRa73t+u+jWecl/9eg6wfj2J99W/qw/+0bCu61Hj96zqpwNLY+eP\nfX/6eO1HjO2q33eg3fX9sQ/76tdlaHwv5TbjB5g56Mce6Lmi6/ev6W+ASK1j6ve//9vU7zuWj/kG\nWf+3v/3tnfdpmUwWk0wyyST3RI7SkAkPK8tyFFRltVoN8Kiz2UxgO4dgZX7EGOsGxkFLmqaRnZFa\ngiaHObb+oigGkLlD4tcPHB/RpzGox8bTs69Zlgn8bGzbtWZ32/r1kXNs/Rqm5Y/RWK4PzUHhQ9YO\nPa8dkn79t8HZas1qTNtDzsyx/CSaX0JjjY+pX3MwUMZy0CRJIt8L/67X69Fzztfqj+HG8Llv4jg+\nCpsc4ubQa92h+n0ujTiOcXFxcbBeLZOGPMkkk0xyT+QoDbltW1xfXyPLsp5tEti9A/nXi6IYPLsL\nnO5DW7quk11Qg9P1Pbvq532z2WwQUTOWS0M/u6t/u54FhhGBh0D5OvhFcw+w7WMCV3RQhmYJ0/fs\nEh2gciy/heYfOdYZ47ddy7GRfxqGNfZZzYOhAyjG1geEIWaHNCx9kuA4+5Fjh0TTafptH6vh6ffN\n9zy27UVR9HggdBmHns2ybOAzOEazDj07Zs6F+q0ji8e0fTabHc1pE5KjZvd7770nocd+CPP5+bl0\nYh/VXejZ2Ww2+tlQlB9gBnDfs3qC8UO/y7PcENq2HfWs/qD4rDbp7BM9Kfns2COsXtT1s8Dh47+O\nutNtBg4T/4ToKkNY1UPiL3BjaRQpWZbJM2Oj7LQj69gIPf1syAk05lltKhl7XA6ZWbgwHfMs5dhn\nQxFrt3n2+vr61s/6YdyHntfP8rn3+9lDMpksJplkkknuiTyzSD1gfFTTs37Wf/7Yo8KuyJ8x9e+K\n+Lnts4eilfZFwB2Klvqw3s/79ewYedbv5y7P3vXdjnkWCEcD7mqn/6z//Pv5fvZF/un690X8HXo/\nx0bq6WeBZ/dup0i9SSaZZJLnTJ5ZpJ6WZxHZcmyE112igu4akXWXKKTbtBW429je9b0AH9zY3jVS\n7ph5dJfouA9rbCkfxjy6SxTmXd7Lber7IL/vUJ1vv/32pCFPMskkkzxPciuUheZ1PeRZDIGlx3AF\nV1U1AJhvNptRnsxQCOihOseQb+9rq27nMXzIbKtOvzTW07srWeuhOjX4fmwffVvhMX30wf537eMx\nUCZgfEZmP7CA9QHj+6hJ5I/tow46GttHPabAB9PHZ/EexyKMQn1s2/1Z1XUfWd/YOeD3cSznc6iP\nt0FeHM1lsdlsUJalVP7gwQMAu6NxQjjk09NTeSZ0D9CHddGAPpvNBvCn0CBp4nm2U9e5qz6/PD8O\nfizmeDabDXgwDuGkdZ1joouapglm/D1UH9DHF/tZGw5hskPp3sce5bTT6lB9QD/n4FiYXKjOscfV\nUM66MaJJkI6NZEzTdMCXcczif0x9cRzLsxrKOXZhZM66EK/Jrvr4rI9r3lWnz+HRtu1oTDD7ptvJ\nZ8e0U2OJD9UZaudd8MeUyWQxySSTTHJP5FYJzxgcAvQDD87PzwFgb5DHdrsdpF86FFSid61QQMi+\n51iXDkbx0zWFtItQAMtisdj7jNYC+Az/HuICCAV/+JE/+57Rzx06DoYCPsZquzp/n5/eaFf//Jx/\ncRyPCrTQkXXHBmhQxnJf+M8A44NvdDqgsdpcCOI21txwbADHXYI3dPAWgzbGHvv1fBxzdNfBS2MC\nxYBw38gfcWhM/OCOQ9/ovmCU257gfJk05EkmmWSSeyLPlKDet9XdBnx9CPC9C8byQdV1W0D6NBbv\nb/t80ff72su+YAFdRyj8e9dzt3nmg2zfB1nXfW/fB1kXnxkLezvKZNF1Haqq6jmeQmTNPPJtt9sB\nUfohkmzfnKGvhQik9T3+EUQTyoeeG1uXLoNtP9Qfv16OG5/hNX8MQkT/JycnuLy8HLRljIwZu33P\n7KpLk8fvI+4OkdZrh+C++aNRMrqusSTvFJ/LYuw4VFV1q2dY1136EyLcv+14Uw59D5SiKAbkPofm\nD7+hEJH+rvlz7PcQIrwfO3/GJqrw+wNg9Lfnrz+Xl5dHm9gmk8Ukk0wyyT2Ro0wWL7300s2v/dqv\nARgauJumGUWVt4smLyQhI/oYzKTGEo6l8fPxh3qnf1ZtO7YO/RzrG9P/kMPiELzp2DG7TR2hMTsE\nS/og3gufo7P3NnPm/RqzZzGXDzka7/KdHUuXyTEeW8cHNZf1c2Paduz696Mf/WiK1JtkkkkmeZ7k\n1hqyL7eBkRzLh7yrDmA37ORYCM5dtDjWcXl5ORo+c2wEkY4EG1PHWM5nitZgPoixvQ918P6xkKsP\nq44xQQ66Dto+x8K/bjMXx/Q7SZJRMFUtx8xDtukYyNwHvV6NdeodtSB//OMfv/mDP/iD0Z74Qx7X\nQ970McQgx6ID7tImHbk11rt6SI5FKtzGS/xht+lZjemhNu26P1T/vrl1G4/92DaNRZeM8ewfQrCM\n/X5uOy9C9e8ihLotouHYNu2q91nM1X39PtSmiVxokkkmmeQ5k6M05F/9J7948y9/71OIowhPVn1C\nnRCkZhdkZQyMZBcsZhfk5Nj7gT6/xRgolc46fQxsBuhrgqHx2QW94/3+WB17v+b1OATt86FCGr6z\n613xfh9W9CzeLXC7d0XxOSmOGc/Q+IyBV4bmfpZlo9+VHnu/PC37YKXP4n7K6enp0ZCx23wrH8S7\n2ne/fle8/y7r2iwx//677/5g0pAnmWSSSZ4nOUpDztLk5uXzE8Rxh7blDmF2gNV6i+LkIYDD7Ggh\nVqZ9kBF9/xgmtNvcz7ZwlzsEXdP3j4Hg6KCSQ/f7gSG77qfjw+e8OOb+0L3HMlmF2LL2MaXFcTzg\n69h1v9+WQ/Ci295/dnY2aq7EcYyzszMA92Oe+wxs/6XM831l6/vHzttj7w/N25fPaDs23C1J4uzM\nf/V37zx7p94/+cVfuPm9T/0msixCVZmFOMs4qJ0szpJ2PBl6SvWRJXS00Mcc/3ijU8pzEA+lSfeP\nIJvNJlhu6Dh0qM283z+a7brfx7keajP7eH19PbrNx/RRR4nt6+NisRgQ4Ow6Dus27yv3/X4veqz3\n9VHPp0NtZh91m/fd/369F93mQ3OJbQbQ4zE/NKd1m8f08Tbf7Zg2j30vH9Z3++Akt/9yCzjXRq6H\nwPgFeTJZTDLJJJPcEzlKQ06Tj948WH4CSQJQo/cN44Db3aKoxdXaHKGSzOwys9lscDwNwUjGZpHm\n0UFTY+4rV5e9L7vurlxa+6BEIWJzfb+fO+1Q33S5dx0zff8HPWaH+gbsHrO79G1Mubr8MeWGcsHt\nGrMxfdNjtqt/tymXfdpVbmhefpBj9jz0bVe5DxYZyspaAdp+4gYAKEvnxMxz827/8ptT1ulJJplk\nkudKduNcDojbXfTV4U5TVea+U5qTmw3yyOwuTTyEr4RkjMPg6dOnPQcNcNiJEjL+99te9eqfz+fi\nDNl3/2w2C0J+QuWOcWxUVTVwJO2LEFosFlLuPufaZrMZ5eTRKYoou8bVnY5cv/Y5SFj/vjKB/rsa\n46QLMQjepVzNj3BorID+qeZQuWOcssc4E30n2L45UBTFAGK3q34/mcGuyEE/qnQX90Toe3369OnO\ntup+7YtaDH2voXJ9P0bIAfyZ8wVae/qPI9OftusQRQwqMfOsrhvwE9Haddcdl87pKJPFL38svvmd\nf9Z3VOjKqZ63LY3aETbbTa8zSZLIxyLOv7ZFmxSD8vZF94yNnhkb+RM6olD00SZ0VNrXzlB/3q92\nhu57nto5xhSgj8nPop3Pan6MaWeorWMj5cbeFzIjhNoZGvex7Tw07ndtp3/f2Hb672hfRN2udlJZ\n5EKbpq5+rmsAZEGmwpllsZgxktiZLwDzzN9++/uTyWKSSSaZ5HmSo0wWUfQCsixG0wxj09u2Veq5\n2ZXKssKmNNfOl0Z77joXeRPHkdyfRua+ujRHhiLL0OUzAGGyEB4tuCtqB9Vt7qMce1/oyHnovn3Z\nuReLxSiSl2PuYzvGHN0P3UdJkmTUsfGY+3hiCmlOGtY09j7ee5f7+P6oSe86fut5A+zOa+jft9ls\nDs7DMfdpgqB9hEU8nu+bO6H7ePT3he0bex9l15x41vf55hPet91u8XBpfqMWXVU1moZasPnbdUDX\nmd8577IsVtA381vTAGnCNc58523bDU6qh2TSkCeZZJJJ7okcCXuLbx4sZ0gSp1gzYk8D3LVmsNoa\nO82iMBpGHEfKxkz7TifXWqtlx1GCxXxmr5nf1nUkZYe0uLGk0b72sYv0fKzWEyIE33ffIYeXf98u\np8zY+9iHMU6exWIxeozH3KeDSg7dt49IXY/xmPue1RiPdZDxvn1jMva+sWT5+r59Y/J+jvE+R6S+\n79CcA8wYj7nv0He97/ufZ2YtKfIMEZ10yuelIWvmWqyyzxvbcFW16GDmQ2Q15DiOxd7cNK3UTxnL\nZXGLBblAmmbScG1I5/qapkZ1L8sKVWsN3Yx0isIqvJhAIhsBmOYSesgBS9NE6r3cmHL9o8ttZCx+\n9pj79tHzHdMu4DCOcwzN4jF1TmPhyvow+niorGkshveFxiKqN4P7uL7kWSEmU+2s4wYY2WojxHKf\nVkS5SEdRav9GEplHJdWgLEy9b39vyhgyySSTTPJcyVFOvZ92HdZlhWZd4nRh8KObyjrh0hxNY3YN\n7hRZFqPdWpiN3XF0ElbuMnEcD6J7mqYZxMTXtbsWdebZeap2t87sVoeOW6FjKv+tcaShYyXvO/Y4\ntuuYNeaIvdlsRh11xx45D7WJWgLxqbtMMByrfW3SVJK3eS9+XZrCct8xeZ+J4Zg28Ui+r0275orf\npmc9V3a9F5ZD53nI4XXsXOm6Luiw8+k8dzk9x8xf/V4OvWN/rmw2G5zO+iaIrmvltE4tt7ZrVNd1\noNLMaLsocusQTa3LeSHlRVZt7rpOIvOoDSdJLOvelcWJm/arBW+ETBryJJNMMsk9kaNsyL+R/eLN\n517+FJIkwetvPOn9lqYpNhYYfb6cATDGb+40+holFHNOG4+BkPR3lzRNxYasdzDA2HcIT6HEcYRt\nuxt2cptY+33l7LN33Ybn4b7X5XNz3LWuQ31iXXe1m/6XPD73qa67vKu4MVo9NdW3LtY4O5kPng3Z\niQHj+6JGq5171xuj3SbWeMBgN/MMyxoaFi5Xa6kjV7bmzzx+AAD44//4zVE25KNMFv/Q3aCsWqRt\nJCGvyxPn0dy8acMUrzYAgHmRSiP1QkzhizCD1n8pbdvKYHARrusaWWbvkxMPUR6xHD1cOGOLPGMI\ns335xcngiLbZbAZHr0PHwX1Hr11IhbHmjjEIgEPHbj7DTBt36c8hc4KmKtx3tN83tmPNCHEcHxz7\nMf3ZZ4o49H7GzhU99rv6cwjVotuyz/TA93PIrBBqy7FmB77vUF3H9Gfs3Od9D05S23YA1plGJSyO\nHaVrVVnHXBRjW5v3nMU0Meh1iIAEe08WyzqRZG6R5jNta8rf1hXmEuHpUBmvPjYh4AQ6rDcVLq+G\nfdwnk8likkkmmeSeyK0I6jUTPiWOY9ktNP0mteXU6uJRqnYhq9rXbYKT3Dr1FA6Z2rDDOrdiWH96\nZXZ6mizquhYccGOhduiALMttGabcPM+FnAVZX5vyZQwfw6GY/ENlfBB1PC/tvG0dIY4GfY9/NL5r\nGf9/qeN5aOfZPFWcE8MDPe9bbUuJdSDUrEMHdFx4+gRBul5q/Gma4Orqutf2Bl3QMffKowcAgLrh\nehXJv5uG7XWcFn/zre9NsLdJJplkkudJjrIhv/fee2iaBlmWoiKzUULok7OlUBtt2w4nM7NLZJmF\n2WQZvmGp8LjzZFENboK13YxmWYSmoRHd7bjc/XzJ81zu4y1RpIDetqc98ujI/FZ1rUDwktkSQBgq\nlGWZ7JyH7Jeh38/PzwEAV1dXch+Fu/Vyudz7O21su9rHv6Hf72L/DvXB13qSJMFyacYvRHeo+3Co\n/bv6qAOB9r2Dffecn58HYVV+mqFQ/UmS9Ggld/VhFyfIB/EOWEfoHej27epjlmV7f9d92Pc7276r\nD5zn/P0kjzGbmfnNvHRmXemvA13XSvsEfrbeDtaGOErQ7lgvtNbOb/pytQKV4ZTwW6UcP35gbMR1\n02Cz7TPFaVY42rWjqEbXHcdlcetIPYYHuka08AcuTVPFG+qEJgi+iAfnZ/Lba28+Mc9GkJfjJHLO\ni5UxOywZXt0O6zHOwmGbaPR35oxWMNTsV55nYtJgTi9dxyHyHH7UPA7pSalDWUOOkV2EKH75obL5\nHD+KQ+Xva7tPt3hM2ymhj1HngttV9qG27yv7UNt3bQjPou37NsxDbacT8Pr6ejCfkyTZW7aeb7vC\ntk9PT4NzeUzb9823Q20viiI4l+mkSywpz2azHaAiOrQ+2ErMlgDErPnWxVrMl35MAzBEZa3L0kXg\nWUDBYjFHzHiA2oasKwUuETI0II6Jb+6kD6yvrvmMC7v+1jtTpN4kk0wyyXMlt3bqUZN0ufRa1DW1\nULMbRlE8yLmnM7HS4ffg9AR13T8+zIoc643ZiVeW5D6LE6y2dBxanGBCx0AsO6feIXmkoVZs7u1D\nYPRRp4ODcOWZgtnAOA6jfMidcRe86V34Bu7KVXAX3oH7+uzYRAP35dlDmOi7PLtP7oIlvs2807wS\nLINt9xNW6H/nedaLvDP3u4g6/r24XolTb1s7rbawkXrUkAkuWCwWUgdJzMqqHhDPt60zZ0apqWue\nzwYZs/M8U2siTbYQLp/vvvP3k4Y8ySSTTPI8yVFOvbC43auxlHRFQtB2pII1XJU0enO30o42IXGP\nHMTtZGaCUPIsxWJh6nvzqbEvlrRlN63spLQTGZicgsAAmBWzgQ0siuLBjpemSS89i61CHIF0Zq7r\nNEj2TTsobXZR5Ij5dW4y1quDK2jL2+cUur6+Hpw+dMZe/zcTmWTa7Pd1V7mUoigEnK/LpVa4r9zQ\ns4fGR5d72/HRdY4dH8ptxl33Y9f47AqU8J/ldUA7pcPjs89WPWZ8qqoaNS9DpPWhcvX45DbpRJqm\n4Bk05OtxdbU9+zDFnXj11a73bNt2Pc0YAKK2E+2XJ2WuBzpyj/biKHL+LWfLduVtt6Y/i8Jd1Nmm\nQ+0ubEDKd/H3O+/TMmnIk0wyyST3RI7SkG9ubtB1XQ9hwR2irjsUacjO5cifAQN1S5IwFEWL0U77\nGip3MsChK2oLt9uUjYQufuOpgdNEbYfrTdUrI29d+9l2A6OxWl7nbEcaymLu61Qcu7Vn1Ruc5Ayz\ntDwbVTxACOiUUFqr8dnJqqqS3/VvfoqckNedf0O/ac2EsCkNi2K55+fnA7jUarUaaFZt20qZOo2P\nrwE1TSMandasqPmFfqOGpZndKPuSogJhDztlH5xOZ4nWz/qafAhpE+r3rpRd+xANh1AgofcW+s1/\nVr+7u8xLPwu7npdpZ97n+cJxpQs9TRQNxicEYS2bRtaQBlw3koHNuixLORkzuOzsZIbF3NiQV2tT\nf55nsmbQv9WnazBC+K3Wmvm7tpPPsv0QNp4m2NU4dvC9sXLUgvyRjwBJYo74XNR0dB4/Mgfvcce2\nvuHe/tv2P4c7enCR1DhBHSlD55/gD4llzmJs7JGCceaM3AOAiwtr4qi26mUOnQSdXeBNc/tkIpuy\nQRzRYM+FW79k88Gf5C3S1H5wdp0JTXZgeKzc9Ruv63xyLNPHfe76jePJD3qxWAzwuE+fPj26PP52\neno6+OC3261AokK5DkO/sdxQPjK9wGvZZxYY85veYPRv7j23g2c0ZSpFvz//aK9/2zUf9v3mL8S7\nfvPnii7Pf7e7fvPHSs8V/n24LNDl5jvUfDP87sSU1VQ9ql0K4W5C8tN2skgncOsFTRGRXTDPzpxv\njPEPVdVite5jg9frtZhAnBPTfuddP2MRRRJqWN6KNE3FlONMK7FsBFyE9RqlfZ3HOq8nk8Ukk0wy\nyT2Ro2BvH42jm08UCZqmw3JO7dNpC9QIeDQEnJagdwruQjSSGxLo/pFCa8i830BY+oEe3LU2VYNH\n50tb7lCzdCTTrexm69LtqNx9qbXPi1Qi/tif9XqNzGoEPPF0XTzQmACn5bmAlEQCTbQWyWepVWgG\nrH2adEi7pYQ01TiOD9bh1xPShvfVwevvdx/vUsez6KN2Dt7HPobqeBZ9fPkst+1s7d8OjXXa81sp\nihTbrfkmHXQtH2jIGm7Ha1drVxe/18cPz+RayPlP0acZnp7LarszZVw/Uo/aeDMweaVpIusE7yOU\nTkuapjIu1PirqsXV2ryLH//kpxPsbZJJJpnkeZKjbMi/+PMRHsxzXK4rF2tu7S9FmknQRkFnXJwG\nbSiJZGo97NzT92vhbsXdMG9bpWmYcjWoXEOy+MxcbMwRZhZUzgzXFxeXyunn7EPCk5Emg994Gohi\nd5+TSMDxpzOXGPG6NDdqTSSkneyzM/ra1OXlpdhZqd1rXtl9tkLt3NLORY4F77+8vAxqcXxG92Hf\n6Yfl8X4tTdNIvYeCF3yonpZ93BOhgA4fcqYlpFFqSJh+ZyFbvC4H6I+dfsd+Hbpe1jG23kNzJXSC\nWF+b3xcF7bzObyRJPNsaSWwd2qAtdQgF69DKOlFaDpzVthIid9qEH+UZVmvroN44LgueaGPWpaC0\nLpVSO9Bu26ZD7KHS+L6TJFY2fvNcUaTSN3cCb6R+/o3aSvojPOtx1EuCau4bssQdkqNMFh//5Y/e\n/MHvfAJZmmMWW4cOBytK8dqbxrFAw7luEI9ZbevwytdrU8bJPBECae39FHJ7a7LI4lTKXlsDvn75\nLscbiY8S+CYOfZ825jO6hhhp7TgiAuPq6lpMGswpWFXVgEawbSuJ7mO9mvyIFKNpkvQ8sgBQdtkA\nA6rTt3P34PYxAAAgAElEQVTh0XkIffzvfD4XilGNI+WCEMK78uPtum6wIej7Haqm7rUPMB++bh//\nfljt4/1s31rlOvPzAe6qcx+GWd+v28f771P7xtx/OqOT0h21uSDrqFdBxtSlzezTl3U53Pi4mL3y\nyCy+642bE5lExFbyXW8q028Txds3d7Rt65x/8p1GAzNmg05iGPw5qMno3feYDUwhK0UiJA7BeqhI\nPnp4Km0hKX3buoxJP/zxu5PJYpJJJpnkeZKjTBbvvfcC2jbCtqxQCcuRMydQu53nhf2tn68KME61\n6+sNAICw5bbtcGU1DJZhNFCP1LprwOOFHFXs7lY1LXKB2FE7SwLG/0g0aEqW5agqQuzcbuy0mFTq\nzMUpYdq52jaAPUJlti3oooEm1GOestrBcq7bZ/uFCrGNJspsCqttVQ1oE5umGZgCNLzL12aaxjks\ntHZEDYxaVNM0cpTTEWEhnK04Xe39p6eng7Q8+qShcbn6qMk6QrwM+7ga+FuY6c+Nie/s8cfAbyvr\nStNUntVQvBCxOsvW5YbeWah91K5DkE/9zthW/c4cNHTYPr/t+v60M++sqbY4XxJHzjrdOGoeB316\nBcw3F0Xm97IeQlQZMRtFsbAzXl7bk1Hr+ruxnDWSns0Tf650cO/bxTk4rgs5AbcuUndXbj0AqG2/\n66qRkzzNVWkESVdHZ11xOsO23JgxEM6LSK65KX8c5A2YNORJJplkknsjt+KySJJYbK6UtgtHpDjm\nI2eYLwrrLLM70+liAdgotzSbAQDeUk6N9dZGhGURNiWDP+zuWqp22J3J2ZwisIvabqx5kPVf9s38\nBZjUUNuwSoHAmN8WxWzQ5/V6LRm4ydfRxZHcS2dGhxZ+6L5xgNj2WM1zXqRy0rhYOweRHwCgIYZs\nH23PWgMNBRFox09IewxdIxm9Jhr3r4XuA4aOp13XKIeCZELX2O/QNb/c0LVd/Q4R2POaHttjxyIU\npMFr2+02eN++sdBRdoAhgI+FeN1FrLlv1NMwEeZqIPti1HZYW4jbg9MTAOZ7p+POJSBuXQo3qzUm\nKqqXyYwvVys5XW+qtfTLb0OEuKclm7br4A63JtCG6ydZzutY7NSvPDJRjnW1AeK+TdycqBvbJn5n\nzoFIrX5bVo7zptHrie34jzFKjlqQu66zL9fhgd3xf+hlrKpWHVkZVtjI4PB4U5Yu9DSzR6mT2VxN\nKPNyHpyfyUBoInvA4IclepAoC9UWPdk0mYm5vwliC/lMFPf/DzgnizEF9I9Ds1khxzAegXQGidL2\nO0c8OIZFscuQq4V9myU2MnGWgiadYtEPGW3bFqm9b7F0HnR+eIuskDL5euZCU9oiteiXbKbbYSb2\n3JYXxx2aZgPAEY23bYQEpg3ntk11XSNpzbPMmxjHMVLrFC6WRDRUyCxxU37i6qUTqphbNEG1lTGo\nSemapchzO78yRo65d+FoVp3TKrF9LK0icZLPJPSdw79Y5DK/t/a+R+en4LjnkZsziW0T7zs7WaAu\nV3ZsIW3pOjNmpzOa3jLEsXlmbnG+VdUhseOTzGg2ShDba5zY81kOflelzRmXzgunLKWco/0jvykP\n9loHZwbEQDTd7dLeQMWi61qpS0wbUSyLrpNInOFMDTSbzZQJy/Y/LwbmJ/299h3FjLxzpkF+IxpM\ncH7a96MxlDpJMjR2U5SQ7BpIUyqOu00c2nTjQABu4abyVNfbYD6+fTKZLCaZZJJJ7okcpSG/+OKL\nlhSkG0Br2tZlZw3H/9tomMYdQUqbnfVqvRV4DXeZPM8UIYjdrboOtT1++ambZnEsR5/Ilvv02h09\ndRbZXLCI9m+jcupJjkAVc2/xj2VdSaSejvIJOgrspVRp5gsLlcttbP6smOGNJxcyfoCBGfkaedM4\nwhLnIOqQ5+64pO8HND7a7fT8PVGmnSgy6hZfZ5IkaFs6Igfd6iUScPU5aKFPNm7KG84HmoP6beY1\nHjkj0eQcb0SLIh2WF4AMS3t1eWwTy+O4h7gsynJIMFXXjaN0ldfutMwQ7l0LtaeuYzRbI+9DZ1d3\nDiFXnn90b5pWxoynvc1m6DANE8q7fzNL+/XaRcKdnVjTwcaeWhRsU0e28Rg/ywl/HZIBSRZ4rw1y\nKlR2O/8drMttD+sLGG2UWjDNn3EcyfdPrb1sWtGI/TK0yVXnx3MEQdSU3TxPeu+4/57TNFGYefsu\nqhplPWnIk0wyySTPpdzKqde2zjbMHX82iwYOL/1v2fm6SGql/RdJ5HbajNFxjRjJ2zV3o2ageXRC\nl9kH1ps2OfspNdpZlqOzdurL1Vba0dmdkxGIcxW8pDVgwnvmudOcuNMn1vjYtHVP85JnVSw+AGy2\nGwkw0bAl9oO25rN0LvZ5bbuuay8ySWkmfkCB+Y3alINttW1/PA17HXrlAUCa9qF15jkbABSx7amU\np2F/fIbaptZaGX11td7gzCMRj+NEtFBe4/zQbTF1w46LeYDOnLM0VTDIzvY/GsCh/PJYJk8EoVMQ\npW07Zd8cflK6XJZHjU0REqp320mizChyJx3/+9JUttppxbFwpz3W3YpNeKHaV5W0jxuH47bcDk5H\n/Ug4pz0u5UNhv7vBs03tkkfECtrGaFZqsZrzgnJunaAAkMNC7JDi6upaxsD0NZIx4Mm6alrANo/w\n2BjK7h9Y/rqu/y3pQDZqyCFYol5KXeLTSPxYP/zxTwd1heSoBTmKXkCWxdCQUFbeNJ0c08lRjC48\nOTk4XASjtpMXxoU5ViYISl3XshHUXnYAza3aCL1mqj5U89dMLFPucu6Ojw4Dan5LE+DrbxhzAlER\nZc9Ab51ceS4mDR7NutY9ExJN3ek7UrquU5hWLniROJzosOxqh7HkpsiPLU8iaWtRWKfUtlFeYOtU\nTJynWS8k4jm25E95EoHfCR1QjaYslMwqtToaumNgK8fLRNpOT7j+KC6uV737zVzrHyFNw1w/AYuR\nbfLhfTCIhM5bJOM4kr5R6rqW9yIbYTzvPcPyfORFmqZYb9a9+lfbUnK8MertLAa4ifE9luuNmN+u\nbNiw2XSI8HHHb7+85bwQ4huOU9W4RZ5kPWzTK4/OEce8Zuo/O5lLZJ04xZQ5g2ORpn1TBWUrCorS\nYCL/OJ+ahqlrVduIc3AhSkkFeixru6Gu12v55jmjkqSSb4QbeqPMjpybaeTydgrGWSOmSAak7F0s\nr2eq8qxPXdeq6D5nHqLyQNOTQWMcp/NOJotJJplkknsiRy3fP/vZP2K7rXs41/5RyddEhqYLAGgJ\nWYl5turccTLm/RF8rHOapirHWL9tZdOIs67PaeF2UNPGIXWnyXxgI3NS1x/uqnO7k2dNJWYJOtQu\nLlbiWKA5I4sTOS3MArwZjt8i7UFk2EeeEpapqbeqKrl2urCOy3wmx36OO3fj2SzHiYqgBIBFoTSg\nyGHCl0tTHs0paZrIuFfVxvQnT5HIkdw5UXhKYZvMdasBluyPI3jiaWC9KZGlGpQIzKNWZXIJQRCd\nCcg3gZRlJf9OE1Ou0/JrMbew3KYpMc9ntjznzOXpKBT5Jxlq6lrefZqeSNucxunqoiZJU4AxbZh3\ntBAyq0bKe5Sf2jqcaW5uaW4XkXsv58nC9qMZcDVcrbcyFsQGb+272JZVX2u1ffVJcDRvhRY/M3xZ\nluCh8cJG4GVZLOXN53SyRfI+YrvkFEUhJ2nHZaEI6sVEF8ORCjnzjIOlOdOXzAuhw21R26hY8k/E\nuZu/7rvxYXp6LLrB7/oU5hJmxPItE544yxKX83OkTBryJJNMMsk9kaM05PYfOqy2VW93oczzwtkS\nqflGGmDtHE4+LAcAahs8wACcOqpFY5F76lp2f+5SOuVLkjAogOBvt7uWjdESkqYeZLZt4CB7NBhp\n+zV38q7rRMOgTTXLU7FJMx+YoUU0/V4rKJGvndR1K1qjHh+OLe3fZamhYc55wr5x0HLRiiMFIeuf\nBgAoB2qH2awP62rbTk4fLC9CrBxEznHiNH2nFTvt1sHKOAfWNlgmguNFoIaR58MAAA3hEm6BNJVn\nGXyjx6iOXXmAtVXm8979cRzJiYiO37KsUFVbe81GgW7qgXZUtY2Cp2mHX2Tb5+aND/8yzsz+d1NX\nDdqC1+hbaAYQLi3OOTlk0jP1cBzt6dBqh11bDfqjn5d0RGgloo/EZpuqdlqrfebkZDaw8cdxJKT1\nAL9zFwyWZI7Xg76XfY7TrmvFVu9SQ0XqREdHeulO7XQsR7HMYZ68mQ6qSLMgPaavjSeJc5ALxHZW\nCMyytevP1brEwprRafM2p91h1Ok+OWpB/sf3zAtK0fVeANCn3OMHrz9GHvmMycCblMoTL/zKekHc\no/VzEQaGZaRq4ZUcWOgGnY5jh2sUQ79yXNBfqPsj5Qaj6hrlYDNtmM0K5Jk9Xiqv6MXFytZr26w+\nEN6W5ynSwBjEggyhM8q1fbEwk2K9doucENXY59M0VQ4ah2zwy8vzWEwMNGPEcSTmC+Y108d0Z6Zw\nOO3aTugidTzZdFZuNi3yvPDKS8VxyY3Y4EL7H1Io991mY/6eLhZyzSkCw41de85LFY7vL3ini8Vg\nYwUqmV8sryzdAuHK63oOMcAQ1wzL69RROLH92SqkC+9zNJRdb0iG47NL4jiW74VmhzyJEM/M72cL\nU5empnT4+E6cijrSlZScspB1jThg6UR+OM9lMdfOUSpLiZ2PHZyTm1GTvkLl97OI6eBt4TY56/S0\nG8OicCYOnfHH4ZDNfdva8TF3kR33zVY5gF35RFsx8vBYcwUwmSwmmWSSSe6NHKUhf+RFQ5kZx5EQ\nTTtJcL0yseHzmXE65Kjx50+MBkgIjo6yCQWxMLIlT2uB3ziazhYru/tQ46ZmnieJHEMpGgon5pQe\nXSgLBmzgGBKlXXBnJGLGaJT9Y/WuiCxirLlLFk2LtUTwbG25qUT+lWtqhbE4Agl1fvPSaVt08lRV\nIxoIHSbltXWm5M6ksi4dvluTbZtCapydLOx95jeDQ7aad+3wphtLKMMyzpczgRQRG1x2HZYFj6RW\nS1CkKxSDk6565S2KHCtL0ahzxzm4IjWXWsqTXIuxcxAyAwt5My5XK4HWEa5mskVwbrgovmF5ncDz\nqAFerla9iDHT3lRxuqhjv8DHnPbsl/f0aiVwQEeElarTTyXl5bZdes6VTb88AFjbZ5j2Umtzbl6a\ndp4vc7Rbc/FU4ZAFSmnNPIuZg3iJca9uxDzp8udlgqN3+ScTdORHiR0kTJsFKPxmabIxeHuWndtr\nOvKPYzvUmruulXK2Hky2aRqnwcrcbx3PjH23TefmN0+9my7CzH6cOWGMaPHgbCl9A8x3eXFp1sRv\nfudHg/aFZNKQJ5lkkknuiRylIf/ciy8gTyJEaSz8CdRgkqRCZlnEVlYb3sSRwHu4kxnSeAsLYWqZ\nspKdkbaz05OlED7Hdgev2kZ2eJ2h1vymNFVb1/WmwsLu5yczC+9qGhQeP0EU6agmB+0RykC1+Q4h\nMA6GRUC85vmgFqV3eufCGdqY2ha4svwBJyczAH0KxChyjjyJZLQ2+XlhHRjdMGJssZgjte9Ma/m0\n0XJXPz13jFssL44d4f7nlw5ytXRxE6ad6AY8D/N5IRC4RNI7RQJP+/xnTB+3ZSn8IZxTUeTY6Gg3\njRQ8rpDggFjG/AuPF1IeAMRlLA4+B/tLFBTO2Wh1eea+COczOmAhvyX96YM8S7EtGcyi24leeWma\niMOQEaHLeSHvUUMBSWfL35I2UdGF5tq63BrqWrhAqTiOJHqOc9pFn6bi4Is72oGjntMRgEAcAacp\nbsoGS0tbp/NF+vb5pnWJECT/ZIyBJq3pM+kHStNE2qC/b35D2k8ljJH2+86SGK0dcBLOm/toz+37\nvGazAkt7OuwBDiy44GJLG36KJ5cmKnBmHZJFkaDyHFuPzhey7klw1DYW5/pYOWpBfuEjLyBKY3R1\ni0YIXmAbninmf3fc8BcG48CzUU8SSeRwjwxT3GxL8XhyUQfcUUZnIAGMIT1EYnNmF5DX3jT8ymkU\nzuJB84kjA6kEY0lzwsk8E4elbpOEBDOXX5zAJhkJJlfVIkgFhVrhUc+/R4tOOqnTsgPGIdIoZwNg\nEBAOJ+kQGPwYOOmrqhb+XJZnIiTN7yRDOpnNe2HcgHk3RAAQvVBVtYSy8mOrEzcmT6+tmSufycLh\nnC3OqUfRyAIeIU/mLXK7EK43JFpyY0ePuFtoXXlV5dpycc0IOPP/LIux2fSxtzoEXa4pLnCW1zTR\n4JhsyuPH7667DYh1dbLJOhRDpLD0fI8OC05Jkkg5zXn852LZDiIzNakS/+ooQ9lMekRhfQQP0DdZ\nNIpKlnX5U7jrOlSs15qZLq7deqFpdX0u41cenePJUzsP55xnKjuINW3MZoW8j1xlgTH318Pvqmmw\nkUvOzMS2SELlOCKABFtb7sXFCsvlie2vK3KfQzUkk8likkkmmeSeyFEa8s0/3qCrW8sNwF3SOW+o\neYTyn/lRPlo0bMnBUlo0zbB5GlYEaMKaWiLmtlu3863Wzmnk6uuX27adkIDkVns7nWdINPMLgKvr\nrcDY6Ixq2w4La6pg9pI0amDRZkJOvq0rcS4FcdgS2QfUHkFOmsYOS2pxqetyOyDVF2hfrOk3pdcD\nSA8A/J9/c2b/Zcqa52t8650vcDQAACezfwuHKaXWXAGKbAUwzqDCQtcubVaLeZ4pnhBIH+q6DxNL\nUwhZks7DxyOfEIsrHPupvW27rUVDpkhUYFnjxOKQHXa9QUWzDc1SkYs43FSMICsG+fCqtkYqc5+m\niFTGlrSabdfJ+6amrMtz76J2NJSNM+n4mm+r8MixcCY4Ih/JYBEncsKhE4zH/0WRDzhWGrSaFsg8\nl7iMHMzs0dSdOBipZS9m6QDa1ZSlI06iMzFucL4kFtyUtzxx3BdEAs4b12fm3TSRf7D9se9nu1XR\nfWqM7H3OVFPJKa5UjjuOV6HKAyy5fo+My8zZNOvT9a6rRpyzMnZxNIA06uzYY2XSkCeZZJJJ7okc\npSFrCZNe7/5NA/H9oBLNBUDR6Z80zMjRMfa7wF2sV2/TDIII2q5BGtPuZLWuJB3soNsGSDy6TP27\nwIdgYD2AAZtTHp+Zf19shvZVPxhEy2bjIG7s4vW2EocCRbOk0Q48V04cIaGXYe0EUM+xe/t7fw7g\nKwCA/+f/MPavL7+xxv/9v20AAP/dH78OAPjmd87we58yNjv3zvoOS6Dv7JFoTSXUbNfrUpjs3Gkl\nAaMkdWQUTz2OgUxDpBiF5caCdlY6ARPEMu4kWy+KVMrjWPSnnwoK6vonu6SNB4EPm42zR0rwRgRU\n6M+fJ9e1pG7SuR59LbxpnDNPHESK+J227lmWqNOW0qA9G2+eDL9RmXtaxVTOsNjzQcyyRDTUB+dm\nrqzWJdLI45tJYtGGGRBT1w1Wq7LXtvVGB0DptptxoR8lz1Ks7SmXEYBZmou27k5d7gUSFpdluZxE\nJGGEvSdJEgnQckE9/XGktPSlRDZit3Vp1lZ2DTHvq+9XM+0aFLdXbrUghzhkNf0c0QbaAUJp21Z+\nj0GvtjaBOOrFreTz0mVw0rojBWAGLRanVf9eQL2wzoXNcoFqmtodcxga3LjQbectj2Thl6PQ2uUD\nlH5FCa5KL+JHeYh5lMzQSpYGZkzRtKPs91kUiRODnvjFoxkSe8S7tBO/EArEGAuL035V0Yk6ZAxn\nzJsgqeHrG/PsZlMiKtfe+C1xPlvbcoi7zAQFw8gkAJhbNAv5as9OT1Vkohm7lx/NZKE5KwjVaLE4\nMc++9tTUtakanJ3Obd/MXeV2i1hCz4fOpblFMeT247hqW1xZvDSP2suTBQqb4WK1duHS9L+S3hJz\nYDEncsht2MyNyPI+/5lHYqpgecuTQkiAvvaf3gIAfPGz57L4MP9h1zWDyK7PvvwQlSWborPsZOEg\nLXQsv/zoTK5d2j7WdS2LGdvOSDizaNj5Y8ew6zrBDXNjyxRRTz/0ns540/YocuYjcZQ3nYx3iFxM\noyy4sfXz15l/M6qzWZcyLlQ4dLwBF98O7vvXizM3Y1HM7DfaNM0ga0ySAF1H8wQX10jMNtzE88Lh\n2CU029Rmaz3OTKFlMllMMskkk9wTOUpD/of3blA2HbK4k81AmydcXP3uYvURzbHzq+i4jrtWFDSH\n+KRGcl2lFdeZcgcSuV1Ydu2mQW4x1Ncbd8z04UOAwzhqohMR9jty40LoTxKnaqxsXwN4zlBmYCB2\nDks6VrYhM4/Twn1MpCZL+s4PFvZfa/z2J8zY/8XrT2z9Hb76xGi3f/T7po9/9XdP8b/+jXlnf/i7\n1IYrOVWQYHy1WgvsjONTVm0vLxtAkh1TzkadoLbWcUdtsFmtcKXyIpoxiVGvyt41Q4mZ236b+1e2\n3AYdXrYOpG/YsrZl1cvQYsapE2eYy7sWS3kCLUTnHFRvXdvySgXDtLjmbYMo6pe33lTirCMJUVmW\nWC5nAJwpa1uWCgKYSnmUucrruBG8LDGyqUSjsu2OC8I57vR8o0a9tpp6Aj0H7TfS9fPHyZiQjJ5O\naf19dv0y9L+TOBk4//V9dOSZtg+jYX2yqyRxwAANLxW8cmBN8s0TkTILaWgk+Wp0sgtxuEfsajT4\nHju0A5rgQzJpyJNMMskk90RubUPuPDtJ22LgNNolw51ROagiR6upgwtMudFOI7nZxZz92dTjCK87\n+1vbdMqIb3dDOLuytKWLepmlAaPZ5R4ULkmigb2tbVvRjGnOalAj8uzp/v+BvqbfEHoTAzxEdAFI\nISUE2KfSUdeN4gAQ1wYq++/Thang9X/7FTz+yp/YZ3j/lwC8bv/tNDf3LpwTTqBldn4YLouQrbAv\naZqgbWnXtJAidZ/WflgeHTarbYVU0XOyPACYI8Ol1fxYXtM06rSiAgCoWTG6qyzlZMLglnmeCRue\n5ptweRWpPXYDDg+TYTru9dEErtg56vFXAC4tWNvGrjw6lFUwBN9FVbUDWCUDNeK2dcyFATtniLBd\n4KiNyzPZI4/v6HB3PhpfOrTw0V9t28mpzc81CaD3m895EfVYHFlHpwKVOEdzF+kYue8fALIikbVB\np4GiCF1okkgdvNa2LkpWiO8Dp/E0yYIQ4H1y1IL84gsvwHDJtqg4Ae3LKdI4uCD4anye51hbZ1Cm\nqCsdTtktfknSd0KlqTv2b8Qj7I7DYpy33ara1tH5sW1Rq16E9fiWleQEIx2k9vDrEFhn5jD3VVWL\nBDzKuQ+BE1DwoZlzBDTt8GPQ4aapcMbKr4MX1Sc6gtTLslybneO0lNxhbuwcCc/GXFlfiYnh1YfG\nm/7N7zjnH8NXDa0lFzhl0qkZrejGwpkHOD8aaZeLmozVRmoX86ZDhv4xVEc+cswe/+SniB8ZM0wt\nCA1nQui8XHm5Ko9mEj0X+ezyZCH4Z0oUOZxv77rdXAVzHKdi8nJYa5f1g308OZmJUy/0UYc2bWLS\n67hR35wNj57n7jgt2GT3LkJ8yO6I7xAlrh/WyRV1gvV287IdfHOIOiEhcqHYCVoMN+EmQE/JOSzf\naOIc/tqRT1MTseARlGlI+NCbQTYdIfuCW4hde9xGHUpy6hbpyEUU2rHbVDWQmXdQWMd71w3H+5BM\nJotJJplkknsiL9zc3Iy++ed/7iM3v/HxjwLox5pTxMCtskn7FJtp5H53GYqjAS1glrmsEjTwN+gU\nkbz5S7hYmqaivWl6z9RTOvwMxJRQ5oPcg8KlkXJW2V1Y98/hcp1ZRB+79bjwWdanM1vz3yy7bTss\nmMVCogGHfXJQnEiuUXP581cf4L/5YwNv+q//qfk7m+W4snCuN774yNTZtpjZPj7+k9cAAI9OZ3hy\nbe7jyeTLj5f4xtXWtnloYgiNM9ui32OUuqMfTySso1E47dDY8vcv/vhdyUj85id+udeWokhlXHRU\nmWRlUW3y50BRJHIk5Xw7XWSi0YbevS6PkZSE0c2yZHBinM/nAjvbNZf0//W1WebyL3I+zGaFaOGh\ndvI+ycpSNYOs6qHvEeibAgFDVlSVfYdb3Q0d71HbDWgtO5UUQn8XLps0MfX9E8quOhLlVNNzn+RD\neq3hOAivRevmm79epGnqtGALH4zjWEyL7L9eawTzrMr63o/e/e7Nzc3poDOeTBryJJNMMsk9kaM0\n5F//lV+6+df/7W8J2BoAYmtAvyqdUT1SNIskNidoezYrBulpjJ3ROk1mhVyTXbJyIHrudNQ6Hj88\nG95fOjuwY3dilFYhwP5TC18yoP++o6/Pp2CuLeY5ykoD5fsOKhfQ0aGqOvk3xWWYdv13ed5iKUPn\nigMce55piws4SZKu9/tm49jSfAhQWZb41jsMJDAa2T//pOMfIfH9m09X+NIjYzsmTem/ee0CP/zx\n0j5r/v5Xv/51PH7gAhMAA7XzHVkmQ3Dft5AkifSDc+VytRayfNoqr65XA43p4nrl0oF9+x0AwAM4\nYNTV738SAIQ8/8HpiTxLAvxUpZC6uF7Jfb5zSVNt6vIoep7x5MTyzpfDfHNJkkDnwzNtcVqzsOHV\ntTi0U8Ly6nIwf/I8HtipDQ8GR6MP29TKOd/TWxdrnCtKVVOGiwYMBSwNU065sYjjSPw2tLWbHHTk\n86BjztnTGYC1Xq/lmyQtRKkghZSua5Flfef6xfUKj85Pe+MzKxL5XnP1DZmxaMXfoPMWcn2Z25yY\nXeLqSdWacGXfMyk8dRCaWyNc8MnffOt7k4Y8ySSTTPI8yVEoi//3P7c9rUVL31trpKpKsXk+Ol/Y\na7XiFtAec4ZlbgBQ43XwGnMPlMbUty0ZPgwHJQL6IZSJ5/kF3C7ctp3arRkoUIlNk57ci8u1aKPO\nrrRVYaFO83Uwulj+EqLkkkRGA012s9mKVzcRD3stDHCEVXVdK4TuhPu5/jotleTaRtPpa7lffeUK\nT+xJ40/euAIAlHWBPDX//szZDIAJD/3hjx/ZZ5/KmAkDm9Vw2rYFAQg6oaifTDIkSRINAw/QIhX4\nXBJ6EVEAACAASURBVN9W6noBLP5QBdn8ndGar6wtua4rCcLguKboBrbJMPRq2N66rhRapI/W0WJO\nBv3PK447eVbPfcf5TC03lrmkT1BEUrgTR4LEhmyTx9to4e7kaWu2fzX6KRQo4foTSiwsCRYCSUYd\nKsEx7bl57k5Jzpbr+usYDGtk2/43HMcJmqYc1OuCOlw/3Ds05V5elaJJr4Tw3mnb/nrVdTFW9kSf\nJEZTzxLXlo2HBAOcpqxPUy7t2PEoi6MW5BdeCGGMd0/KKHJRNi4CJhrAXcyzHCh39NKkH4AZMD+6\nRsfac4B1Jl6fjD5NU5XKvundb+5zMe+ctIklUKngTAa6XDdPOOn6JgfXb+tssZCiRZKoxdyU16BV\nC7ERQ0PaP7ZFkYOJufGRXsh9dKKYNj61V83R/bXLGK9dbAAAX7KHqfmJW5ieWIfflx6c4Ds/IA7Z\nHdt4hNYfsj8PtPlEWte2suhK6vmmQ5z3FzoD1+LRXk08a6rg55SlHwMPpNd4V8rzRUPbWD8UQT2F\nmOK6dh++PjY7on+38Ycgaz4mG4iFcEePicsCo4/Qpt9UCrTph1LXDTr5RvoKBeDwuFRG+vh0hXcn\nqZHgleNeHkv3jOO/MPXXskj2N6f+WNRNNYCBmu/VUeeadmrHpY+Zd+PYX5jthl0PI1ejKLReuI3L\nwUaHRFguithB+zRm3SeEMvWxP/0IyWNkMllMMskkk9wTOUpDvrkxWljb6gghHkejXi45wJI7Ez6z\nh8qSz5trwyMFMxTHcTTYzbSDYXhEjIP/ThJfA20H18wzVhNoGGjSIIsyeYblDiLwokQCTLiZhwJ2\n4rgbRPTNskQcgqRyLMtapd7R2kQ/Ao7wnRSJXFsuzJHt1bMFytpoyKUlh3/two3/W2//FADwZ/9z\nji98zWjGPF5+5fVL/P6njJPpy4/M33V5ij+33At+uh/z+0bKcJq8hZOlGXwnKuCI1M8ttwPgAjcY\n5LDaVig++XHzbPT3prwsQWJZ467+8tvQcrUu8eA0865tMS+G2gvzMmYqf59/Mrlalzg76T9bNg0K\nryx9XI0ksAFS3kZp5o4XhYFKDQox0znTD78HfmeGKa7tlWfI8vuapM5FRyieHnetVet26H/3tU86\nZONeDkqgn39Sp1tjxKFjguxA7ZcpprZ1pbRlN8aarwIAWjRAy1OKubbI3Lev+78rMYY51fCEZ7+b\nWEfxunb6Tsyu6wZrXdu2Es3p1q1azGVjZdKQJ5lkkknuiRylIb+H94yjJU7A3Y2actM2SMQJ54ol\nSLrwNGotbduitXYpP+wSUGDuNB1AYJx9xz1DzUDbhmlfDcXpa+1YZ2QO2SFpP9Oag+/cSeMO7R77\nETXPt6q1XJNgEGXSzDKj9Z2dzjD3QngNG56FvdlQTTqA9Di1Vjv74mtP8cBC2z5jOYa3dYNTC696\n9GdvAAAW6TlefWCdehaCdLWtsLVj+8XXnkofmPLIZUt20Kw0Nc+2eSfvj1zJ603ltMGNGYPlPBcN\ng+VlqUvdxHlzvoxw9q0fAgBe/h+M4y758lNga7T14g//hWnn3xpb8vpffko4nNNkZsfVZQPW4cDn\nc+MmbBW00IW+V7b+ubwjav6z2RK0/1K7bxo1H1JnIyaH8+nCXFueFLhe0WnV2jKWcioUvu82VbAz\np/Vx7gr0sdriZGahY/RVFDNQsowww41cY2CKDgwhOxqZE41T2AZl2dNuhRRLm5RB7LvQ35Jrp59g\n1syJfnlN0yGd9e3duo+aQsF3FNcdJA2bz5Xu6tNtcqdIZxsP807sS07cT0zAdU8FSB3JUH/Ugkxp\n2kYGnote3bWIPGq/OI57x0/ADLqfe6q/SLoFz6Vtd44A/0ih+SPc5IU8F4r4YWRZ0SMp4aDXtl8u\nQwG5A5qmk2O1jqSSCB21mnJyP7Z8ECGzSJ5n8tGQ3yKC4xEgjvNytcbMLhzu43YoC5Kjs86maQc4\n33d+cIZ3fmDG4PWLtwAwGtGgMGa5Kfe1i2t8/Yn5QBmdVzbOsfHjnxBKmaCqLm2b7H1VDbJ8uneX\ngK+XRO2LIhOcbUi4CDdNK++SCIPTd34kbp7ZmYJ1ZjPzJ/2YaZN17q3XpWCsS4kqi1DbOhbWpFNV\ntctkfDKz9bu5Sk6H9bp0Dh3mVVMON4fMaPHWhRnbh2dzW94Qw/vWkyt5p33kDuQZwCwuJSNRhYg9\nEkw9zR5t2w3Mg53NXJLEqdCj0nGcZR0WdgGftURROIwweU00lpgZQ77x9EqyeVB0FB0Vj+W8EHNa\nEZsJEsUYKFdA/xuiCB+NfN9DbLsW/Z35CCj3nDNFhNpB8xVqvca4xZzrniCC2kb4W7SE2rdPJpPF\nJJNMMsk9kaM05I+88CLSyDixuEOIZtm4nY27dRS1Ek9fKCeKzq+3T3zqOuOw6N/DXc6QlJOWUcPY\n+Nf842q9UfwR1oyCoeZrHjJ/Xj43R9m4qfHGheWBkHRJrkG9/liNnDAn4wDpmzbqusLW0wgSOB6G\n2Dr18iQWZ4k7LVRIU3e01rLdboU0nhqUO9AB5/b2+ifv4tTC2L5unUIrhY99YJ2Zl0WK/NpoR3+l\nYG/khsjta+raoUkqjjvBcTMvYBS7qKcN29I1cDSIsP1yTHZCbA4g+m125N+Zv9kcsOafxRf+wvRj\n+6/NLW//CKssl/IAo0Fx7Dj8PYek1frSNBUo2HZLPP1SNEVOsyR1eHKaJLoukig3lpfnuZS3sfNt\nOXcshW5+ROoE6DQ6aviddewWad6b16Zvbv46aJZckpRhS8ulGdW1cqRZ2FZTibbHU5fG219em7FO\nEGFho/yYomwWJ85kaNNk5XmM1VOraS9NuU+vnLlORzSyLXRiZ5kzHZJbpkGHyn5XueeQ3CW+Uy9N\nU4X/hu1/hKu1uVZYU0yWxcF1yqV/I6Od1o6d03Nie5tkkkkmeU7lVrA3ky5l+LuLkOMO4eA4MwH9\nu/tpK43jqAfa5zXu9uSt2NZVj1EMcBpy2bSA1VyoubWtcyjRAXWSpSitFZLm2KvrrUQSSlSO4pNg\nGiEDu+vbrrRm3ot06jgmBKI7OzZ3/0WRwdukkebpwI6W56lwCzw8ozM1BoM0fPC74UpmkAGvOk3i\nxDqA1lWFyx//2Fz85Z+HL/wtX34Srz4+BwD81V8PNe59/NdN00JzlgDAPMtV4lPb5iQTLU/D6ATW\n9PZ3AQCz3wZe/eIfmYfmZ64A6wClPH75dwEA1dvfxpXndI3jTvgGdBCNJPkk+Xingknsgejq6noA\nU8tzB1V0/Xflan4PgdQl5h1fXG9FW6XkeSInq1KI92OxhW+VjXg2m/XuK5tWAmxCTirWJQEVTYeU\n323AMUfZVLW8FzKcpWnqHHIM7Ushzn0I05r7ZtmvIs1EI88yZztnG5j1ukgzxLllo1POP/pUmLwi\nSSLH4Ww16bpqxFdBjmQH+3MZ6blGbLe1g5CSuzx1EDcNud1PPN8P1jlGbhWpp1EUxIm2raMv5ISt\nVFpxOogWRY7WThRNiccJ2igTAKkMOYiL+Uw+gourS3vNLKRp0ick8YVtu+o6kGxd3/XGW8ahw2O1\nJsNneTpbSI/AuvUdBq1MGD67XJ5Ysm1AZ4fnMw1cXUJJyePtxk2Uwi7+UbdFbs1ASWIm3WzW/9gA\nd5QFSvwRvmeqbww64XyxwJ8ww7SanJyopQ0/LqIET58yyi+T8j778kNTx7V5FydFgtwS/zBH3+c/\n80DaUuTm/m1ZwmUDNm09Xc5QWLIkhqgu5oUsPlyy8hTA2RexU7gwP/iyGZt//i+wtMiCVmFb/RDm\n5UkhufyuVwaxkaYRZhbfTMfptqyQ2jmaqPmgN3fTnxPp46U9nicJcLKgk9f2sdogsd9Tkrv2Fcu8\nV56Z+1R0TGVPLq/x8tK++9Olbd9a0Cxc8OhMzROXNd3llXTfcmghlv5Fw812ta165EMsQyOV2G+N\n6zV/W2mfVn58VIKO9OQ7W+Z5AMHizDWlogSlQ+5UMMQ2THtdylqjKU79RbquG2xs5C1/y7JY1qRY\nIcvYdm0+agKRoPtkMllMMskkk9wTOUpD/s8/63C9qRDHtRjYuak2TSfXVlbDAFyk3oNzd7wk0Ucc\n8+ieK9IgwslaFZFkd9WuQ2nT01RN37SgMw+zW3XtiG0oTVP3ooBMnU4zFkdVHknnhL5xW0ofe6RB\n6JsxdAqcwrapaVppM4+wcRyLOUYLKUV5HKtQi8bNo/51WeNq1dfcWb+f9w8APvXJFf7qHXOM/x9x\n7fqeWq3aRtZpIW65ajvB5gJGY/z0b18IfpYnnShKEdl/k6byL15/Iv/mkdPgSE1pDZwWKal37Ptb\nrStAiJaMnJ/8et9UsUss4dFy9jGs/tKcDFa/a6g58zwWLYrwptU6QtMYTZZdTdMCTy8I7eP8jLGm\nSclqTrNZIXjq2r4DnYnaQa7yQXnmdnuyjNz8ee3NJwDcXDFmwr72liQRLq2TLs855ztEER3Jlv7S\nzu08z+WbenplvtHHD0/whs2erQnoOd8oy0WOjc39J5DFHrGWhczF+tvgt+xOykwvRfOHfrZsWvm+\ndN5LHyqosf8aXECHskQeRjp3IZc683exmA/oP5umQjQz5S14wkWK2sIhC2v2vN7UWFnCo32JOtA6\nSOwPMU4mDXmSSSaZ5J7IURryx+Kfx9nJJ5DGreyW3LUenmZYrRkpRvhZ42BpapcUJ8dW/582VGen\n0pqA+a3ExfWmV0ZfAx7aqSR5qNpJfVtYHMe9NE3sFzU6pmtpq0ZUtT7DGYfRaQtryzGQJabf9bYW\nh+RyaTTGbekcOtSU8yTqEakDxvFCZrHUEmafKz6FELSG7Xvl0QO59t13TP3/8Sfm/fwvVSXOzDee\nOE1dxj032nAC4E/fpi3MaAYPF84RtbGRYNreR0Lwxw/OnL1PQZR47eVz0595VOO6KHq/tW2L9K/f\nBgAUf2ie65afHfQVq7ect3j5svlrtejFconZyvBb1Eorm1n7+2K+e+xmcYNlZtq0UdwJbea0Zd5/\nerLsPQsAMSP5CtXHbDF41h8TwL038VV0HXJqutYJkZ+fDwMPVB/l0vkMgKagdO94s6klPVgmyTmd\nuvfAnrYuNw6mVitoJE+CPNUksfu+JHBG2abpXAeSgWMsSaKBXbdLXUAVfU+RWrZ00Ihrtvl9va0H\np1enZXcSGcqgI8PTY0ro7GmlqjZ49PBBr4z5zARBmfrt6bBrhbL01EIB16uVXPvej36AMXLUgvzz\nL/0cZkWOqi6F85THo5V7XyoSz+XIcpFbznBOWa/LQepyk0mB97novEen5gMhXrYf6twPnTbX7csh\nbhE6Yy1ffj3IpaWjCflS8yRSCAmaMSrkcuSyC+68wCsLhq/a41bhaD+54OqMJpSy6STa68HctHMx\nfyi/82PMshiX1ZDghGMxs+agUn0M/9PnzP2zK1Pum9/7AR6tMq8Mh0yZ2Zx57TLGv/+kKac85VHO\nySyupC5BHth2lnE2CC3XwvvWXQb/5ziOxVQhEXgnDwZl/Luv/YVENX71awv7gHXuZTPMc7Mgb+zi\n3n7u01IvQ55XjcqyojYOLsR6PEMUo24sLH42iXFZ9RfL6x3j40uOCqV1nrI800bzlzpLmrSqXbtJ\nbKSv1uEKAOcz047Xnq4H/YmiSHiBGaFYpOo9ps4kobOmAGEiIy0uk84QgZAnCdalQ00AALohuVCI\nj7lp3MIq31zkTCSMRNXiK2vrshSERmvfe9s1uLg0VAKMpdBYe5rrsszxIXPja9rjMMjAZLKYZJJJ\nJrk3cpSG/NOf/czmyHORehoC505PTjPgTufuTwZ8FEjckV1oI9MUNEEQWpcnMS43ll+CUBV7T2g3\nTpJYsIOZwhAKcbii6euEaMQ5BBl/r7NEn1s4UiLYxEKeyVTGEk30QxGOBtvm+WyGhdValicBYLfK\n0yX4aPQj3LRoLY4aE/MRaq3vqXW6Lj73aWzstcetuf+1N59Krjy6ZmeXJS7fMaTwxelsZ5uAoca2\nC6O8L4KJbY6/+R0QXVwQrpW5+rEycK6nF5eu7Mr2iBry/AGy1BAnFZbf4rptVZuNLJIGZdJvM7Vj\n/e84Vtqt0krZ5lZFIO4zJekxo4jzuIkFRqfHkZaxOHFae2icfaEWrjVv/l0s+nOD7SDhUGXnYAZH\nYMTfdOZoH5sNAIu5JUNqGue8Jv1m10iezZAIsVei4XPDCN9QlhydGZ4GnMbaMyqrwZOACXCn4SyJ\nXVZ3G8XbNu7EyDZFUdcjsDe/xQqvPDSZjpVJQ55kkkkmuSdyHJfFR15AksRoGmdHcXb5VgIZdLoW\nx0vAXa0ZsL0Bjn5SGqY0bzJTNW2HueWQ2Gw2ACC55lZNKVAanWna5QEb9metAeQRe2HkZFFgI/no\nTLk5gNras7aKPYqRRuRsKNtKnBwk5l5ks0HmW1OmswkDwGUViQOUWtQsrnpaDrDfZqhlKbc1WFl2\nuMxG3cVvXQnLXPyy0YpfffxwUMflO++g+NynB/X79tAkinBZ0wbnNHNfSzCaWryzHxwnF4uopDgR\n3opX/81XzP3XW7xpq3jwmVcBAE9e/7q5cPZ5FK+b+xKrIWvN0kmDeWTe1WXnIFe+9sh+Auhx14fe\nrfymIoH4HiXSs2mkDt7XRsngHegxDM2Bvq2578jK7bObNhnY6Xe1+1XLUqjrcP2w82JVCTxOYGho\nJWqP9uRFMRPntc6TqTlnALL7DXkjGO07t8Oe56nYbrWwvKZx2usje6JjYJqj8k16eQ0BE9nH3ITU\n6H34n6lnyFiZpo4hrxEa0+PZ3o5akP/hH/7RRcFwTtLBkCZC1KKP6c5LzJU5cb9vnUNgPudxyBK3\ndIGOdBEkMSLPb16qdQC4th5hna2CR6uy6fDqY7P4LE/MJNmWVS8xIWCOMUIjqDJJ+LnBALcpMSX4\nEv0NxfRnONEAoIrMYr9iOGoSw58CvaNxYAFjXWU3/E3fv0j6H3L7cImOH5JyvPl1cDHW7dUsT3J/\nB2RpHzsODCflBsOFIdSfBtpoo8R69ulQ+sL5T7Ep/ykAx9+Ly6+bv4uHkk2kwY/77YU3dsQfcw1W\ni7FbBGOsu/4iBOw3wciYATJuspmp5yrJ1dfJhhWqIzQHeC2KInANCUW9UXyMv19XyNnIOhgJuJhH\ngzouLq8kIrVViyYVI42R56LG8O+2beR3RkZmSSyJc4nLX5eOhpemyqapg84+C3ZSG2C/bi15niPN\nrDJkTRebqlabDiP1IlEEY8mj16jpwjVpUMVBmUwWk0wyyST3RI7ksngRURQjSTqbCdlhdIEhttJg\n9CxERnb/KKjGszzmgNNpyOlcSxCJFu4fWQwExxxLXn54aspohhwDORgB5jLbZlmusIgs1w1NZxuz\njZ0p4sxyB5wkrTgMNBQsJNRKZuoIu7FaB4+tURRJVFwcsLOcpqYtV00i2gn/UouronxAUD9DxcNE\nT0LZcyk8wut2SsbhpsHCOj56GqD3bNN1A22rrCo5BWitTT8DAOXnPo36L78JANgSbggAF18D4ObZ\novgY4rofHVZbd04KoNnaKDrVBj57YvvQoBtonvqUw9NHFEXu/SVubPzx1uOp+6jfH9DXULPORj6q\n98f3Y97fEOLm1xcaT4puZ2U/pEXSyvvTddExxejSkAMzZI767NlMfQ/21LvZiiObWqQmndJkWpKw\nQYiJWjEJzufOEffmUxPxuDw3p9xvPL1CwlgGL6YAgHB4xOojcLkozf1Floo2nlo4r47Ac3k/I1nP\nwpDO2JZbDjLIH5JJQ55kkkkmuSdylIZM6TptA+wHb/QKj9IBPMTPGg0oezAcNWUURShrD+KWRGLg\nJ98Dtcjr1UpsSCtLCF6WlSNqV9R8/J3OOs15QSdX23Z4/OCB7e8QxiJA+C4VJ5winhponHEci1bS\n9bg0+qxncRwHHUn83fJ8I447Va/l4aBtOKCdXbdhsuyQZkXp20r796VJgpIqYtfvAwCsA+lsKLsc\nSayPossT2NvqLXz5LUvpajkVFssZEguHJDfHl2zuv6+fXYh2zdIr1Y+1AvmHIjh96boO120/SrRt\nGxVk0A2e1e/W14z1nHLQOg3rMv9eQ0eYKiih5KIczpnrxo9mdeW6OZPIfBEC+iZS5fU1QcC9v9BJ\nd92lqr/m2nxWYO6l7GrbVgIohGI0S8T+nEYMwmjlu+Z9SRLhxEYX8uQ7zwuhwdUny7cuL21brHar\nvjefO+Xp1WaYQb1pxS9xlvLdtUpbtmOSZ2o8GJgSB+3a++SoBfnm5j07QMOwx9kswdp6LUO5pbQw\nekYa0XYuh5WVeVYgz+widLUBYAiKrq4NOvbqyiAgdCTgwkYf0dyhCVk0bpnXuEi/fH4ebOc+p0jv\n3wHTgkvq6H4LfTSh8sb+zvL8hTnkvEmSJOhYHIOTPHSPX78W3f9jEz62NrIOUCaLusLaYrx5lG22\nK5CfubYkSU1uI/aqDaraoCuGNE59GYsZHXPfPKrFzKPvHzunRtcbWBRpqrj2jsu7cLGDbBqB9zjm\nOYpvBtMh9ZqUi05w8K+SyysTHdd1zrRIh996XYoj8MKuB7nKo0dTSFlWOF0sbFus81rlwmvWTtED\njImjx6+OPliAC/jJbA6f67ifx9AhSY6VyWQxySSTTHJP5CgN+cUXP2Jp7rRhnMe2SDgfiD9smm6Q\ngXbWtkKcQudAnmXYWs3G0eS5Z7hLXV2vHK2lF5mXJDGIyOPGrZUR1hlF0UC715pDyDEWgh7twxdG\nUYQtMxQIjKfr/Q6EHWlxHPfawGdDx2nROiKHm+VvrINmkiyQyjw0FrqPY8fCr1//FmrD2LHQstoa\nLTdfX2Jt3/PLj4xj9XL2GNHVf+rdT24LrL8hPBiwOGRNJk7Z9T7HjoX/vNFOm0EZggUPjMW+ObVr\nXoSeDWnGfhm6Hc96LHwJ/XZoLE4WQ62Z0jS1RL3ScVc2LWbybbpIXGKDq5qOSLduEBL74NSYP1fr\nNRbWcahPfaRC5Sn7iY1PAFz8RJqmKis253IksRljZdKQJ5lkkknuibxwc3Mz+uZf/OhLN5/6xK+J\nYw1wjrGyLIXtjPLKI2ebpQF/Nitkx6TdJY4jsbdw58sz5wSgnWhepOLoSwLKPW1MNO4DYfsuhbug\nthPFwmkRsJntKMt3rGiN288dt6ucZ9UWtke3hWWMbQv7o8HzH3ZbTv/mWwCAz//738Xizy2z21e/\nKve9/vrrvefeeMPwV6w+2+Gt//AT82+W+blPD2yax7THpe+iAy+61bgAZs58EG051J7efer5Y9+T\nIY93bfHLu0tbNBWpX86TpxdYWK26tIkDIkRCBcwTmE6EwQAy5jlsu0ZSpAmXBoYw3WURo41zqRcA\nonSYnbppOjCL3bvvvvvdm5ub00P9Pdqp17Ym+k0IaJjO/OQUr71psYE2TLKqSzFBsFNVVQ+i2Oq6\ncfzGEfG4reQOk8wLbSskQXQIJMoBwUHXkyP2HBRt18lLFOdF0n/JfNZHA4RQFnEcSygt6zehuQ6N\nADjzDIBemwZe8oDfT394uk3+h6zpIx3RiTpmJv3+lFUlByrtEKyJM7XOkUOLQZo5jO4AtaEmM99V\n13WCYdYRZpWX3DWOnLefv3zhaSQkN3G9cvfa+rZb8zG2FkfatLFE+7U24lC/x0jNBZe7zb2EYJts\n/UKCFMcHFynWIWXLBjycU9qMcKhNDvUS9epkWw+1KY7jwbvQbWIZ2ikcapOEq8axmAzFlDWyTVpc\nm4YOQQTe34PzM2kPER1N02Cz7btyuVinCulEs0LbdMpMOjQ10OH32hY4O7HflVUClyfFwJlZbrd4\n89LU9+6gtLBMJotJJplkknsiR2nIP/dzL6EoCsxnqdpJLBeAjlu3MJWrddNLgQ4YWhLuNJrFn+TT\n3F2apsOJjdp79fHDQVsc7d1uZ1AI7xo6zEVRhNjXkPdA2YC+tuc7RQyetL/DpkniIvCURpQJbG+I\nB6bzZNMmopXtI3vXWkOoTU7BcBpbSHhd7/h+fTNUWHmjGbpv1/HZp43cqWXyPdv/d+kcKI1m/JWv\nGTNFUTjzFp15r37m8wCA09dew5e9skIwtChyR309dr65Rf9bj1MQUsj7o+HRfRdO2a8/1Fb93gbO\nXk2HG7g/1Cbiyf2TqxbtWNbt9E9i8YE26bZJu5TGLdplgDiJlLMhGB3boyVJEnHSSb0aE/6m1Z5J\nxdK2wrnDeAlNE0pzRwqXKYT3l1ULVITW8XQWidkGP/npYAxCMmnIk0wyyST3RI7SkLuuQ1VVSJJY\ndgGxF8VxD5wNGEiKD13L4kZ2mrWFui11HjD7zzSgdfQoOfdwMFDqphnAvUJa3C7xd/qQLSyk9YSC\nIQzBeCTt4rXE00a1JuI4IIb2rH2Qq0Pt1LY/zU1hC5PfCN1bZo6JjPcbEvW+hh3SXHbBoXy+jpBd\nV0v9r35PyiX1KsterVZ45ZVXAABvvfWWXOP9lbUd09Z/CGqm7ayhYBtKqJ299+JrbL1kDkOI4CEG\ntlA7fe2z954DjIN8z3rceY0QzZ69mO8pVP9IOrNQO7WfQ8/9UNkrG025sPkpq8jZ4mlPD5329o1j\n27Z4dH7a+01zc9BZl6Zpj+2RwnVN26JdoJb5/7p0Po6xchTKIk0+evNg+Yke96c2gl9Y0p4zS6W5\n2lYqxNAMjjY/hI4d+47k++gO51E9IDUJLchAOGJsX70huU07OfFPUxsO24UTro4p+zabiv/Mrt98\n4h1NShPCOu8bz2P6MabfJTJZdMdImqZiIvuC5YHe1d73890D5v3rd7/r/l1tGRPaHXp231iHymnb\ntvfugb6ikBy5+B0zrsf2m9/UIml73z1gNp1Q1OCu8g+1k1HC2rTCuVUUhZhdKdebNaxlA9/5wd+P\nQllMJotJJplkknsit9KQjQOEHBHmt7Zt8cQSfbzycDl4dt8RNo7jYLQOhZoY6Qm1CORMPReKWNPB\nCQAAIABJREFUcNN4yn2aor5n13UAA6jQMW0OCdvcNE0Y+3lg/LSE7ikUMfhtxrjxntFRWtoEE2rT\nvjYfGmN/fJ+uazFHfO1rXxs8y2vaZME6qCGHxnhse3f9rscX2D3Gx84HPcb7NMVnOcZ+m4H+vNj3\nzQFOG73NNxf6ndf3rRG72rpP9DcH9KGcuu5DEZSAge7SGsD4itW2lAwkP/3pTycNeZJJJpnkeZKj\nNORf/lh88zv/zNDc+dAoAAM7Y0j0Lhhyfmlug327asg2NCZtUqiMsU4e7TjY1/Zd3BP+s2MdjKE2\n63pvM958fqwdODRWH+R4P7UEFpvNBo8fPwag+CoC8vTpUymfQSKff2A05ENjvmu82fbnbbwPtf3Q\n/Pbbvs+Rrr/DfXbbu7ZZ17erzX45u+oNwUVD9vLbjPdXXzP+jklDnmSSSSZ5zuQo2NsvfNRwmGpv\nqGby2mcX0uxjqbf7h+xFmu8gBN3yicB3wYhC/LO8dx8/sO6jDo1mm/aFMI+VQ3bq0P9DWschiBnQ\nD03W4+lrO0mSCNxNh8+GTgH7NPREwcV8rVCPLbWzHutY2w7uWy6NX2Kz2eDKcuX+2Z/92aBeQuIu\nLTH5WLSHln3BBrvKCZXrz/1DNnaOGcd/TJ372u+/s9A9Gom0j/lPz599gSk9LVO9z7HfRqjNoZNy\nqLzQfOSY+r6kPMsG302ozNApNrSu8F6/nC+9YlBlf2pTkR2So0wWv5GlN597+bzXYcJjLutogAXc\ndQQLTRBOwjTwIVOOhavp+kMk6vqoJing6yG3gRb/KBla6DT+VOfKo4Re3F36sattu8rYh/et6no0\n4b7fj7GOGu3sCWWf0BhT9pMmi6v1Jf7iq18316xZggs04KL2yHfx2muvyW+ftc7mscfQse8CUBF9\nz2hOUXx+Fl3X+zmn9Eap27arjLFzSuPPn+Wc0v3YN6fu0g/KtiyDGPx95qg//ctvTiaLSSaZZJLn\nSY7TkH81vfnv//BBb8cbq+3ojLqaFY3iZy4+lO4nV5mbgTD8bBfMiCYIgt6P6Q/brGFJIbjN2P74\nz94GRsf+UBMDMOjPLt6KMf3RfTn0Xsb2x39/WgrLMvfW1tX11pWBVD569EioNfcJNeTFYiHmCz9A\nBHg28yx0nxbdn2fx3Uim7K57X76bXf25T9+N3867fDf7+nOX70b//r//X387aciTTDLJJM+T3NmG\nrGWs4T4EixkLyPY1B581TLcjtEOvmngQTn0MtGafhOBsh9pF0f26TZ+AfpqmYyFxh/q1z7Z3qF/P\n4l193XJta6k7y0fcKdunhbilkdOYHHjfacihUPAQDCqUPdy3L+p7DoXm7vOLhNqyDyqoQ3hDXCgh\np1UIurbPlnuoX6FnQymh/LaEGPLath3AWUM8zB/Uu9rXr9AY73tX/+Gv3372BPUROuSoJE8e4KJ1\nmMJ88Mwe43ioA4Ks6LrgwskB3dpxjeNhlBjU4uHSnZuBSwPNjKJIFgFNyOL6xrqGLzDUPz2J9AvT\nhOZAeAHdtAnqpho8i8Bk4wK3q09sC2AWep9spukalXp+f998jKX+KHSkXqhv/kJcqw2rDmCJSSaj\nCaa4+M7zheCPuRCnqi7hbrdNz7JsgFfWx0vdL38+Nk0zIHRH0wyi8vRY6Hc82KBUv0OY31BbZjQd\nRCoyzr6qTTNEVPjl+P8fLIJdN3gHRZ4PFlAdKXhoXvrH/FC/AJcdm9LEHcquPw9DVKSR4qjw697V\nZt03lsvfBu8Y+P/a+3oQSY5s3TNSspss+SBh63ILtmALVLxto0ENr0ENamOMgdvGwBtjDBkyZMgY\nY40xZIwxxjVkyLjGGNeQoQcyxhhjjDXGkNHGGHOhBX2hBW30Qi3UhRLk8nIhF3IhtTdH84yME/ll\nxInIyJpZqAfxOV2dGT8nfjLixPkLq11YPgYhksaOny2pFC+c8CGKLCIiIiL2BJM45L///B5tmi7Q\nOnNAMxrarCLGRBuSrSHyxJOCzzuE7z5acEetja2sa2NXxzLlI7dspyjBbGOapjQ3dnBXXivk40jb\nfLRwGdIJpmujLAIiGhd7mHbayM34xg45qFy4REDCPOvM2bIs02EN53n3bD6fU2HYTkvHemxH6E3L\n2sQMzL9C25aa4+iAjxa+QboLYWmXw6cT/T22BRWt3KfiN5Akgyu4mB5LhJgk1nVo+L+vjS5TM+t2\n7LalNGmsdKYJHor1zLQumn30abv8ke/MZ/ss4bqZ5pdAFDnkiIiIiL3BJA75F+/9rGWthdoZtJmK\nY6cI4ZxDPdZ8igOpXBc9ptwXFU8DqGfMXWaZ3/NPontMieB7Fxr3IKQ8lHEN2mq0W1JOjI2h5IE3\nZthvts3l/cT5JVm99v4sS0uWx/nKstQXUbJSb04ljKmfg/I5V7gcFMz32NZdxzRNEivmRJIk+lyH\nY2qarGFbfRwdKm59c3XMI1Ty6gwBOoug2Zmp+yCSzfI2NBxTqV5prvqAbZWUo77561rXfJi0ILeU\nWEcFX2e7FhmfQN5b/8jAuiwwXOVgZ5kiC6SJSXPV76Ndaqtvwkh0uuoM8WTEd1pU4tE0pGka9AGN\nLVCSsgPLNesY69uZINq4c6e/7IAVd3zrNP9f17XOyxYYKL4Za+tYOxm+oO2+do+Vi2O8VPP6Ru3x\nKB6RLFhMjAWzCrW68XmzvU1bsUwUT9xUQ0UoftdSe0OYoKltRdok7LI2SIgii4iIiIg9wTQO+fVr\nK+DPGKy70xw2vyInElAHls+/S6UsSJMGbqy1FRBYpxmwvGka6ybqUJqQLrNMqzyj3W9TPpYh3Tko\n9bHZ7qosdx6LMbp8gX5c5T897+JU4M3SbO72+PFj/ezly5dERNqL72rd2S2n1AdQWuRLXbdksifR\ntmu7cT76xh3T+epq2hbmde9NlhonkWVaWwpcSbSC9HrFMjt85yHtDq2jO80Ny5PujBxrNwPn+7ts\n9+DZRDEFInLIEREREXuCaY4h77/fy3GUjI4NrZ3mX8ZOk2eZNuNC3+8yGfeec91eS6Q4WmFX8+2a\nWP9cJWOvt7Erj7AM0dxMiphmKJAw1gbLxSTlgMtcyQeJJokjQFkdEdFVQ8EXw/IcGJh6eeiSTJBQ\nOWNeWyTRjmCuOBQDztcw2Zu1BV0pUqa2fzGfy44EgjmVpKheemKG6DEDzhPL5/J4vCX9gMT9MxXD\neC9dOpc3qwQtV8WTWIAZGepA8BsQFYGCOZsJ0azT892kaSquQ3oOGM4/Fk0KlfQNBJqpSpi0ICPY\nDtiniZeAxyxGN4lsjx9T+SVpWzG9CWlApAAuqCRgr7dQzXDbtpMF94y6ruEDch/vJIURkRz8h2ky\nIWmLsQ7ug/ksvO2WLfgO4A2wmxNu+14EK+c4/CbRMAQnUS/WoMSeB9K8aNMZLebTrALwG9h1DrRt\nC33gb78ZdlPyOOyYm2kbm7mIz3K/NYZJP9F05RXmxe/PZ+PNGFO0hQbFMvuipmzy9/8uvgFEFFlE\nRERE7Al25pAZY8FrrJB02Uw8OtshH23fcF9wDxct5s49p1LvyGO7YKhSwpd+8N7wOEJFBNrt+syK\nGG3b9pyFEHxlaht8Crdd0vnqx7yS9xXnubq51uklzgmDz/swdgccY8xc0sS7mh9m8Ha+0ZyIBvPD\nF/iGgbeBS/Wb9ImnhRFu8l3Nj5DAV5JYD9uBdsBTwsWO9YX0LbF4bdNkbzX3fYgcckRERMSeYBKH\n/PPPP+udRZTPCuY7kpzIdODoEnZ/fPIvojB5IHKbvKtxuejnL4XzM8swf4fWa6VvW+suwYJm2ulE\nR4hKa9KyVOiCKXLBMa6UlVFHs35cJK7a1x6kWTq5jME0n2zTflzOVUD5ed6bPP2jgPReqW6XZMlj\nc8V85vI8xDxm/dwXN3UK8UzsPMgZSx6mrrkS6jmG3KM0V8y0SNtYeaGQnbbcdbVtH6zf1yeMUHqQ\njg3I2k2jBmmsd5Gr7ySykAYV7fuqxjcR+ypLkeDKehJqEyjRwgqTPAtbNLQNJdorChpkMa9nk2qa\nhgq2oxT6h/tlS66BfPt+Yfr4Y7+p+2AuYx5cov1qtTtN5jG9Kku6vLkhIqKjVWdzvEiJiqrzvGtB\nhFVVttusCy6FqEiT6oMS2mUGepKO5BgAp/GJR6BvxtykJftoad5s02F/dzTZ4+KjRYI5V7oQtMM5\ngFYeNg1DmPbuu9CEtEm2ztw/6wEtXGbhpG0XWnLDzh8twMb6x4cosoiIiIjYE7y1Uo93gHlq30ZQ\n1bUdSpJs7gCVf3z79Hw2E7kInw+8NsER6pE4Gzxym3apYwpEDHvosxvmXTOfzSwlTFlVOq1kPuOj\neVsUOmyiefyXlKXi8dnRPyYS2Oml3h9T5PhEG9+96sQTq0WuOeNjZRRe1Y0SWxA1INLYVhtnXWb5\nUgwMKS8G6pkqgknT1OoXlz25j2b8frR9PYaondk047dG5OfKJJPPsbaKcTtGRFXS9y3lCVXWi/fb\ncTp8lg1pxu+NTz3I+fuUelgGrklMj/mtYUlo4x1qPseIHHJERETEnmASh/zee+8FyUQkrhlNUszd\nuaprbepTe67UaZom2FQG8yNq4VoX5NAZkkmRxPlKJlXLtBYVIOZuiV6LWK/uq8w23+Gd/mhGtDHk\nfC5aXZDajfAFvHdd7xRi2I/A68BM5Fk/146OjvRvjlPhA9OJ+Rh/eP6tVe+dY/91Zz5Ts9BLClzz\nUYI0fqyg7vUi2eBb47rMeearf1AnmGUyXb6g9RLGnDZ8ZWRZJtLIppGhcwu5bIu7Fq5aQucWa72i\nuaW4y7MMjAX6Ncosb+q3QPQWIouxjjWBWk5eVLijm6ahGx04pS/fN6Gliw9d7/B9qMUEKiFQ7GIq\nt/IsG96hRUSXVSrec+c7Eg6OV4ZGGOnhPrsqCkrTlgu2yvSFg0Q6gty0hT6TJpvUZ4OyhT5jzGc5\nLQL0H6enp9orz7wrT6Lv9PRUP/v84WdERPT1/RPalt37y6Irw7U5+eZZCHMglYEY6zMsjz9+VlCj\nSBDr8lnZSHF8GTy3yqrSDMfWuImFaCga8/VZaGhTpjfUsxD7jMUIrjbP4KYXzIv3NUp1sWVHmrYi\ns1bS8BtFMQbW4Qu2JSGKLCIiIiL2BDtxyKjkkWBeGU/U2+3NZ7Pe9lTtHov53NqxxwJd+2xvfbfI\njgUNwt1tzA6X26XNpVR35lkmKzZ2vMmgKEvNCXDfzmd2IBb8P0Sh5DLTs57DTct4vJPyS2Ziul6V\nvqprrcxj0UHT1ESpEVMBRBbMFT9+/FjHsDDjV0i4ffu2DkJ072jV0ZimlC/4xNTVvy7WlM1XVv5Q\nm2zpdmoG953EXedZZpmESV55kqmXdCQfU16b80IKz5pn2eB75XKl8sxyXPbs4lwzTpsY+pXf4drA\nwLBHGOBKGiM0S8N3aBgonUywrWbbpFPIoAxMP9FbL3LIEREREXuCnTjkMSUG7i4oCOdnh6zAawWP\nNCWbCfV0Et+pv4lgkjbmUTM3uACioexaBwJn//ps/PZbs+5Q2SPTgCaAun4qqTbbYng7Yp1jnof6\nXVfJoH6Js5LKmQu0S303JTjhfKZk5usursXx8TFVrfIeW7kVcWzudgzKui/udGZ1h8tFb+a42UCu\noUy6TOZWG119p58KSiMxvTDnEKYOIMsyq98w/sdAbqkdkNzepz6ONU1TzRnjrekmVkml52Bvkdev\nDayUHouGJ/XdmPKNIfUdrj++OdfHyFCnC/CmnRqHAtOjCesUxySiHV2nEcOwf8PK67rW73ER5iGT\nlFdT7fYYw7iuHVzXoYfklTx2syTRmv/zLU8E24tP8kqaj7itSuETpfesOMTF+B/Wj6qI0GA7Ul4J\nvjCqJg7zrp9vlGihqRpip0vJgoLB4oymaejsuOuflVrc8QjNtGRJQt+tO6/Ag5W73FCI83GXfvQg\nSxI6Vt/VVdIr5BjWfAxUIDZNQ2Z4A2mTv25S+K7d83EXe1zErv0o5ZWwSro+yxcpXdVDbzsi8CoF\nkRtjzMJp6oIcRRYRERERe4JJHPIv3vuZlmktcmRlMreE34dZowNyXNfSkcV/35up2BgzsZF2ztCr\nyH1B3rmMom2p2g7Nz6Ry0f4aFW5iUCWuX/WT1LdEPWe8UW2sG+TC3fbCDJc3nXRztS+Qka8/Ma/5\nHPHsxUsrTVW3xIeOtO5pXc27fr7bduKG582G+JCLd+qZ+PLLL4mIaLu+ok8OlQfggVLqwW0Yfb+U\ntMqH8QkQvuBLCOxPvAWGiHSYVFd6aS77xB5F21JSD9+jSSGXJx3rfeXO2oKolcWSmDfLMrpWySQx\n5dix31TCScpMIjtYkovjluayi5semJly+qolfozf99T+lEKChiJyyBERERF7gkkccnsrsTgg5iay\nLNPypHXb7SCXVaKN2HnHq+ra4i47r5euHIk7C/UQkuTb0m5t3sjsUjiwNw7LotH4O9QLB01leBd1\nccEM04xtlVS0bpmrsxUbkq//3FRQZfNgzsWkXeo7qQzJcQfx7LuXQfUOaFB9cLjoxmBdt1SUncyP\nzdlu375t5WMZ8vHBjBaK852DyaNJf922tK66flwt+7kqXZvEN5mjziTUO8uca6hnkThFyQtScjph\nzKmkm4qdrKbp7Llc5Cp9nDTmQXpZJrtOuzkrObBg2drbkMqBdyrRNA9B3/z2zWXsV2neSgHyQyHd\ncejDTlYW6MmDC86aes87om5CmEeQPMusiTU4VkCHmALxqYoB12CaH42rg2/UxjLLe/tiH8bcTCWv\nL8m13Oyz9cDy0qbZpKtpGipS46iGSooAERACFwCfvekurqJjqIx+PJ5nxI7TX3zxhTPfyYrnpzBm\naU6pEaKyaVp9FMX5y8B+6W/x8PeVdISW0vtsxn2X1UrYUNZ7u4IrtFm/JBowg4OZ7yUM43gT3VQt\n1QlbW3VpcH76FsuCZjp86ZhVkK9vQ7xxXf0uYeO599LnWzAWTEpCFFlERERE7AmmiSza1wPumGjo\nbSMdkXw7xJg9rk+kMNVOcGyXNY8laNpSCoHYQ+ELWI7vSmLurPDaQkq3sphweUb50k3tY2n3H+vj\npQqrOctTurxR5nt1155ioJjsOLoVcCTstXe0mNHZYaec2xbnRES0Kew+LquursUcwiyqIzQ1/Xiu\nVSyLqmloo0QWi6zvY4mzCvF4NPNwutB+luoNVSayApjFXL7565ufIe8ZJQSgNIP1hwbDJ9r9Wx/r\nn5Ayp/QxrxeZmqNrykVz19D+Y0QOOSIiImJPMIlDTt5/35JXahOYwDLQawe5a4sjgGcLQVkkCekt\nX38sd2Sn6q+TEoJhK4wpOBizthCdH0zPuoLsIPwzwXcfEepUIYUuZWDsgJ44O9YHlmHFDmnbYX4y\n+ltAqczZylpOw/JipgDDb3KejtvqOL4v7nQOHOuyokuBSybqg913RPecYqGimHFfFHVDdWN7lYYA\nlce6KuiHAvopNcZjMbfD0Zr5iboTG85rKQ1jq3/1kewYbzuvieQ5iHNbUn6FcK0uOkIiF2KsD3Ne\nEg3ntVkGw1WGtNb0fcxl2OnHdE4SIoccERERsSfYyXXaZXbj852XoOMJuLT9Hi5PrIM5CPZfFzgX\nCS6zHJEmBR9XUSZzKoQIbJJ8ypQJuyw00JpF53VwSFK/D7TW3Ibctt4INTfE4Ppo8uTrb5YXj+Fm\n23Fiq2wx4JKJuv662qpYFovekmTACQMWQmDqqqqo4it9FL3Ite8i0/TJ06UYH4zQGMBSf6cQ0D10\nnksmkr68Upxu5D+lSIPSPPddMxbKSUvppHkdEk/E1WZNn3T1HDjEhFp8hVzuipi0IOONIVOVamMm\nYf0tuyySCCRNNAtSZTjeSWUX5pHcczO0/bzrE771BE3XBmXy8VgtOPNZrk2Sek+8zLKPHDsSh2DM\ngwr7P6TvJfFEd6y2x0/qSwaH3zxaLbR9MZq6VWqhHNwesmBvRWXWlaWU87zkI6fKx/EriIhu1K3W\nHU2qXJXvelNpWrD/TS8tV7+HKJRceYPmvmCK1jRNUF7pnTnfMR3CNxfSNO3nvBGwnekjUuKtisPG\nduORJEk/54UAXQzXwheyKYZ6DfK8G+t3hqi8c3w3vrkvIYosIiIiIvYEbx3tDSFxz9KRQfJh13yM\nZMcfyJVPNcL2IZQDxZB9m4bbWntpZi4hyzIreltL9pEvVIwwZl4lhVLk8sauwpHakxpinrH+v96s\n9e8Q8cV1WdG9wyUR9aKH5XyuxQ1LFediUxSG0o/0bdVpmurYFUz7pig0d11UYHpYdu99lyOEmhRK\nY+C6gmys77FuTOe7cfldYYym/n4/NyeaZ5ll+lbXteaM8RswPXWxH6eED+U6zLxSLJa5EAVwMmAo\n3mYMIoccERERsSd4pzJkyzTKIUfV8UXBxETvZIG+8yZCTXdwZzQVDS40AoeKcuMy6TleF70uV2rJ\nLTpEUSNxCwPTthAuDsail0U24liEKmylcXj6onPgYMcQl9kbcyqtvng0o3WhYtWq+jZFHz1rqTjf\ngzSly5u1blP3rleAMe3MWddNq9NtK7/sXOKopBMeY+oYENnzsBEUY1IYgjHX3JBvQjp9uZy8TDTC\nt9R9D7Y8WQqDwDqSy1ad3JLUckeX4npIAe+nON042wNmb9JY+K562sVNWsJOsSyQkMn5pN/CLcQS\nOn/5oUD9bYKzh9oJshVIXdfaQZ+VcUXbBnnjDBZ1w5PJpEkMz6naKR173/VYMMbGRA4jao8H42Zj\ne4zN1FUTh3mqAwiRCpdZNQ0dL7vfzy6udR72qFvedNagtw8WdKjS8U3UfGNIWZaDRZxIHguiXgGD\nShu2+TWDNRGRFQiHaLexMHOkRMPbMxTM8ZAYgMGYqKZJweVFOnjx91Jr0GnQdlNVNDN4jLEAQYPF\nz+fZqjZUViSaxJgXZEyxhoBi7P8964R0Mw4iht+MiIiI+P8UOyv1Qv3FQ493EkxBvC8sYOhOOOUI\nY0bcStPU2kEx8Dxj6q5I1EcQI5JDkErBzRlT7b9xLHYZF04v0WlOqLZt6ZOz20Qkh99krvS6apgx\nptV8putghdxnt/t59O2rLrQmiz5e3mzptirvRF3rhO15ddlx1xwPI00SLaooBPEJcmCsqOXoeYPg\n6CNz7l2PS0g0vTa1byNnOpMkCb6wwaQpFBjNkU8fOdhLS3ib7wXXiCnc8BQvQp+oaMysbmr4zcgh\nR0REROwJ/uGOIaFmWrib6meBHmtT6xdNuAROXvJqYvgi0SEnL8keGVmS6Ihcy1QFw6fU4oQkjsll\nDmTC1/+SAsoVzUwaHwlTZahsanaTFnStLjRdK0P9WZbS/dunXb3gVfjZaccFv7rZ6PTMLbPibn6o\nLiqFW6VdykQXsiTR7cHLCiTOd6rCjTGmHA/1GtTKKPJz0u9C8TT2/ZiyaMkTFtP5Li11fT/mZRh4\nA/fYd22+88E1FiHr2lg6CdOUerfc14JL4gkJLvdHS5j+ju0pXTS4no1NotBFcCMcWXQ/1Q3rCAeT\n0uzHt9GmS5NjiqUMp3/X46MVaKq6gyShp5edpcTxUink6oYu1sPj7CxL9T17d44OiIhovtnS5aZL\n9/Kms+i4rxbieZ7ruvStI2WlPfleXhcDOoiGx2HuDwz4L819E9IiLY2TywtTGiPfd4Vz9V1+O6GW\nCz460ySxmBuXWMaEFIBrSF/3d9Nk+nfIt+n6LnyKU1+A/12sayREkUVERETEnmCaUu/1z5bAPtRk\nhTFm3xhqT8wY82JrPNwH0lQZIghXcPIQ+LzkiIYxKqR2hNSH70LiXEg2x2PQYzvi+bcL2Ob37lHH\neZ6tFvrd8XKu0vRc1LPLm46WtqVrFVzoQNlXsckbEdGrdWcKx/bL/JeIKFN1XhY1bZWtsxy/oedo\npbgijDGzQ/P5lLgMkm2yD1Iw+HcxZu/q+9K24PDMLDe0HyVIdw9mgacG37ckQeKGJf+KXRA55IiI\niIg9wc5KPdGAWiHUNM6ncAvFmLzGUsgJCjrc1Y9z3t1aHV/CdxO25NWEcsFdOGMfQpQJUh+KTiBC\nuf+IMWNzN+RGFwsVIlX1z7qs6FRxyazAy/Nc1/fgTvfs1c1ac8ivlNz4dpL0N0urv/zudDnXsuOl\nClLeNNfitU954NR712M2Nl7S/1zurorDdyVnNud+PpMVdOYFsk3bwrfWoW7r0W/NrBO990wnDbzR\neypClXb6meP3VOx+5lSQlHTSTRPYiDHvFs47Vh8Cj7e+Z0gLDjQv3GuhWkkUg5PDtECQaEd3T7S7\n9E1AceDfQoNrtcGhwZdCTpqL/djRl/N++4fzgcKMUa27Pr1Wf1eLlJ58t7bSnRx0C/Hnp50CL00S\nOlMBhzguctU0erF9cLdLd/zo34moW/AXhnUFbgxMW54SfXbvjrMdbzN2XuuICfawEiTFoXS7OWOq\nO7VZjwmfAtq7ETQNXSr5xYBp8ojfUCTAv/m+wFxggtK0v1la9LSc+O256ApJH4oosoiIiIjYE0y7\ndfr1a6dCCBVjjMEuIuQTOUBhFwyBTzEyRqeUP4OYEqLpGtDmi4lhKkGJiCpCDqKrQ+SiMKZCSLCX\nUNtw6fRhHAut8gLHj/Hti4sgWhjrrUz7xQ2rgTql3nKW6cBBLIrovO7UWKm4FctZl+a6qOhkpUQV\nbR/QvqjUqcbD2OAJTwfCwtuVm8LK05s0ur8TF95m/FCpZ76X7pgb0DTyHeL/Vr0BppODOSUoHyvP\nPB98h2yOiIpL121DBjjvsG8qi96x8XPV+banVUbkkCMiIiL2BLfevHkTnPhX/5S++Z//e0ENCAVT\nYDEaQ1iYL3P9zJXHfJcvlUfWq4yOj4+JiGijjPxRcG/KOV3hESWZmaRw873TAdGXS9ukJ011hDHm\nhhNHqESzPpT3jdEk5TV3ZewTn7MBpmea+RZmyVQI60HFpOQF5TOY98lSt9utfr9cLgfwWvqlAAAZ\n/ElEQVRtfxtI9SdJoqPBvXr1ioiIDg4O9HzTpxpBHopjgR5xIXMK308db1ces8+2262er9J9d/yb\n2zOfz+ngoJO78xyoqkr3z9OnT4mI6OTkhFar1aDcqqosr0Dsb0kHMeZFZ85pqS9wHuF8lcLW8jNu\nD1/jtZldW+uVay3jNWlsLfOtfz/8nz/955s3b45pBNPskNufrUqrjTqC5HZRlRBuMTTP0fKQzs87\nr6uQoCouhCwMoceNm5sb+uSTTwbPrq6u9Iaxi4WCiXeltPO1Efvk7OyMiPo+ns1mlnIWF25u62Kx\n0NYQ+OGZYyUt8BkGoFF0lWWpF4b79+8TUbdA8HtO/80331jK4HneUlHZiyQR0aeffmrVP5vNaK6U\ngPyBHh0d0fPnz2lXhPT3lPe+9BJ4XI6Pj/W4SEpzaQG9uOjES+u1rVRl2larFX333XdENH2THFMk\nch3Sdy4t5thfzAxJG4FUN8/3m+2lVVe1qSetY5yHaNr650MUWURERETsCSZxyG9ev6GmaqmpGkHs\n0Gru2XzXvbffNUqxIpVXp7Xe/TCwTEjcBt4tj//4R62O8/HY1x9+2NMEec33ZVnqIy5zH5vNRv+W\nAqwc/vCDVd/l736n05u7PqbndBKQPizPBJd3+bvf6TzcF9cffKD7WJd7fCze7sD9zulPTk68nN8G\ngvocqZCYZhoiOEJuNtYx+ebmRtfB9d49mdNKKfXWKoTmzdY+OjOePn2qy0WYx9tXr14NOGii7kgu\niahcbTbfs+hAyoN9bM6R9UcfWRzxyZ/+ROuPPiIiotX33xNRN44bNTe3260uF2/XdoHLXyx6L0du\nN4439w9z0Ui769TF3yuan0nfrSl+dJ0CpFC7Zh5JJNc0jWUCeHnZccZNKq9XuCaZ7/o009e/UEQO\nOSIiImJPMIlDTt/7BR1kC6rampKm302JlPyw3XQJN73ssU2UGVLWyQebujcVaqpuJ8nzxmZhYZMx\nOVAJLq8llvjOPe36XOBWHhMbUhE9BDqYE2HkeW5xBOipd1elOyKiL9VvpJPzMteD9a6N25Jd9Jnp\n0jQdlEdE9OSPf9TtYDpms5klD7y6urI4khcvDujs7HrwDDkmxsnJiT5BIFfKskcJKO80ufA8z/XY\n56lyIMkzWqkYFswhHyxyOlCM3osL2xRN4pB5HPnddrvV3OKYc4Cp1EMOUFJkIcxnq++/12PElD+G\nsJ84jk/Ub3McifpvRFJySfXjyYifoVLPPG0mQJOrLQzUC7jqb5qGttvu61wsnonlYFlmGSF5Eaz0\n1Ir3IqMsVdz1TTd2P/74o07/m9/8hoiI8iShXN/PqEwlFxlVau4lbb8ONq3SdxQbRfwMvqU/B9E5\naUH+77alouBOVgtT030cdV1TUbBXk4pnO5tRU6nGZILNZi4cUZr+A+WPFQfRPGpKgT60koD6dR7V\nFfyMpysuSnchPb/nRfD6ww8H4hMit5b8SH08K/WsgHolryqpXv4YUaQSkm71/feDdETdR8y/ce8z\nNztcBBlHR+da4YZWFma658+fW+m4TCLS7zabjT46Sq7l/BeP0/cOup68LEq62Kh789TRsKwa/Zs9\n+zj28Xrb6CM81y+5K4951jFC3ZV9+bGOu9SPC/fm8j/+Q6fHcXwIv4lkMdxisQhSMpuKW6RtNpvp\nMcPN0bfp4Dia9xqmaUoXF11c65OTbsO+uDilo6NzTbOvPG4Pl5dlmc7Lz66u7gzKxrqYfqS9rmu9\nNvmQ0pzaWolW1LOiQKasXwd7i6V+HZSYAR+iyCIiIiJiT7BTLIu6rijLhsfkpmnob3/7GxGR/ju0\nKxSC+iiRBXLKLPZIkkMrvRTMe0zJYqIgvacNKGIuM4N3zIGw2OMx2XBxIZzHp0xs2za4XubSQ+lj\ntU4BaSRaTMWKxPFfXd2hTz8txHyI+/fva25UsgtnLJdLzSH7jvhN03O3V1cC8e8APpMrkxaG2XaJ\ndjTtk0wZcdxNxfMnRMQHcRxHaU5JtuU+SONscrRZllniG6mMT2Y3dHDSCQO/Op9renwmgMy9Hh2d\n09VVFzuEOVnMx/PiwYMH9PXXXw/KqevayivRiuPCJwFWtErtl9BQQWki2eNXqtx+HeT3uA5OPU1F\nDjkiIiJiTzDNMeT1a6rripoq0+YhZbkhIqKffvpJzMO7RrVhk5CFVvRVpaq+7DkULufsdk4m6rq2\nZExjShTm61ipt4Lfj5XM9fD77zX3wXhBROxWw+kPf/iBXiph/1gErccff0xERL9X8kBJqbiaUG9o\nOqLObKo2ZNi2yX8HU04vcTi//70QdFxIVxSFqAzC90S9dxymcynAzADxq0VKBwulCFXKlrPDJZ39\n27mjhcPyTPhiMUjmWhhBDNtq6S+A48Z3PEY4ni/UXxxPNnHDcZTG0KQ/z3PvN2LK/bE92Nem96nk\nmVkWP9GNUqImyWJQlwnmZLEvWA6MMHVEjx8/FuWwnDc0vCaXwe3q1qAur7R29dx4S3Wp6k97Rw+t\nG4N10LUGTsGkBfnvP/1M23VDP/3016D0P/74I/36179W//XHjdRYa+u6thqDWneG5FoZ8m5QF9nK\nOumwuqb+A+FFHdNxXa7Yx1p5MELL1HpD0jVNQ4X6oEl90LchHduuZnAjhs9mdGyx9C2qGJOW35+d\nnYmWF5IowBQlrLcNPbzdffx8t97tkxN6eLYZplNacLS68IWDxHokpSvS6HNhlu6cRLGeayyJhuOp\n7blZofvDD3Qb3vM7aQN0fQe4cWB7eHzYdbwoCr1woeWQ2bbr7F90X+R5T4dpfy3VK9GI35JpBYPl\nSSEHTk97qx9poZdENdJ8ZesK3Z/tnNq06/GaL9ItaNIaOAVRZBERERGxJ5jEId+6dYuSJHGy5ry7\n8O6aZRllSbertEnPFjO7P1MhEvN5S2xN16pTAR6lfPAJzTPq7/BiRd6C+iM+Hxu/IdsULrTeUDMo\nibvG52P1TqEPuVxMv3WkJeo5h6urq6Cj/Vg6FIFcX19b6cxjqCQCQawW3TsWVxD1N4AQET24c0JE\nROdX415qWJevTtdFAyHvJU5syribeVzjaHKtaZrS559/PihLMi1kDnSz2ehnrGit65oODzulOn/L\nksLNLNv83xc0SAKeUtBkzhR/SRdfvHp1ovMyhyyZMkr0IZij5ZM9nujZ5jhJKq94AtdBPn385S9/\ncaZHRA45IiIiYk8wiUN+//33B55pRESJYmnrNh3IVYkUl2vcMJXmNSWtMpHJWa5MNJ8pjiDpPY4k\nmSJDirdgvlt/9BFdC7vgI8OEjIiIYz+xHO+Iek4alSk+WaHvhl6XLDm03tB0JvgUkFHvZHCbvf0+\n/tiSIUu0u+BLt0ukOolbZWeOtt4QUafIW5fq3KO44TuQ/u7JMG5Gnqb0r3+wOXRJ6ShxshJMZZlE\nt8RFYjocT6YYx9Ok4RPq5yuOIyuPsQ2mGRfKdJkLRc6X6UNTN86D5TJNj+513PXzmwP9/vqqL2/s\nNm6izpRSkvWa3xcqGCUuF8HlsUkclj+Wl/HLX/7SesYn+jTv+i6n3HYQayuqW+NUA4rQUExakH/1\nq1/R0dERlfWa+Br3qujY+Bw0kOXWXhzZsiJLlpQp7Xg2U5OuOaCGD2LKOUsSWeCkkO4N48bzO7b1\nRKyon9j8UZQff0ykrCFuq2efUK884ZZtPvxQvDmDYX6oLjCdmw8/JFKLo6/eSfRR53Uu2dVyevQI\nS1U4UVTQcTtOTvpjIFtGcLnmhGRw21EkYX7cJycn2vWay1mv115XX27O8XJOp4fdQvDiwjZO5rCa\nXz39Q1e3RwmMf4lk5eSYG/XUdKikI+rGkzdNczxdwHFkrz62ypjNZvTkyRMikmMvYyxjok7BymPF\nYoq6rq3wqOg6/eXzuUrXC0/QdVta9HybHVp3mExdWZba7VliuhC8EEv1hoRfIBJEQHmtbY3n885m\nKWnnVNZrla5fB3ENJBqug6GIIouIiIiIPcFOnnp5nsnHuoZtWvvdVds4pssu77ylVAX1yHi3Smtq\nDLYgSRKvwoAh2QMjd4jBfRjMYeiYAHVNzM/V8Lc20ks0uHZcy3NLKAPT+OpFzmCMPqKhohGVR0wD\n90XtaI9J+/Pnz+3gLMAxMWd1c3Oj3zMH5op5wZAC2COefdqV/cXzrpXPrtb04GnHGX91r+OUl8sl\nPfqm82178t1TIiJ6eLYyi9JAMyyfGZRLQWWeziRuTTKZk74ZHEfXePI79NIk6saR86B5mhQOlv9H\nEzx+xiIBNkVM09S6PUX6zvCUZNqLI6S+QKAdNP9mxeSTJ08sGiRx0OnphU4nccpm+k5s6vfOZKRa\nmNTX2wcc8q+DU0V3kUOOiIiI2BNM4pDf0H9TmxRUlynVLSsO1C7Z5NZ1KnmeE6VsAjdX70pqa7Xb\nzsD4Wm1WqBCYclUMcoW4m780/mbUc8Za5pqm+vczJdtbUc+pXEO6qQovlrJtqedsUOVSKqXMMyUL\nFOsl0o4ez8Bzy0wnget/AfU/U3/RsYCBDh+M+/fvW6ZrUrqDgwOdDmWQUnnMJfucNRDH8+4Ms5rn\n9NkXD3U5RESf3zujXOXhcvndv96zY6IgfI4cktmUFB/BZf4mKf9Mh49nP/wwiAhINBxPrmtLvUcf\njqOew4K5ne9EKbWNlXp1XfeB3GF8TI7bF98D30snCFS4SZ6Rjx93kVkkLz3JMYRjZBAN41uYtEtc\ntpguZw/FXHvoNQ3PlW4NJKLhOtgMI+Q1TePUtbgQOeSIiIiIPcG0WBZvfu7cP9vSfol+3uwH3m4p\nU2ZvbdLnSbIubcXXpTTXenfBMkzuqW1bp5xPdNslWUq08XCUpsuxWZckU0P6mHbGWJAyzuOrF7ky\nU0svAWlDXkHqC9NqBU11MPYEyxQZkknP8+fPdTq8UgiN/E0gJ22Od5qmtCmGEtV1UdHnqmzmfvG6\nqBPjHVEvT/6uTybCZykxdh2RmQ5v/vadqtDSBmHSMDaOjDHHFd+1SdyPd+/eHTiJMFjG/ODBAyIi\n8VLY+Xxumb1JdEhmb3iaQhM85jgxPrY0Bj6zN2sM0pKoCYiHjGattYrsRtXoGsh11q3kjuXGpAX5\n9eufdaR8vYCmtpEOh59Lst67hThMHRWk+0aX0fTlaIG4rCwLMS2z4gDsgI1wbMQ6MYxfLtzYIZWj\nyxPslfUdfUK9mM5H31g7LDoc3nFSH0vp0DRJKpvLMI/2RVHQ7du3iYhELz7Ev7/q3nMgoa8+/4Q2\nm26xOFl1YrDlckknauGu2+7do0/vdemf/oGOl1267zY2nWNtNIGLhs9TzxXIB8vhZ9IY+cZbKlMS\nLbguT8A06PXGG+f19bVXPMEhUaVbqmezmXfDGutjs08fPnyozfgYGGQM4VPmMTBGhXdra0BUwmtS\n1tVZVrX1zrUOJqGuvwpRZBERERGxJ9jJ7I2IrB2hrisqy+Gu/tc/E2W/GaZPm3m/2yJXzL/T3lyK\nj798lJIiq/m843wcpvleMhEy66iqSpvjMJdwfn4+esQNocEnCkGRhe9YLXFJErckKdxMLgnznJ2d\nWeZMUv1SOp93pYt+zLNtlMlRZRuDMfeMv//t/txK9+XLoacXli/FnmBI0fvGlM2uPGa92H8+r9Mx\nhxPzHsKyLL3R3sz5hs/YtPHy8pIePuwUp3iDNb+XTjUoTgil3YT0rUjhNyXl+pjZ22Tv05QVczmx\nurWRGGpYBzloPa6Dfw27Sk8jcsgRERERe4JJHHL79zdUbpVijXeQiuWIjRgBiWNd9CYjFZGOjcy7\nS2WJc16+fKnjGNy7d0+lry2ObkCfh9sZ41LM3RzrQnA6dtE9OzsL4sxdJnM+zhcVbj43WOm0YHLo\n5+fndHp6aqUzzc9evXrl5WyYS8Ig8xIGpocvXw7eSZyadOWRJIdFBR7Llb+77p8xfZgO6zVpQw5T\nMlHSZmdKWcicIwIVnKHcoM+cTHo2iBQnBI3vucMrPc7S/DVPMJvNxnIkOTw81JwxXn2EVzzx34GJ\nKw2VmZKzCD+TTNOwjea1UmPAOlCZZ54gxq7q0rTwBahNYq11RDDO8I4547cJVD9RqfeaqqrSSrsO\n7kDN//zbX1KtbI7zBjSU6qaQP/3JnXez3FjKMiJb6C+F5GO4Pg7fhylpxiUlDk+UpmnI9PvHssdE\nDL7jqknbWDosl+vDmxKko6Y5QfED8IkbZrOZd4PB/jTpR8UPW3KUZWn142w20+3cKEXK1+cXVp1r\nULKwxx4r8i6LktKUbeDtxY3Lv3fvnrYskBReZ2dnHR2w0EtiHqkvsF5pHkjppLmvrZeEYPqYTgr+\nb4LTHx0dWf4DbdvqTRRvnTbHZ5vfULq2LWh8inffgjhmwTKwXjIsKSSGB8VL5t9gpKV1O7W9BhK5\n1sF//m0XrOjP/xW2SEeRRURERMSeYBKHfOvWLUrTVNgdhtABmouSmLGqChUdKV9QVQk2fAaqqtK7\npRktjEgWWYQe681dGJ/hsdl3E/OzZ890OlY+4i4sRaOT2iBxQq72uJ5heWZdLFqpqsoyU5JiDIwd\nuV1XHTFNEn1SG8fCIJq4Xnf13j9M6VLdZvBv9zsuKUuOtbnbF8/VcXXTcd6bsqY2cSt0mKblcjmI\n5WDSy0qri4sLa2zzPA+ejyhuwHdmPtdJ0FUHQ4ovwcCxRVNFrovbL906jXOaxTdp2Z+S8FIKCWaf\nusJvSukl8YWZ9+Li1LJDPjl5Zdlb6/bAnXo+NBWIhZRNcZpmo2sgUbcOlsX4WoeIHHJERETEnmAn\ns7cPPvjA4gjwuhLNOcyJEhW0mQPV13Wtd/F8bnMHVZFY6SQ5Gsv0GOglhrurqeRZLpf09GkXEewT\nFQv42bNnWhHC6cuy1HI0lBfzb+SKud0sD22aRpfHdT148EDTgJwYcx1373Jcut7UiGW+9+/fpxcv\nukgGKHtlj6mvv/5ap+O++Oyzz4ioV6g1TWNxXW3b6vYgp8V0Yh9zO1jBulgsNMeNpwoeH+awsG9R\nucf9jJBM5ngsWcF7WZRamXe57ui83FzQ2WFXHr/j2BbX65qOj92RyHiOffvtt7oO5Pw5z/n5uU7P\n7WE8f/5c9xW3e71ei0pKbrcZc4Oo5zJfvnw5eM7p+bTDsu7VamV9c3hpKY+t1G6kicee69xsNnoO\ncN66ri05bHqnpWaj5MrXfbkhHoouWkxOuixL0aSOY1f4uGzJrDOUJrMcIqIWPPs++OADqzzzlNA0\nDeXKCvPP/xVW16QF+b333tMfHQffYIH3YpVqrzwORp+0S/g9h3fs7WcrQ1aHHPDZXoRRyaKv8+bb\nnetaDxzekMAfHL9L0/7OMVSS8IfCi9ujR4/0wikd23DC4oLN6cyjf57nmoZHjx4RUXf85Y8L22OW\nJ2n/UcMuhVs0NzNUPuLiywsNo2kaWq1WFk2mTe1XX32l+5HrqqrKEmNg3/IGst1uLQVaqHXCi4uC\n7p4oC5esz8P3662VPfKLi26zODo6EsUykgu+pJxlcBnL5VKce9gvRF34SOwXrpef8aJelqUuhzeE\nFy9eWN6PuCBy3svLS2tjG7j6euygUZzC9fImkaZ24Chso1bqPUOlL1l1IHwLoSRqY2uWr7/+Wr9H\nqw3p3jwJU5V6Pq9bvB9Uv4d1cLFScx/WwaRdEhHRn+mP3noZUWQRERERsSeYxCEnv7hFswVnUdea\n5B3Xl2VZfw0TsUKgpf6mMMactP8376pUQBBo9nbpOQQU5jOXyc94xzs5OdG7FgY94Z0Wd1nmbrFc\nfsZcHHoooVKGy7lz545+xmnRJpTT8RG/qiqtDPnmm2902UdHR1Z7TAVjVVUDLothch1YJ6f3xVvA\nIPP8t6oqsY/5Oieu49GjR5ob/PTTT4lo2O948zGfOjAojSSyCPHsOjg4oE3Z9SPaH7M53Kbs+oz7\nVTI1a5pmcHLgNoYoGtFkT5p7fPp58uTJQITE4PnIpy8WLREN55xpSocXNvhih+DvkOM5irIwn9k/\naHMsjY+keGagjTnj9NQ2X8R8HH5zsVhYttYm/URDu2bJxtlM353OA0UqigtOWnXKzjFfvw5qUQWs\ng90aSBTIIEcOOSIiImJfcOvNmzfBibP/kb45/F9LIiLiy/1YNkzU73BVs+keYIwKBc6H0LsI56Fh\nJCl0cjBvoB1zTpB2VcnkSLr+idOxXAmjYJl1IS0ooxzzyPLB50mYZZmlRJDaytzudru1uK66ri2P\nvu12GyzPlej0OdgwJM+29XqtuVqMXWL2WZZlWv58uOrafWc1184hLy46nYWPQ06SRCtgWVFWQJhP\nnyPOLkohCWPz19VvmBfHD+k0x9QnE0dZsTRXeY6h7Nw3P5qm0TJpyVvSTBtS3tT5iDAVm9w36+2l\n6KzBjhzsvDbLiBJ1uUY2G54kut8j66Baz77//vv/fPPmzTCGrYBpdsjv3aI0TShp59TU6qhHLOgu\n9bM+tnFD+ljAnnpCmLqmAkVEwse2/jjNx+/ZbGYtKtKdbNLCGAppEuNENBVpEvDYL6VHLWwofVJ9\nvjjDksLPTLfdbkUNdqi7qvlBSVpyKW4yKhil8nCR4bHnTbGua/3B83baxTnu3h8f2+3xQbJZn+zN\nFQjXQmvClYbpQpGF2bdVVWmlLN6z5yoLFbEuW3kit9WTiTRNLbth16ZjMhKo9BxjXkwFrOSpi+Vo\n22n1v8tzTocLVqLTuk0p5XL1jesLvQDjOsgx34fr4LQNPIosIiIiIvYEk0QWt27d+r9EFGhRFxER\nERGh8Ns3b97801iiSQtyRERERMQ/DlFkEREREbEniAtyRERExJ4gLsgRERERe4K4IEdERETsCeKC\nHBEREbEniAtyRERExJ4gLsgRERERe4K4IEdERETsCeKCHBEREbEn+H/dCYlWO2O6kwAAAABJRU5E\nrkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1393550f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "obs = env.reset()\n",
    "obs, _, _, _ = env.step(action=1)\n",
    "\n",
    "plt.figure(figsize=[6, 6])\n",
    "\n",
    "print(obs.shape)\n",
    "def show(observation):\n",
    "    plt.imshow(observation, interpolation=None)\n",
    "    #plt.imshow(observation.reshape((HEIGHT, WIDTH)), interpolation=None, cmap='gray')\n",
    "    plt.xticks([]); plt.yticks([]);\n",
    "    \n",
    "show(obs)\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Deep Q-learning\n",
    "\n",
    " * https://github.com/coreylynch/async-rl/blob/master/model.py and https://github.com/tflearn/tflearn/blob/master/examples/reinforcement_learning/atari_1step_qlearning.py with 84 x 84 images\n",
    "\n",
    "```\n",
    "    net = tflearn.conv_2d(net, 32, 8, strides=4, activation='relu')\n",
    "    net = tflearn.conv_2d(net, 64, 4, strides=2, activation='relu')\n",
    "    net = tflearn.fully_connected(net, 256, activation='relu')\n",
    "```\n",
    "\n",
    " * https://github.com/ebonyclock/deep_rl_vizdoom/blob/master/networks/common.py#L20 with 80 x 60 images\n",
    " \n",
    "```\n",
    "    conv1 = layers.conv2d(img_input, num_outputs=8, kernel_size=[6, 6], stride=3, padding=\"VALID\", ...)\n",
    "    conv2 = layers.conv2d(conv1, num_outputs=8, kernel_size=[3, 3], stride=2, padding=\"VALID\", ...)\n",
    "\n",
    "```\n",
    " * From http://vizdoom.cs.put.edu.pl/tutorial#learning with images (30, 45)!\n",
    "```\n",
    "dqn = Conv2DLayer(dqn, num_filters=8, filter_size=[6, 6], nonlinearity=rectify, W=HeUniform(\"relu\"),b=Constant(.1), stride=3)\n",
    "dqn = Conv2DLayer(dqn, num_filters=8, filter_size=[3, 3],nonlinearity=rectify, W=HeUniform(\"relu\"),b=Constant(.1), stride=2)\n",
    "dqn = DenseLayer(dqn, num_units=128, nonlinearity=rectify, W=HeUniform(\"relu\"), b=Constant(.1))\n",
    "```\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "convolution2d_345 (Convolution2D (None, 50, 67, 8)     872         convolution2d_input_173[0][0]    \n",
      "____________________________________________________________________________________________________\n",
      "activation_517 (Activation)      (None, 50, 67, 8)     0           convolution2d_345[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "convolution2d_346 (Convolution2D (None, 25, 34, 8)     584         activation_517[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "activation_518 (Activation)      (None, 25, 34, 8)     0           convolution2d_346[0][0]          \n",
      "____________________________________________________________________________________________________\n",
      "flatten_173 (Flatten)            (None, 6800)          0           activation_518[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dense_345 (Dense)                (None, 128)           870528      flatten_173[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "activation_519 (Activation)      (None, 128)           0           dense_345[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_346 (Dense)                (None, 4)             516         activation_519[0][0]             \n",
      "====================================================================================================\n",
      "Total params: 872,500\n",
      "Trainable params: 872,500\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "\n",
    "from keras.layers import Dense, Convolution2D, Flatten, Activation\n",
    "from keras.models import Sequential\n",
    "from keras.optimizers import Adam\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "K.set_session(sess)\n",
    "\n",
    "def create_q_model(conv1_weights=None, conv2_weights=None, dense1_weights=None, dense2_weights=None):\n",
    "    model = Sequential()\n",
    "\n",
    "    #model.add(Input(shape=(agent_history_length, resized_width, resized_height,)))\n",
    "    model.add(\n",
    "        Convolution2D(nb_filter=8, nb_row=6, nb_col=6, subsample=(3, 3), border_mode='same', weights=conv1_weights,\n",
    "            input_shape=[HEIGHT, WIDTH, CHANNELS], dim_ordering='tf'))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(\n",
    "        Convolution2D(nb_filter=8, nb_row=3, nb_col=3, subsample=(2, 2),  border_mode='same', weights=conv2_weights, dim_ordering='tf'))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(128, init='normal', weights=dense1_weights))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(4, init='normal', weights=dense2_weights))\n",
    "    model.compile(loss='mse', optimizer=Adam())\n",
    "    \n",
    "    return model\n",
    "\n",
    "acting_model = create_q_model()\n",
    "target_model = create_q_model()\n",
    "\n",
    "def copy_model(model):\n",
    "    conv1_weights =  [w.eval() for w in model.layers[0].weights]\n",
    "    conv2_weights = [w.eval() for w in model.layers[2].weights]\n",
    "    dense1_weights = [w.eval() for w in model.layers[5].weights]\n",
    "    dense2_weights = [w.eval() for w in model.layers[7].weights]\n",
    "    return create_q_model(conv1_weights, conv2_weights, dense1_weights, dense2_weights)\n",
    "\n",
    "acting_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collecting and preparing experiences for learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# from https://docs.python.org/3/library/itertools.html#recipes\n",
    "from itertools import tee, zip_longest\n",
    "\n",
    "def pairwise(iterable):\n",
    "    \"s -> (s0,s1), (s1,s2), (s2, s3), ...\"\n",
    "    a, b = tee(iterable)\n",
    "    next(b, None)\n",
    "    return zip(a, b)\n",
    "\n",
    "def grouper(iterable, n, fillvalue=None):\n",
    "    \"Collect data into fixed-length chunks or blocks\"\n",
    "    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx\n",
    "    args = [iter(iterable)] * n\n",
    "    return zip_longest(fillvalue=fillvalue, *args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <bound method Env.__del__ of <SkipWrapper<ToDiscreteWrapper<_Monitor<TimeLimit<DoomBasicEnv instance>>>>>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 252, in __del__\n",
      "    self.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 190, in close\n",
      "    self._close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 346, in _close\n",
      "    return self.env.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 190, in close\n",
      "    self._close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 346, in _close\n",
      "    return self.env.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 190, in close\n",
      "    self._close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/wrappers/monitoring.py\", line 38, in _close\n",
      "    self._monitor.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 193, in close\n",
      "    self._flush(force=True)\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 183, in _flush\n",
      "    'env_info': self._env_info(),\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 300, in _env_info\n",
      "    if self.env.spec:\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 98, in env\n",
      "    raise error.Error(\"env has been garbage collected. To keep using a monitor, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)\")\n",
      "gym.error.Error: env has been garbage collected. To keep using a monitor, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)\n",
      "Exception ignored in: <bound method Env.__del__ of <ToDiscreteWrapper<_Monitor<TimeLimit<DoomBasicEnv instance>>>>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 252, in __del__\n",
      "    self.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 190, in close\n",
      "    self._close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 346, in _close\n",
      "    return self.env.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 190, in close\n",
      "    self._close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/wrappers/monitoring.py\", line 38, in _close\n",
      "    self._monitor.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 193, in close\n",
      "    self._flush(force=True)\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 183, in _flush\n",
      "    'env_info': self._env_info(),\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 300, in _env_info\n",
      "    if self.env.spec:\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 98, in env\n",
      "    raise error.Error(\"env has been garbage collected. To keep using a monitor, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)\")\n",
      "gym.error.Error: env has been garbage collected. To keep using a monitor, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)\n",
      "Exception ignored in: <bound method Env.__del__ of <_Monitor<TimeLimit<DoomBasicEnv instance>>>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 252, in __del__\n",
      "    self.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/core.py\", line 190, in close\n",
      "    self._close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/wrappers/monitoring.py\", line 38, in _close\n",
      "    self._monitor.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 193, in close\n",
      "    self._flush(force=True)\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 183, in _flush\n",
      "    'env_info': self._env_info(),\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 300, in _env_info\n",
      "    if self.env.spec:\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 98, in env\n",
      "    raise error.Error(\"env has been garbage collected. To keep using a monitor, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)\")\n",
      "gym.error.Error: env has been garbage collected. To keep using a monitor, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)\n",
      "Exception ignored in: <bound method MonitorManager.__del__ of <gym.monitoring.monitor_manager.MonitorManager object at 0x10d9efb00>>\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 306, in __del__\n",
      "    self.close()\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 193, in close\n",
      "    self._flush(force=True)\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 183, in _flush\n",
      "    'env_info': self._env_info(),\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 300, in _env_info\n",
      "    if self.env.spec:\n",
      "  File \"/Users/gui/.virtualenvs/gui3/lib/python3.6/site-packages/gym/monitoring/monitor_manager.py\", line 98, in env\n",
      "    raise error.Error(\"env has been garbage collected. To keep using a monitor, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)\")\n",
      "gym.error.Error: env has been garbage collected. To keep using a monitor, you must keep around a reference to the env object. (HINT: try assigning the env to a variable in your code.)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "from collections import namedtuple, deque\n",
    "from itertools import islice \n",
    "import operator\n",
    "import functools\n",
    "import random\n",
    "\n",
    "\n",
    "SARE = namedtuple('SAR', ['state', 'action', 'reward', 'end'])\n",
    "Experience = namedtuple('Experience', ['previous_state', 'action', 'reward', 'next_state', 'end'])\n",
    "\n",
    "\n",
    "class EpsilonGreedyQAgent(object):\n",
    "    def __init__(self, model, epsilon=.1):\n",
    "        self.model = model\n",
    "        self.epsilon = epsilon\n",
    "\n",
    "    def act(self, observation, reward, done):\n",
    "        if random.uniform(0, 1) <= self.epsilon:\n",
    "            return random.choice([NOOP, SHOOT, LEFT, RIGHT])\n",
    "        else:\n",
    "            return self.model.predict(observation[np.newaxis])[0].argmax()\n",
    "\n",
    "\n",
    "def generate_sares(env, agent, episode_count=2000):\n",
    "    reward = 0\n",
    "    done = False\n",
    "\n",
    "    for i in range(episode_count):\n",
    "        if (i % 10) == 0:\n",
    "            print('episode {i}'.format(i=i))\n",
    "        observation = env.reset()\n",
    "        while True:\n",
    "            action = agent.act(observation, reward, done)\n",
    "            new_observation, reward, done, _ = env.step(action)\n",
    "            yield SARE(observation, action, reward, done)\n",
    "            \n",
    "            if done:\n",
    "                break\n",
    "            else:\n",
    "                observation = new_observation\n",
    "\n",
    "\n",
    "env = create_env()\n",
    "sares = generate_sares(env, EpsilonGreedyQAgent(acting_model), episode_count=1000)\n",
    "\n",
    "experiences = (\n",
    "    Experience(previous_s, a, r, next_s, end) \n",
    "    for (previous_s, a, r, end), (next_s, _, _, _) \n",
    "    in pairwise(sares))\n",
    "\n",
    "\n",
    "experience_batches = grouper(experiences, n=200)\n",
    "\n",
    "print(len(list(next(experience_batches))))\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 0\n",
      "32 32\n",
      "32 50\n"
     ]
    }
   ],
   "source": [
    "from itertools import islice\n",
    "\n",
    "class ExperiencesReplay:\n",
    "    def __init__(self, experiences_iterable, memory_size=50, batch_size=32):\n",
    "        self.experiences_iterable = experiences_iterable\n",
    "        self.batch_size = batch_size\n",
    "        self.memory = deque(maxlen=memory_size)\n",
    "\n",
    "    def __iter__(self):\n",
    "        return self\n",
    "\n",
    "    def __next__(self):\n",
    "        for exp in islice(self.experiences_iterable, self.batch_size):\n",
    "            self.memory.appendleft(exp)\n",
    "        sampled_experiences = random.choices(self.memory, k=self.batch_size)\n",
    "        return sampled_experiences\n",
    "    \n",
    "env = create_env()\n",
    "\n",
    "experiences = (\n",
    "    Experience(previous_s, a, r, next_s, end) \n",
    "    for (previous_s, a, r, end), (next_s, _, _, _) \n",
    "    in pairwise(generate_sares(env, EpsilonGreedyQAgent(acting_model))))\n",
    "\n",
    "replay = ExperiencesReplay(experiences)\n",
    "\n",
    "print(len(next(replay)), len(replay.memory))\n",
    "print(len(next(replay)), len(replay.memory))\n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:42:55,986] DEPRECATION WARNING: env.spec.timestep_limit has been deprecated. Replace your call to `env.spec.timestep_limit` with `env.spec.tags.get('wrapper_config.TimeLimit.max_episode_steps')`. This change was made 12/28/2016 and is included in version 0.7.0\n",
      "[2017-03-12 08:42:55,988] Clearing 42 monitor files from previous run (because force=True was provided)\n",
      "[2017-03-12 08:42:56,320] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000000.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 0\n",
      "Batch of 128 exps with avg reward -3.7265625 and dist. [125, 0, 3] and actions distribution [48, 40, 22, 18]\n",
      "Batch of 128 exps with avg reward -5.3125 and dist. [8, 0, 120] and actions distribution [117, 8, 1, 2]\n",
      "Batch of 128 exps with avg reward -5.078125 and dist. [2, 0, 126] and actions distribution [117, 2, 6, 3]\n",
      "Batch of 128 exps with avg reward -5.15625 and dist. [4, 0, 124] and actions distribution [116, 4, 6, 2]\n",
      "episode 10\n",
      "Batch of 128 exps with avg reward -5.078125 and dist. [2, 0, 126] and actions distribution [121, 2, 1, 4]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.25 and dist. [127, 0, 1] and actions distribution [119, 2, 2, 5]\n",
      "Batch of 128 exps with avg reward -5.078125 and dist. [2, 0, 126] and actions distribution [124, 2, 2]\n",
      "Batch of 128 exps with avg reward -5.0390625 and dist. [1, 0, 127] and actions distribution [121, 1, 2, 4]\n",
      "Batch of 128 exps with avg reward -5.0 and dist. [0, 128, 0] and actions distribution [124, 0, 0, 4]\n",
      "episode 20\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [116, 3, 4, 5]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [121, 3, 1, 3]\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [114, 3, 6, 5]\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [116, 3, 6, 3]\n",
      "Batch of 128 exps with avg reward -5.078125 and dist. [2, 0, 126] and actions distribution [114, 2, 5, 7]\n",
      "Batch of 128 exps with avg reward -5.15625 and dist. [4, 0, 124] and actions distribution [111, 4, 9, 4]\n",
      "target_model <- acting_model\n",
      "episode 30\n",
      "Batch of 128 exps with avg reward -5.0 and dist. [0, 128, 0] and actions distribution [121, 0, 4, 3]\n",
      "Batch of 128 exps with avg reward -4.3671875 and dist. [127, 0, 1] and actions distribution [113, 5, 7, 3]\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [90, 3, 20, 15]\n",
      "Batch of 128 exps with avg reward -4.40625 and dist. [127, 0, 1] and actions distribution [84, 6, 24, 14]\n",
      "episode 40\n",
      "Batch of 128 exps with avg reward -5.15625 and dist. [4, 0, 124] and actions distribution [77, 4, 42, 5]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.328125 and dist. [127, 0, 1] and actions distribution [100, 4, 20, 4]\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [75, 3, 38, 12]\n",
      "Batch of 128 exps with avg reward -5.0390625 and dist. [1, 0, 127] and actions distribution [26, 1, 87, 14]\n",
      "Batch of 128 exps with avg reward -5.3125 and dist. [8, 0, 120] and actions distribution [28, 8, 66, 26]\n",
      "episode 50\n",
      "Batch of 128 exps with avg reward -5.234375 and dist. [6, 0, 122] and actions distribution [38, 6, 51, 33]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.078125 and dist. [2, 0, 126] and actions distribution [38, 2, 46, 42]\n",
      "Batch of 128 exps with avg reward -4.2890625 and dist. [127, 0, 1] and actions distribution [54, 3, 41, 30]\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [57, 3, 25, 43]\n",
      "Batch of 128 exps with avg reward -5.1953125 and dist. [5, 0, 123] and actions distribution [34, 6, 54, 34]\n",
      "episode 60\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [46, 3, 43, 36]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.0 and dist. [0, 128, 0] and actions distribution [28, 0, 51, 49]\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [42, 3, 48, 35]\n",
      "Batch of 128 exps with avg reward -5.0 and dist. [0, 128, 0] and actions distribution [45, 0, 36, 47]\n",
      "Batch of 128 exps with avg reward -5.0390625 and dist. [1, 0, 127] and actions distribution [50, 1, 45, 32]\n",
      "Batch of 128 exps with avg reward -5.234375 and dist. [6, 0, 122] and actions distribution [33, 6, 44, 45]\n",
      "target_model <- acting_model\n",
      "episode 70\n",
      "Batch of 128 exps with avg reward -5.0390625 and dist. [1, 0, 127] and actions distribution [43, 1, 41, 43]\n",
      "Batch of 128 exps with avg reward -5.1171875 and dist. [3, 0, 125] and actions distribution [30, 4, 44, 50]\n",
      "Batch of 128 exps with avg reward -5.3125 and dist. [8, 0, 120] and actions distribution [41, 8, 42, 37]\n",
      "Batch of 128 exps with avg reward -5.3515625 and dist. [9, 0, 119] and actions distribution [25, 9, 45, 49]\n",
      "Batch of 128 exps with avg reward -4.484375 and dist. [127, 0, 1] and actions distribution [42, 8, 35, 43]\n",
      "target_model <- acting_model\n",
      "episode 80\n",
      "Batch of 128 exps with avg reward -5.15625 and dist. [4, 0, 124] and actions distribution [62, 4, 33, 29]\n",
      "Batch of 128 exps with avg reward -4.4453125 and dist. [127, 0, 1] and actions distribution [44, 7, 39, 38]\n",
      "Batch of 128 exps with avg reward -4.5625 and dist. [127, 0, 1] and actions distribution [33, 10, 46, 39]\n",
      "Batch of 128 exps with avg reward -5.703125 and dist. [18, 0, 110] and actions distribution [37, 29, 33, 29]\n",
      "episode 90\n",
      "Batch of 128 exps with avg reward -4.5234375 and dist. [127, 0, 1] and actions distribution [46, 10, 36, 36]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.3125 and dist. [8, 0, 120] and actions distribution [40, 8, 35, 45]\n",
      "Batch of 128 exps with avg reward -5.5078125 and dist. [13, 0, 115] and actions distribution [44, 16, 35, 33]\n",
      "Batch of 128 exps with avg reward -5.390625 and dist. [10, 0, 118] and actions distribution [37, 10, 47, 34]\n",
      "Batch of 128 exps with avg reward -3.9296875 and dist. [126, 0, 2] and actions distribution [25, 15, 43, 45]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:44:59,577] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000100.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 100\n",
      "Batch of 128 exps with avg reward -4.1171875 and dist. [126, 0, 2] and actions distribution [36, 21, 29, 42]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.71875 and dist. [127, 0, 1] and actions distribution [42, 15, 28, 43]\n",
      "Batch of 128 exps with avg reward -5.390625 and dist. [10, 0, 118] and actions distribution [34, 11, 30, 53]\n",
      "episode 110\n",
      "Batch of 128 exps with avg reward -4.046875 and dist. [126, 0, 2] and actions distribution [33, 21, 29, 45]\n",
      "Batch of 128 exps with avg reward -4.7578125 and dist. [127, 0, 1] and actions distribution [42, 20, 40, 26]\n",
      "Batch of 128 exps with avg reward -3.171875 and dist. [125, 0, 3] and actions distribution [33, 19, 44, 32]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.03125 and dist. [127, 0, 1] and actions distribution [37, 23, 32, 36]\n",
      "episode 120\n",
      "Batch of 128 exps with avg reward -4.46875 and dist. [126, 0, 2] and actions distribution [23, 36, 29, 40]\n",
      "Batch of 128 exps with avg reward -4.28125 and dist. [126, 0, 2] and actions distribution [27, 28, 35, 38]\n",
      "Batch of 128 exps with avg reward -5.78125 and dist. [20, 0, 108] and actions distribution [36, 23, 30, 39]\n",
      "episode 130\n",
      "Batch of 128 exps with avg reward -5.03125 and dist. [127, 0, 1] and actions distribution [31, 25, 36, 36]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.15625 and dist. [126, 0, 2] and actions distribution [49, 22, 23, 34]\n",
      "Batch of 128 exps with avg reward -4.875 and dist. [127, 0, 1] and actions distribution [25, 20, 37, 46]\n",
      "Batch of 128 exps with avg reward -3.5234375 and dist. [125, 0, 3] and actions distribution [15, 29, 43, 41]\n",
      "episode 140\n",
      "Batch of 128 exps with avg reward -5.1484375 and dist. [127, 0, 1] and actions distribution [22, 33, 37, 36]\n",
      "Batch of 128 exps with avg reward -3.4375 and dist. [125, 0, 3] and actions distribution [26, 27, 40, 35]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.8359375 and dist. [127, 0, 1] and actions distribution [39, 19, 35, 35]\n",
      "episode 150\n",
      "Batch of 128 exps with avg reward -5.78125 and dist. [20, 0, 108] and actions distribution [49, 20, 29, 30]\n",
      "Batch of 128 exps with avg reward -4.046875 and dist. [126, 0, 2] and actions distribution [46, 21, 33, 28]\n",
      "Batch of 128 exps with avg reward -4.125 and dist. [126, 0, 2] and actions distribution [42, 24, 32, 30]\n",
      "episode 160\n",
      "Batch of 128 exps with avg reward -0.9609375 and dist. [122, 0, 6] and actions distribution [26, 29, 36, 37]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.4140625 and dist. [125, 0, 3] and actions distribution [30, 30, 30, 38]\n",
      "episode 170\n",
      "Batch of 128 exps with avg reward -3.5625 and dist. [125, 0, 3] and actions distribution [32, 34, 30, 32]\n",
      "Batch of 128 exps with avg reward -3.859375 and dist. [125, 0, 3] and actions distribution [25, 60, 25, 18]\n",
      "Batch of 128 exps with avg reward -5.34375 and dist. [127, 0, 1] and actions distribution [29, 37, 35, 27]\n",
      "episode 180\n",
      "Batch of 128 exps with avg reward -1.875 and dist. [123, 0, 5] and actions distribution [35, 30, 36, 27]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.390625 and dist. [126, 0, 2] and actions distribution [23, 30, 36, 39]\n",
      "episode 190\n",
      "Batch of 128 exps with avg reward -4.875 and dist. [127, 0, 1] and actions distribution [27, 18, 35, 48]\n",
      "Batch of 128 exps with avg reward -4.9140625 and dist. [127, 0, 1] and actions distribution [40, 23, 27, 38]\n",
      "Batch of 128 exps with avg reward -5.7421875 and dist. [19, 0, 109] and actions distribution [35, 21, 32, 40]\n",
      "Batch of 128 exps with avg reward -3.4140625 and dist. [125, 0, 3] and actions distribution [24, 26, 33, 45]\n",
      "target_model <- acting_model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:46:34,827] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000200.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 200\n",
      "Batch of 128 exps with avg reward -5.3046875 and dist. [127, 0, 1] and actions distribution [19, 32, 32, 45]\n",
      "Batch of 128 exps with avg reward -5.15625 and dist. [127, 0, 1] and actions distribution [27, 32, 33, 36]\n",
      "Batch of 128 exps with avg reward -5.0703125 and dist. [127, 0, 1] and actions distribution [36, 32, 36, 24]\n",
      "episode 210\n",
      "Batch of 128 exps with avg reward -2.546875 and dist. [124, 0, 4] and actions distribution [41, 26, 38, 23]\n",
      "Batch of 128 exps with avg reward -5.1875 and dist. [127, 0, 1] and actions distribution [33, 36, 27, 32]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.46875 and dist. [126, 0, 2] and actions distribution [30, 39, 25, 34]\n",
      "Batch of 128 exps with avg reward -5.0546875 and dist. [127, 0, 1] and actions distribution [29, 28, 25, 46]\n",
      "episode 220\n",
      "Batch of 128 exps with avg reward -4.15625 and dist. [126, 0, 2] and actions distribution [26, 23, 37, 42]\n",
      "Batch of 128 exps with avg reward -3.3359375 and dist. [125, 0, 3] and actions distribution [25, 24, 28, 51]\n",
      "Batch of 128 exps with avg reward -5.03125 and dist. [127, 0, 1] and actions distribution [14, 32, 35, 47]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.8203125 and dist. [21, 0, 107] and actions distribution [21, 26, 35, 46]\n",
      "episode 230\n",
      "Batch of 128 exps with avg reward -6.1328125 and dist. [29, 0, 99] and actions distribution [33, 41, 25, 29]\n",
      "Batch of 128 exps with avg reward -6.171875 and dist. [30, 0, 98] and actions distribution [28, 46, 25, 29]\n",
      "Batch of 128 exps with avg reward -4.359375 and dist. [126, 0, 2] and actions distribution [39, 33, 33, 23]\n",
      "Batch of 128 exps with avg reward -5.703125 and dist. [18, 0, 110] and actions distribution [35, 23, 30, 40]\n",
      "target_model <- acting_model\n",
      "episode 240\n",
      "Batch of 128 exps with avg reward -4.8359375 and dist. [127, 0, 1] and actions distribution [49, 20, 27, 32]\n",
      "Batch of 128 exps with avg reward -4.9140625 and dist. [127, 0, 1] and actions distribution [35, 21, 27, 45]\n",
      "Batch of 128 exps with avg reward -2.5390625 and dist. [124, 0, 4] and actions distribution [26, 29, 35, 38]\n",
      "episode 250\n",
      "Batch of 128 exps with avg reward -5.265625 and dist. [127, 0, 1] and actions distribution [31, 32, 28, 37]\n",
      "Batch of 128 exps with avg reward -4.3203125 and dist. [126, 0, 2] and actions distribution [23, 33, 32, 40]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.9375 and dist. [127, 0, 1] and actions distribution [41, 23, 30, 34]\n",
      "episode 260\n",
      "Batch of 128 exps with avg reward -1.484375 and dist. [123, 0, 5] and actions distribution [36, 20, 31, 41]\n",
      "Batch of 128 exps with avg reward -5.8203125 and dist. [21, 0, 107] and actions distribution [27, 26, 38, 37]\n",
      "Batch of 128 exps with avg reward -3.53125 and dist. [125, 0, 3] and actions distribution [37, 40, 18, 33]\n",
      "Batch of 128 exps with avg reward -5.9765625 and dist. [25, 0, 103] and actions distribution [24, 35, 27, 42]\n",
      "target_model <- acting_model\n",
      "episode 270\n",
      "Batch of 128 exps with avg reward -5.03125 and dist. [127, 0, 1] and actions distribution [29, 30, 32, 37]\n",
      "Batch of 128 exps with avg reward -4.1640625 and dist. [126, 0, 2] and actions distribution [30, 35, 29, 34]\n",
      "Batch of 128 exps with avg reward -3.4453125 and dist. [125, 0, 3] and actions distribution [24, 27, 46, 31]\n",
      "episode 280\n",
      "Batch of 128 exps with avg reward -3.515625 and dist. [125, 0, 3] and actions distribution [25, 32, 37, 34]\n",
      "Batch of 128 exps with avg reward -4.1640625 and dist. [126, 0, 2] and actions distribution [57, 34, 23, 14]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.9765625 and dist. [25, 0, 103] and actions distribution [19, 41, 8, 60]\n",
      "Batch of 128 exps with avg reward -5.03125 and dist. [127, 0, 1] and actions distribution [25, 31, 38, 34]\n",
      "episode 290\n",
      "Batch of 128 exps with avg reward -2.546875 and dist. [124, 0, 4] and actions distribution [31, 32, 40, 25]\n",
      "Batch of 128 exps with avg reward -4.125 and dist. [126, 0, 2] and actions distribution [26, 22, 31, 49]\n",
      "Batch of 128 exps with avg reward -5.2578125 and dist. [127, 0, 1] and actions distribution [24, 43, 24, 37]\n",
      "target_model <- acting_model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:48:22,095] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000300.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 300\n",
      "Batch of 128 exps with avg reward -5.2265625 and dist. [127, 0, 1] and actions distribution [34, 38, 22, 34]\n",
      "Batch of 128 exps with avg reward -5.0234375 and dist. [127, 0, 1] and actions distribution [26, 29, 29, 44]\n",
      "Batch of 128 exps with avg reward -2.6484375 and dist. [124, 0, 4] and actions distribution [37, 33, 23, 35]\n",
      "episode 310\n",
      "Batch of 128 exps with avg reward -3.5703125 and dist. [125, 0, 3] and actions distribution [35, 35, 32, 26]\n",
      "Batch of 128 exps with avg reward -5.703125 and dist. [18, 0, 110] and actions distribution [65, 22, 15, 26]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.6171875 and dist. [125, 0, 3] and actions distribution [38, 38, 23, 29]\n",
      "episode 320\n",
      "Batch of 128 exps with avg reward -5.3828125 and dist. [127, 0, 1] and actions distribution [24, 47, 26, 31]\n",
      "Batch of 128 exps with avg reward -6.0546875 and dist. [27, 0, 101] and actions distribution [15, 43, 34, 36]\n",
      "Batch of 128 exps with avg reward -5.09375 and dist. [127, 0, 1] and actions distribution [30, 31, 31, 36]\n",
      "Batch of 128 exps with avg reward -4.875 and dist. [127, 0, 1] and actions distribution [22, 22, 38, 46]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.546875 and dist. [14, 0, 114] and actions distribution [24, 16, 38, 50]\n",
      "episode 330\n",
      "Batch of 128 exps with avg reward -5.6640625 and dist. [17, 0, 111] and actions distribution [32, 20, 33, 43]\n",
      "Batch of 128 exps with avg reward -2.71875 and dist. [124, 0, 4] and actions distribution [28, 32, 31, 37]\n",
      "Batch of 128 exps with avg reward -3.9609375 and dist. [126, 0, 2] and actions distribution [45, 19, 29, 35]\n",
      "episode 340\n",
      "Batch of 128 exps with avg reward -4.0390625 and dist. [126, 0, 2] and actions distribution [21, 21, 21, 65]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.9140625 and dist. [127, 0, 1] and actions distribution [30, 28, 39, 31]\n",
      "episode 350\n",
      "Batch of 128 exps with avg reward -3.3671875 and dist. [125, 0, 3] and actions distribution [23, 30, 42, 33]\n",
      "Batch of 128 exps with avg reward -2.3046875 and dist. [124, 0, 4] and actions distribution [29, 19, 46, 34]\n",
      "Batch of 128 exps with avg reward -5.390625 and dist. [127, 0, 1] and actions distribution [29, 49, 22, 28]\n",
      "episode 360\n",
      "Batch of 128 exps with avg reward -4.2734375 and dist. [126, 0, 2] and actions distribution [35, 30, 22, 41]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -2.421875 and dist. [124, 0, 4] and actions distribution [36, 23, 36, 33]\n",
      "Batch of 128 exps with avg reward -4.8359375 and dist. [127, 0, 1] and actions distribution [32, 21, 41, 34]\n",
      "episode 370\n",
      "Batch of 128 exps with avg reward -5.5859375 and dist. [15, 0, 113] and actions distribution [37, 21, 34, 36]\n",
      "Batch of 128 exps with avg reward -4.34375 and dist. [126, 0, 2] and actions distribution [22, 36, 38, 32]\n",
      "Batch of 128 exps with avg reward -4.8359375 and dist. [127, 0, 1] and actions distribution [35, 21, 31, 41]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.5234375 and dist. [125, 0, 3] and actions distribution [25, 26, 34, 43]\n",
      "episode 380\n",
      "Batch of 128 exps with avg reward -1.0078125 and dist. [122, 0, 6] and actions distribution [22, 27, 29, 50]\n",
      "episode 390\n",
      "Batch of 128 exps with avg reward -1.953125 and dist. [123, 0, 5] and actions distribution [32, 38, 32, 26]\n",
      "Batch of 128 exps with avg reward -4.6953125 and dist. [126, 0, 2] and actions distribution [39, 63, 13, 13]\n",
      "Batch of 128 exps with avg reward -3.46875 and dist. [125, 0, 3] and actions distribution [24, 38, 42, 24]\n",
      "target_model <- acting_model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:49:55,603] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000400.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 400\n",
      "Batch of 128 exps with avg reward -1.0703125 and dist. [122, 0, 6] and actions distribution [26, 30, 35, 37]\n",
      "episode 410\n",
      "Batch of 128 exps with avg reward -1.484375 and dist. [123, 0, 5] and actions distribution [40, 17, 34, 37]\n",
      "Batch of 128 exps with avg reward -3.5234375 and dist. [125, 0, 3] and actions distribution [21, 34, 32, 41]\n",
      "Batch of 128 exps with avg reward -6.2109375 and dist. [31, 0, 97] and actions distribution [18, 40, 31, 39]\n",
      "Batch of 128 exps with avg reward -5.9765625 and dist. [25, 0, 103] and actions distribution [21, 31, 37, 39]\n",
      "target_model <- acting_model\n",
      "episode 420\n",
      "Batch of 128 exps with avg reward -3.9296875 and dist. [126, 0, 2] and actions distribution [27, 19, 37, 45]\n",
      "Batch of 128 exps with avg reward -3.8515625 and dist. [126, 0, 2] and actions distribution [43, 13, 45, 27]\n",
      "Batch of 128 exps with avg reward -4.71875 and dist. [127, 0, 1] and actions distribution [44, 16, 42, 26]\n",
      "episode 430\n",
      "Batch of 128 exps with avg reward -5.9375 and dist. [24, 0, 104] and actions distribution [36, 27, 32, 33]\n",
      "Batch of 128 exps with avg reward -2.6953125 and dist. [124, 0, 4] and actions distribution [20, 31, 37, 40]\n",
      "target_model <- acting_model\n",
      "episode 440\n",
      "Batch of 128 exps with avg reward -2.6171875 and dist. [124, 0, 4] and actions distribution [21, 29, 34, 44]\n",
      "Batch of 128 exps with avg reward -2.046875 and dist. [123, 0, 5] and actions distribution [27, 42, 28, 31]\n",
      "Batch of 128 exps with avg reward -6.1328125 and dist. [29, 0, 99] and actions distribution [23, 52, 35, 18]\n",
      "Batch of 128 exps with avg reward -5.9375 and dist. [24, 0, 104] and actions distribution [29, 30, 42, 27]\n",
      "episode 450\n",
      "Batch of 128 exps with avg reward -2.625 and dist. [124, 0, 4] and actions distribution [29, 31, 34, 34]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -2.1953125 and dist. [124, 0, 4] and actions distribution [43, 14, 29, 42]\n",
      "episode 460\n",
      "Batch of 128 exps with avg reward -4.3046875 and dist. [126, 0, 2] and actions distribution [35, 34, 28, 31]\n",
      "Batch of 128 exps with avg reward -5.2578125 and dist. [127, 0, 1] and actions distribution [25, 42, 21, 40]\n",
      "Batch of 128 exps with avg reward -5.78125 and dist. [20, 0, 108] and actions distribution [29, 26, 42, 31]\n",
      "episode 470\n",
      "Batch of 128 exps with avg reward -2.65625 and dist. [124, 0, 4] and actions distribution [27, 28, 41, 32]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.7109375 and dist. [127, 0, 1] and actions distribution [32, 15, 38, 43]\n",
      "Batch of 128 exps with avg reward -4.6015625 and dist. [127, 0, 1] and actions distribution [68, 13, 13, 34]\n",
      "episode 480\n",
      "Batch of 128 exps with avg reward -2.453125 and dist. [124, 0, 4] and actions distribution [22, 20, 35, 51]\n",
      "Batch of 128 exps with avg reward -1.8671875 and dist. [123, 0, 5] and actions distribution [21, 28, 35, 44]\n",
      "Batch of 128 exps with avg reward -5.9765625 and dist. [25, 0, 103] and actions distribution [26, 36, 36, 30]\n",
      "target_model <- acting_model\n",
      "episode 490\n",
      "Batch of 128 exps with avg reward -2.65625 and dist. [124, 0, 4] and actions distribution [38, 30, 32, 28]\n",
      "Batch of 128 exps with avg reward -5.2265625 and dist. [127, 0, 1] and actions distribution [31, 40, 36, 21]\n",
      "Batch of 128 exps with avg reward -5.2265625 and dist. [127, 0, 1] and actions distribution [29, 38, 29, 32]\n",
      "Batch of 128 exps with avg reward -4.8984375 and dist. [127, 0, 1] and actions distribution [43, 29, 33, 23]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:51:21,209] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000500.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 500\n",
      "Batch of 128 exps with avg reward -4.8359375 and dist. [127, 0, 1] and actions distribution [31, 19, 35, 43]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.1640625 and dist. [126, 0, 2] and actions distribution [38, 25, 31, 34]\n",
      "Batch of 128 exps with avg reward -3.515625 and dist. [125, 0, 3] and actions distribution [31, 35, 29, 33]\n",
      "episode 510\n",
      "Batch of 128 exps with avg reward -0.34375 and dist. [121, 0, 7] and actions distribution [22, 36, 44, 26]\n",
      "Batch of 128 exps with avg reward -5.1484375 and dist. [127, 0, 1] and actions distribution [24, 42, 49, 13]\n",
      "episode 520\n",
      "Batch of 128 exps with avg reward -5.1484375 and dist. [127, 0, 1] and actions distribution [26, 39, 34, 29]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.78125 and dist. [20, 0, 108] and actions distribution [31, 30, 30, 37]\n",
      "Batch of 128 exps with avg reward -3.0625 and dist. [125, 0, 3] and actions distribution [39, 17, 32, 40]\n",
      "episode 530\n",
      "Batch of 128 exps with avg reward -3.09375 and dist. [125, 0, 3] and actions distribution [20, 18, 45, 45]\n",
      "Batch of 128 exps with avg reward -4.984375 and dist. [127, 0, 1] and actions distribution [30, 28, 27, 43]\n",
      "Batch of 128 exps with avg reward -4.3046875 and dist. [126, 0, 2] and actions distribution [18, 34, 31, 45]\n",
      "target_model <- acting_model\n",
      "episode 540\n",
      "Batch of 128 exps with avg reward -5.3046875 and dist. [127, 0, 1] and actions distribution [40, 44, 21, 23]\n",
      "Batch of 128 exps with avg reward -3.6875 and dist. [125, 0, 3] and actions distribution [40, 42, 31, 15]\n",
      "Batch of 128 exps with avg reward -4.234375 and dist. [126, 0, 2] and actions distribution [44, 38, 28, 18]\n",
      "episode 550\n",
      "Batch of 128 exps with avg reward -2.6640625 and dist. [124, 0, 4] and actions distribution [32, 33, 32, 31]\n",
      "episode 560\n",
      "Batch of 128 exps with avg reward 0.2265625 and dist. [120, 0, 8] and actions distribution [14, 56, 19, 39]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.578125 and dist. [126, 0, 2] and actions distribution [19, 54, 18, 37]\n",
      "Batch of 128 exps with avg reward -5.296875 and dist. [127, 0, 1] and actions distribution [23, 45, 25, 35]\n",
      "episode 570\n",
      "Batch of 128 exps with avg reward -5.0703125 and dist. [127, 0, 1] and actions distribution [24, 28, 34, 42]\n",
      "Batch of 128 exps with avg reward -4.2109375 and dist. [126, 0, 2] and actions distribution [27, 29, 37, 35]\n",
      "Batch of 128 exps with avg reward -4.953125 and dist. [127, 0, 1] and actions distribution [33, 24, 31, 40]\n",
      "target_model <- acting_model\n",
      "episode 580\n",
      "Batch of 128 exps with avg reward 1.9921875 and dist. [118, 0, 10] and actions distribution [21, 51, 15, 41]\n",
      "episode 590\n",
      "Batch of 128 exps with avg reward -2.984375 and dist. [124, 0, 4] and actions distribution [24, 56, 17, 31]\n",
      "Batch of 128 exps with avg reward -5.0234375 and dist. [127, 0, 1] and actions distribution [79, 34, 10, 5]\n",
      "Batch of 128 exps with avg reward -4.5625 and dist. [126, 0, 2] and actions distribution [20, 54, 29, 25]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:52:41,626] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000600.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 600\n",
      "Batch of 128 exps with avg reward -1.9453125 and dist. [123, 0, 5] and actions distribution [18, 34, 35, 41]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.796875 and dist. [127, 0, 1] and actions distribution [24, 22, 38, 44]\n",
      "Batch of 128 exps with avg reward -4.71875 and dist. [127, 0, 1] and actions distribution [23, 14, 37, 54]\n",
      "episode 610\n",
      "Batch of 128 exps with avg reward -4.03125 and dist. [126, 0, 2] and actions distribution [28, 23, 32, 45]\n",
      "Batch of 128 exps with avg reward -5.5078125 and dist. [13, 0, 115] and actions distribution [26, 15, 40, 47]\n",
      "Batch of 128 exps with avg reward -4.0234375 and dist. [126, 0, 2] and actions distribution [24, 24, 43, 37]\n",
      "target_model <- acting_model\n",
      "episode 620\n",
      "Batch of 128 exps with avg reward -2.578125 and dist. [124, 0, 4] and actions distribution [48, 25, 32, 23]\n",
      "Batch of 128 exps with avg reward -5.8984375 and dist. [23, 0, 105] and actions distribution [28, 32, 34, 34]\n",
      "episode 630\n",
      "Batch of 128 exps with avg reward -1.3828125 and dist. [122, 0, 6] and actions distribution [31, 57, 26, 14]\n",
      "Batch of 128 exps with avg reward -3.625 and dist. [125, 0, 3] and actions distribution [23, 45, 35, 25]\n",
      "Batch of 128 exps with avg reward -5.2265625 and dist. [127, 0, 1] and actions distribution [40, 36, 28, 24]\n",
      "target_model <- acting_model\n",
      "episode 640\n",
      "Batch of 128 exps with avg reward -4.15625 and dist. [126, 0, 2] and actions distribution [26, 27, 37, 38]\n",
      "Batch of 128 exps with avg reward -4.5546875 and dist. [127, 0, 1] and actions distribution [37, 11, 38, 42]\n",
      "Batch of 128 exps with avg reward -4.5234375 and dist. [127, 0, 1] and actions distribution [20, 11, 44, 53]\n",
      "episode 650\n",
      "Batch of 128 exps with avg reward -3.296875 and dist. [125, 0, 3] and actions distribution [34, 21, 25, 48]\n",
      "Batch of 128 exps with avg reward -5.1484375 and dist. [127, 0, 1] and actions distribution [42, 30, 26, 30]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.7421875 and dist. [19, 0, 109] and actions distribution [33, 22, 36, 37]\n",
      "Batch of 128 exps with avg reward -3.796875 and dist. [125, 0, 3] and actions distribution [18, 63, 22, 25]\n",
      "episode 660\n",
      "Batch of 128 exps with avg reward -6.09375 and dist. [28, 0, 100] and actions distribution [27, 40, 31, 30]\n",
      "Batch of 128 exps with avg reward -4.1640625 and dist. [126, 0, 2] and actions distribution [37, 25, 33, 33]\n",
      "Batch of 128 exps with avg reward -2.6171875 and dist. [124, 0, 4] and actions distribution [25, 38, 30, 35]\n",
      "target_model <- acting_model\n",
      "episode 670\n",
      "Batch of 128 exps with avg reward -3.4765625 and dist. [125, 0, 3] and actions distribution [29, 32, 28, 39]\n",
      "Batch of 128 exps with avg reward -2.609375 and dist. [124, 0, 4] and actions distribution [25, 28, 36, 39]\n",
      "Batch of 128 exps with avg reward -6.09375 and dist. [28, 0, 100] and actions distribution [27, 40, 25, 36]\n",
      "Batch of 128 exps with avg reward -4.2734375 and dist. [126, 0, 2] and actions distribution [18, 37, 35, 38]\n",
      "episode 680\n",
      "Batch of 128 exps with avg reward -3.2578125 and dist. [125, 0, 3] and actions distribution [38, 27, 32, 31]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.234375 and dist. [125, 0, 3] and actions distribution [32, 22, 45, 29]\n",
      "episode 690\n",
      "Batch of 128 exps with avg reward -0.765625 and dist. [122, 0, 6] and actions distribution [42, 25, 31, 30]\n",
      "Batch of 128 exps with avg reward -2.9375 and dist. [124, 0, 4] and actions distribution [26, 54, 30, 18]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:54:17,831] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000700.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 700\n",
      "Batch of 128 exps with avg reward -4.4296875 and dist. [126, 0, 2] and actions distribution [26, 44, 34, 24]\n",
      "episode 710\n",
      "Batch of 128 exps with avg reward 0.453125 and dist. [120, 0, 8] and actions distribution [16, 46, 39, 27]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.8984375 and dist. [23, 0, 105] and actions distribution [30, 26, 38, 34]\n",
      "Batch of 128 exps with avg reward -4.421875 and dist. [126, 0, 2] and actions distribution [20, 38, 40, 30]\n",
      "Batch of 128 exps with avg reward -3.1328125 and dist. [125, 0, 3] and actions distribution [47, 17, 38, 26]\n",
      "episode 720\n",
      "Batch of 128 exps with avg reward -2.6328125 and dist. [124, 0, 4] and actions distribution [31, 34, 31, 32]\n",
      "Batch of 128 exps with avg reward -5.9765625 and dist. [25, 0, 103] and actions distribution [34, 28, 20, 46]\n",
      "target_model <- acting_model\n",
      "episode 730\n",
      "Batch of 128 exps with avg reward -3.609375 and dist. [125, 0, 3] and actions distribution [13, 38, 21, 56]\n",
      "Batch of 128 exps with avg reward -3.515625 and dist. [125, 0, 3] and actions distribution [30, 35, 38, 25]\n",
      "Batch of 128 exps with avg reward -2.8671875 and dist. [124, 0, 4] and actions distribution [15, 58, 32, 23]\n",
      "episode 740\n",
      "Batch of 128 exps with avg reward -1.3203125 and dist. [122, 0, 6] and actions distribution [14, 50, 40, 24]\n",
      "Batch of 128 exps with avg reward -3.4765625 and dist. [125, 0, 3] and actions distribution [18, 36, 36, 38]\n",
      "target_model <- acting_model\n",
      "episode 750\n",
      "Batch of 128 exps with avg reward -5.625 and dist. [16, 0, 112] and actions distribution [44, 19, 27, 38]\n",
      "Batch of 128 exps with avg reward -4.2265625 and dist. [126, 0, 2] and actions distribution [45, 32, 20, 31]\n",
      "Batch of 128 exps with avg reward -4.9375 and dist. [127, 0, 1] and actions distribution [55, 29, 18, 26]\n",
      "episode 760\n",
      "Batch of 128 exps with avg reward -3.453125 and dist. [125, 0, 3] and actions distribution [42, 41, 23, 22]\n",
      "Batch of 128 exps with avg reward -4.9453125 and dist. [127, 0, 1] and actions distribution [28, 29, 38, 33]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.125 and dist. [125, 0, 3] and actions distribution [32, 24, 39, 33]\n",
      "episode 770\n",
      "Batch of 128 exps with avg reward -3.484375 and dist. [125, 0, 3] and actions distribution [18, 36, 43, 31]\n",
      "Batch of 128 exps with avg reward -1.890625 and dist. [123, 0, 5] and actions distribution [30, 42, 31, 25]\n",
      "Batch of 128 exps with avg reward -5.3203125 and dist. [127, 0, 1] and actions distribution [25, 49, 30, 24]\n",
      "episode 780\n",
      "Batch of 128 exps with avg reward -3.2890625 and dist. [125, 0, 3] and actions distribution [52, 29, 27, 20]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.1875 and dist. [126, 0, 2] and actions distribution [31, 30, 39, 28]\n",
      "Batch of 128 exps with avg reward -5.546875 and dist. [14, 0, 114] and actions distribution [34, 17, 34, 43]\n",
      "Batch of 128 exps with avg reward -5.46875 and dist. [12, 0, 116] and actions distribution [30, 15, 43, 40]\n",
      "episode 790\n",
      "Batch of 128 exps with avg reward -3.2890625 and dist. [125, 0, 3] and actions distribution [29, 28, 40, 31]\n",
      "Batch of 128 exps with avg reward -3.4921875 and dist. [125, 0, 3] and actions distribution [27, 34, 31, 36]\n",
      "target_model <- acting_model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:55:52,776] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000800.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 800\n",
      "Batch of 128 exps with avg reward -1.84375 and dist. [123, 0, 5] and actions distribution [27, 33, 28, 40]\n",
      "Batch of 128 exps with avg reward -2.84375 and dist. [124, 0, 4] and actions distribution [25, 45, 27, 31]\n",
      "episode 810\n",
      "Batch of 128 exps with avg reward -2.1484375 and dist. [123, 0, 5] and actions distribution [27, 59, 24, 18]\n",
      "Batch of 128 exps with avg reward -2.6953125 and dist. [124, 0, 4] and actions distribution [42, 35, 31, 20]\n",
      "episode 820\n",
      "Batch of 128 exps with avg reward -2.5625 and dist. [124, 0, 4] and actions distribution [39, 30, 37, 22]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.875 and dist. [127, 0, 1] and actions distribution [32, 21, 35, 40]\n",
      "Batch of 128 exps with avg reward -3.3671875 and dist. [125, 0, 3] and actions distribution [24, 27, 36, 41]\n",
      "episode 830\n",
      "Batch of 128 exps with avg reward -2.7265625 and dist. [124, 0, 4] and actions distribution [18, 42, 34, 34]\n",
      "Batch of 128 exps with avg reward -0.9921875 and dist. [122, 0, 6] and actions distribution [21, 38, 31, 38]\n",
      "episode 840\n",
      "Batch of 128 exps with avg reward -2.7734375 and dist. [124, 0, 4] and actions distribution [27, 44, 29, 28]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.140625 and dist. [127, 0, 1] and actions distribution [47, 29, 32, 20]\n",
      "Batch of 128 exps with avg reward -5.0625 and dist. [127, 0, 1] and actions distribution [43, 30, 31, 24]\n",
      "episode 850\n",
      "Batch of 128 exps with avg reward -3.90625 and dist. [126, 0, 2] and actions distribution [47, 21, 42, 18]\n",
      "Batch of 128 exps with avg reward -4.140625 and dist. [126, 0, 2] and actions distribution [34, 29, 36, 29]\n",
      "Batch of 128 exps with avg reward -3.4453125 and dist. [125, 0, 3] and actions distribution [26, 41, 20, 41]\n",
      "target_model <- acting_model\n",
      "episode 860\n",
      "Batch of 128 exps with avg reward -3.6875 and dist. [125, 0, 3] and actions distribution [13, 46, 29, 40]\n",
      "Batch of 128 exps with avg reward -5.0703125 and dist. [127, 0, 1] and actions distribution [27, 31, 33, 37]\n",
      "Batch of 128 exps with avg reward -5.8984375 and dist. [23, 0, 105] and actions distribution [32, 36, 31, 29]\n",
      "episode 870\n",
      "Batch of 128 exps with avg reward -4.03125 and dist. [126, 0, 2] and actions distribution [49, 25, 31, 23]\n",
      "Batch of 128 exps with avg reward -2.7890625 and dist. [124, 0, 4] and actions distribution [22, 44, 31, 31]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.546875 and dist. [126, 0, 2] and actions distribution [10, 51, 29, 38]\n",
      "episode 880\n",
      "Batch of 128 exps with avg reward -2.7109375 and dist. [124, 0, 4] and actions distribution [18, 47, 18, 45]\n",
      "Batch of 128 exps with avg reward -3.65625 and dist. [125, 0, 3] and actions distribution [13, 53, 21, 41]\n",
      "episode 890\n",
      "Batch of 128 exps with avg reward -4.8359375 and dist. [127, 0, 1] and actions distribution [23, 20, 32, 53]\n",
      "Batch of 128 exps with avg reward -3.28125 and dist. [125, 0, 3] and actions distribution [26, 27, 30, 45]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.6640625 and dist. [125, 0, 3] and actions distribution [18, 44, 25, 41]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:57:18,827] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video000900.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 900\n",
      "Batch of 128 exps with avg reward -0.515625 and dist. [121, 0, 7] and actions distribution [43, 55, 13, 17]\n",
      "Batch of 128 exps with avg reward -6.3671875 and dist. [35, 0, 93] and actions distribution [27, 83, 11, 7]\n",
      "episode 910\n",
      "Batch of 128 exps with avg reward -0.4765625 and dist. [121, 0, 7] and actions distribution [26, 63, 29, 10]\n",
      "episode 920\n",
      "Batch of 128 exps with avg reward -2.359375 and dist. [124, 0, 4] and actions distribution [35, 26, 36, 31]\n",
      "target_model <- acting_model\n",
      "episode 930\n",
      "Batch of 128 exps with avg reward 2.15625 and dist. [118, 0, 10] and actions distribution [24, 46, 21, 37]\n",
      "Batch of 128 exps with avg reward -2.1875 and dist. [123, 0, 5] and actions distribution [14, 62, 16, 36]\n",
      "episode 940\n",
      "Batch of 128 exps with avg reward -4.5390625 and dist. [126, 0, 2] and actions distribution [16, 45, 25, 42]\n",
      "Batch of 128 exps with avg reward -3.359375 and dist. [125, 0, 3] and actions distribution [21, 27, 38, 42]\n",
      "episode 950\n",
      "Batch of 128 exps with avg reward -3.3671875 and dist. [125, 0, 3] and actions distribution [19, 39, 38, 32]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -2.859375 and dist. [125, 0, 3] and actions distribution [60, 11, 28, 29]\n",
      "Batch of 128 exps with avg reward -5.546875 and dist. [14, 0, 114] and actions distribution [58, 15, 33, 22]\n",
      "Batch of 128 exps with avg reward -4.4453125 and dist. [127, 0, 1] and actions distribution [61, 8, 38, 21]\n",
      "episode 960\n",
      "Batch of 128 exps with avg reward -1.65625 and dist. [123, 0, 5] and actions distribution [33, 26, 40, 29]\n",
      "Batch of 128 exps with avg reward -5.6640625 and dist. [17, 0, 111] and actions distribution [25, 24, 41, 38]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.5859375 and dist. [15, 0, 113] and actions distribution [34, 18, 40, 36]\n",
      "episode 970\n",
      "Batch of 128 exps with avg reward -3.2421875 and dist. [125, 0, 3] and actions distribution [26, 26, 35, 41]\n",
      "Batch of 128 exps with avg reward -3.734375 and dist. [126, 0, 2] and actions distribution [27, 13, 44, 44]\n",
      "Batch of 128 exps with avg reward -3.8671875 and dist. [126, 0, 2] and actions distribution [27, 18, 44, 39]\n",
      "episode 980\n",
      "Batch of 128 exps with avg reward -4.0703125 and dist. [126, 0, 2] and actions distribution [24, 22, 36, 46]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.2890625 and dist. [125, 0, 3] and actions distribution [35, 24, 34, 35]\n",
      "episode 990\n",
      "Batch of 128 exps with avg reward -1.765625 and dist. [123, 0, 5] and actions distribution [28, 35, 36, 29]\n",
      "Batch of 128 exps with avg reward -5.9765625 and dist. [25, 0, 103] and actions distribution [14, 39, 44, 31]\n",
      "Batch of 128 exps with avg reward -3.5390625 and dist. [125, 0, 3] and actions distribution [23, 35, 49, 21]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:58:32,712] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001000.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1000\n",
      "Batch of 128 exps with avg reward -4.171875 and dist. [126, 0, 2] and actions distribution [33, 32, 34, 29]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.9453125 and dist. [126, 0, 2] and actions distribution [49, 23, 25, 31]\n",
      "Batch of 128 exps with avg reward -3.2265625 and dist. [125, 0, 3] and actions distribution [43, 27, 25, 33]\n",
      "episode 1010\n",
      "Batch of 128 exps with avg reward -4.0 and dist. [126, 0, 2] and actions distribution [44, 22, 28, 34]\n",
      "episode 1020\n",
      "Batch of 128 exps with avg reward 0.4921875 and dist. [120, 0, 8] and actions distribution [15, 51, 36, 26]\n",
      "Batch of 128 exps with avg reward -4.3984375 and dist. [126, 0, 2] and actions distribution [16, 42, 31, 39]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.0546875 and dist. [125, 0, 3] and actions distribution [28, 17, 49, 34]\n",
      "episode 1030\n",
      "Batch of 128 exps with avg reward -1.6875 and dist. [123, 0, 5] and actions distribution [37, 29, 39, 23]\n",
      "Batch of 128 exps with avg reward -2.6171875 and dist. [124, 0, 4] and actions distribution [43, 36, 28, 21]\n",
      "episode 1040\n",
      "Batch of 128 exps with avg reward -3.453125 and dist. [125, 0, 3] and actions distribution [26, 45, 33, 24]\n",
      "Batch of 128 exps with avg reward -3.9609375 and dist. [126, 0, 2] and actions distribution [40, 18, 40, 30]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.5859375 and dist. [15, 0, 113] and actions distribution [29, 20, 44, 35]\n",
      "episode 1050\n",
      "Batch of 128 exps with avg reward -4.5234375 and dist. [127, 0, 1] and actions distribution [25, 11, 49, 43]\n",
      "Batch of 128 exps with avg reward -0.0625 and dist. [121, 0, 7] and actions distribution [25, 35, 39, 29]\n",
      "episode 1060\n",
      "Batch of 128 exps with avg reward -3.3046875 and dist. [125, 0, 3] and actions distribution [36, 34, 31, 27]\n",
      "Batch of 128 exps with avg reward -2.7578125 and dist. [124, 0, 4] and actions distribution [25, 43, 27, 33]\n",
      "target_model <- acting_model\n",
      "episode 1070\n",
      "Batch of 128 exps with avg reward -3.6640625 and dist. [125, 0, 3] and actions distribution [15, 46, 37, 30]\n",
      "Batch of 128 exps with avg reward -5.8203125 and dist. [21, 0, 107] and actions distribution [21, 31, 46, 30]\n",
      "Batch of 128 exps with avg reward -4.3515625 and dist. [126, 0, 2] and actions distribution [26, 36, 38, 28]\n",
      "episode 1080\n",
      "Batch of 128 exps with avg reward -3.96875 and dist. [126, 0, 2] and actions distribution [36, 22, 38, 32]\n",
      "Batch of 128 exps with avg reward -4.4453125 and dist. [126, 0, 2] and actions distribution [33, 48, 23, 24]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -1.9921875 and dist. [123, 0, 5] and actions distribution [29, 56, 16, 27]\n",
      "episode 1090\n",
      "Batch of 128 exps with avg reward -0.9609375 and dist. [122, 0, 6] and actions distribution [39, 43, 24, 22]\n",
      "Batch of 128 exps with avg reward -3.6796875 and dist. [125, 0, 3] and actions distribution [12, 46, 34, 36]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 08:59:57,712] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001100.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1100\n",
      "Batch of 128 exps with avg reward -1.0859375 and dist. [122, 0, 6] and actions distribution [30, 52, 23, 23]\n",
      "Batch of 128 exps with avg reward -4.7890625 and dist. [127, 0, 1] and actions distribution [51, 22, 25, 30]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.390625 and dist. [10, 0, 118] and actions distribution [68, 12, 22, 26]\n",
      "episode 1110\n",
      "Batch of 128 exps with avg reward -4.03125 and dist. [126, 0, 2] and actions distribution [41, 25, 35, 27]\n",
      "Batch of 128 exps with avg reward 0.0234375 and dist. [121, 0, 7] and actions distribution [25, 37, 26, 40]\n",
      "episode 1120\n",
      "Batch of 128 exps with avg reward -1.046875 and dist. [122, 0, 6] and actions distribution [19, 37, 25, 47]\n",
      "Batch of 128 exps with avg reward -4.0859375 and dist. [126, 0, 2] and actions distribution [23, 25, 40, 40]\n",
      "target_model <- acting_model\n",
      "episode 1130\n",
      "Batch of 128 exps with avg reward -1.6328125 and dist. [123, 0, 5] and actions distribution [27, 32, 34, 35]\n",
      "Batch of 128 exps with avg reward -2.65625 and dist. [124, 0, 4] and actions distribution [29, 41, 33, 25]\n",
      "episode 1140\n",
      "Batch of 128 exps with avg reward -6.0546875 and dist. [27, 0, 101] and actions distribution [41, 44, 25, 18]\n",
      "Batch of 128 exps with avg reward -0.421875 and dist. [121, 0, 7] and actions distribution [33, 45, 32, 18]\n",
      "episode 1150\n",
      "Batch of 128 exps with avg reward -0.4921875 and dist. [121, 0, 7] and actions distribution [27, 51, 27, 23]\n",
      "target_model <- acting_model\n",
      "episode 1160\n",
      "Batch of 128 exps with avg reward 0.6796875 and dist. [120, 0, 8] and actions distribution [17, 38, 32, 41]\n",
      "Batch of 128 exps with avg reward -4.890625 and dist. [127, 0, 1] and actions distribution [31, 31, 36, 30]\n",
      "episode 1170\n",
      "Batch of 128 exps with avg reward -3.8359375 and dist. [126, 0, 2] and actions distribution [25, 22, 38, 43]\n",
      "Batch of 128 exps with avg reward -3.234375 and dist. [125, 0, 3] and actions distribution [30, 25, 37, 36]\n",
      "episode 1180\n",
      "Batch of 128 exps with avg reward 2.0546875 and dist. [118, 0, 10] and actions distribution [13, 50, 40, 25]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.265625 and dist. [127, 0, 1] and actions distribution [14, 41, 43, 30]\n",
      "Batch of 128 exps with avg reward -5.421875 and dist. [127, 0, 1] and actions distribution [19, 72, 25, 12]\n",
      "episode 1190\n",
      "Batch of 128 exps with avg reward -2.53125 and dist. [124, 0, 4] and actions distribution [64, 28, 29, 7]\n",
      "Batch of 128 exps with avg reward -4.109375 and dist. [126, 0, 2] and actions distribution [30, 33, 40, 25]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:01:10,479] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001200.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1200\n",
      "Batch of 128 exps with avg reward -2.71875 and dist. [124, 0, 4] and actions distribution [27, 37, 36, 28]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -2.25 and dist. [123, 0, 5] and actions distribution [4, 76, 27, 21]\n",
      "episode 1210\n",
      "Batch of 128 exps with avg reward -4.4140625 and dist. [126, 0, 2] and actions distribution [5, 51, 37, 35]\n",
      "Batch of 128 exps with avg reward -4.2734375 and dist. [126, 0, 2] and actions distribution [17, 34, 32, 45]\n",
      "episode 1220\n",
      "Batch of 128 exps with avg reward -3.0859375 and dist. [125, 0, 3] and actions distribution [35, 23, 28, 42]\n",
      "Batch of 128 exps with avg reward -4.65625 and dist. [127, 0, 1] and actions distribution [41, 20, 26, 41]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.6484375 and dist. [126, 0, 2] and actions distribution [11, 67, 26, 24]\n",
      "episode 1230\n",
      "Batch of 128 exps with avg reward -2.0859375 and dist. [123, 0, 5] and actions distribution [14, 72, 30, 12]\n",
      "Batch of 128 exps with avg reward -1.140625 and dist. [122, 0, 6] and actions distribution [20, 50, 33, 25]\n",
      "episode 1240\n",
      "Batch of 128 exps with avg reward -4.5 and dist. [126, 0, 2] and actions distribution [11, 53, 32, 32]\n",
      "Batch of 128 exps with avg reward -0.9765625 and dist. [122, 0, 6] and actions distribution [19, 50, 34, 25]\n",
      "target_model <- acting_model\n",
      "episode 1250\n",
      "Batch of 128 exps with avg reward -3.4296875 and dist. [125, 0, 3] and actions distribution [15, 33, 44, 36]\n",
      "Batch of 128 exps with avg reward -2.4765625 and dist. [124, 0, 4] and actions distribution [15, 35, 42, 36]\n",
      "episode 1260\n",
      "Batch of 128 exps with avg reward -1.640625 and dist. [123, 0, 5] and actions distribution [23, 37, 45, 23]\n",
      "Batch of 128 exps with avg reward -3.5390625 and dist. [125, 0, 3] and actions distribution [29, 49, 30, 20]\n",
      "episode 1270\n",
      "Batch of 128 exps with avg reward -4.1484375 and dist. [126, 0, 2] and actions distribution [47, 24, 31, 26]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.3046875 and dist. [126, 0, 2] and actions distribution [30, 43, 33, 22]\n",
      "episode 1280\n",
      "Batch of 128 exps with avg reward -3.6484375 and dist. [126, 0, 2] and actions distribution [32, 11, 68, 17]\n",
      "Batch of 128 exps with avg reward -4.671875 and dist. [127, 0, 1] and actions distribution [22, 18, 55, 33]\n",
      "Batch of 128 exps with avg reward -2.0234375 and dist. [123, 0, 5] and actions distribution [19, 49, 24, 36]\n",
      "episode 1290\n",
      "Batch of 128 exps with avg reward -3.4453125 and dist. [125, 0, 3] and actions distribution [19, 48, 30, 31]\n",
      "target_model <- acting_model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:02:34,129] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001300.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1300\n",
      "Batch of 128 exps with avg reward 2.984375 and dist. [117, 0, 11] and actions distribution [29, 48, 33, 18]\n",
      "Batch of 128 exps with avg reward -4.234375 and dist. [126, 0, 2] and actions distribution [22, 39, 47, 20]\n",
      "episode 1310\n",
      "Batch of 128 exps with avg reward -3.3984375 and dist. [125, 0, 3] and actions distribution [24, 47, 44, 13]\n",
      "Batch of 128 exps with avg reward -5.234375 and dist. [6, 0, 122] and actions distribution [26, 7, 87, 8]\n",
      "episode 1320\n",
      "Batch of 128 exps with avg reward -0.328125 and dist. [121, 0, 7] and actions distribution [32, 52, 25, 19]\n",
      "target_model <- acting_model\n",
      "episode 1330\n",
      "Batch of 128 exps with avg reward 3.7734375 and dist. [116, 0, 12] and actions distribution [34, 51, 12, 31]\n",
      "Batch of 128 exps with avg reward -5.4453125 and dist. [127, 0, 1] and actions distribution [8, 63, 26, 31]\n",
      "episode 1340\n",
      "Batch of 128 exps with avg reward -1.7890625 and dist. [123, 0, 5] and actions distribution [26, 37, 32, 33]\n",
      "Batch of 128 exps with avg reward -1.5234375 and dist. [123, 0, 5] and actions distribution [27, 25, 37, 39]\n",
      "episode 1350\n",
      "Batch of 128 exps with avg reward -1.6953125 and dist. [123, 0, 5] and actions distribution [26, 30, 37, 35]\n",
      "target_model <- acting_model\n",
      "episode 1360\n",
      "Batch of 128 exps with avg reward 0.3984375 and dist. [120, 0, 8] and actions distribution [16, 65, 29, 18]\n",
      "Batch of 128 exps with avg reward -2.265625 and dist. [123, 0, 5] and actions distribution [7, 79, 29, 13]\n",
      "episode 1370\n",
      "Batch of 128 exps with avg reward -4.3671875 and dist. [126, 0, 2] and actions distribution [29, 58, 22, 19]\n",
      "Batch of 128 exps with avg reward -5.0078125 and dist. [127, 0, 1] and actions distribution [44, 32, 32, 20]\n",
      "Batch of 128 exps with avg reward -4.859375 and dist. [127, 0, 1] and actions distribution [30, 29, 37, 32]\n",
      "target_model <- acting_model\n",
      "episode 1380\n",
      "Batch of 128 exps with avg reward -4.109375 and dist. [126, 0, 2] and actions distribution [27, 30, 39, 32]\n",
      "Batch of 128 exps with avg reward -5.109375 and dist. [127, 0, 1] and actions distribution [11, 31, 46, 40]\n",
      "Batch of 128 exps with avg reward -3.328125 and dist. [125, 0, 3] and actions distribution [19, 35, 41, 33]\n",
      "episode 1390\n",
      "Batch of 128 exps with avg reward -0.6796875 and dist. [122, 0, 6] and actions distribution [15, 21, 51, 41]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:03:33,681] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001400.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1400\n",
      "Batch of 128 exps with avg reward -0.4140625 and dist. [121, 0, 7] and actions distribution [13, 55, 33, 27]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -4.015625 and dist. [126, 0, 2] and actions distribution [33, 27, 40, 28]\n",
      "Batch of 128 exps with avg reward -5.140625 and dist. [127, 0, 1] and actions distribution [23, 42, 41, 22]\n",
      "episode 1410\n",
      "Batch of 128 exps with avg reward -5.7421875 and dist. [19, 0, 109] and actions distribution [36, 24, 44, 24]\n",
      "Batch of 128 exps with avg reward -5.703125 and dist. [18, 0, 110] and actions distribution [48, 23, 33, 24]\n",
      "Batch of 128 exps with avg reward -5.171875 and dist. [127, 0, 1] and actions distribution [37, 31, 34, 26]\n",
      "target_model <- acting_model\n",
      "episode 1420\n",
      "Batch of 128 exps with avg reward 1.6328125 and dist. [119, 0, 9] and actions distribution [30, 31, 38, 29]\n",
      "Batch of 128 exps with avg reward -4.2421875 and dist. [126, 0, 2] and actions distribution [14, 49, 34, 31]\n",
      "episode 1430\n",
      "Batch of 128 exps with avg reward 1.4453125 and dist. [119, 0, 9] and actions distribution [17, 41, 33, 37]\n",
      "episode 1440\n",
      "episode 1450\n",
      "Batch of 128 exps with avg reward 5.3359375 and dist. [114, 0, 14] and actions distribution [10, 62, 41, 15]\n",
      "Batch of 128 exps with avg reward -1.0859375 and dist. [122, 0, 6] and actions distribution [23, 50, 32, 23]\n",
      "target_model <- acting_model\n",
      "episode 1460\n",
      "Batch of 128 exps with avg reward -1.1953125 and dist. [123, 0, 5] and actions distribution [40, 11, 38, 39]\n",
      "Batch of 128 exps with avg reward -3.2734375 and dist. [125, 0, 3] and actions distribution [43, 33, 29, 23]\n",
      "episode 1470\n",
      "Batch of 128 exps with avg reward -3.2734375 and dist. [125, 0, 3] and actions distribution [60, 32, 23, 13]\n",
      "Batch of 128 exps with avg reward -4.046875 and dist. [126, 0, 2] and actions distribution [45, 30, 32, 21]\n",
      "Batch of 128 exps with avg reward -5.625 and dist. [16, 0, 112] and actions distribution [51, 16, 35, 26]\n",
      "target_model <- acting_model\n",
      "episode 1480\n",
      "Batch of 128 exps with avg reward -3.3125 and dist. [125, 0, 3] and actions distribution [14, 27, 48, 39]\n",
      "Batch of 128 exps with avg reward -3.1171875 and dist. [125, 0, 3] and actions distribution [15, 23, 57, 33]\n",
      "episode 1490\n",
      "Batch of 128 exps with avg reward -4.9140625 and dist. [127, 0, 1] and actions distribution [8, 23, 52, 45]\n",
      "Batch of 128 exps with avg reward -0.2421875 and dist. [121, 0, 7] and actions distribution [10, 49, 39, 30]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:04:45,336] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001500.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1500\n",
      "Batch of 128 exps with avg reward -0.828125 and dist. [122, 0, 6] and actions distribution [35, 28, 38, 27]\n",
      "target_model <- acting_model\n",
      "episode 1510\n",
      "Batch of 128 exps with avg reward -0.890625 and dist. [122, 0, 6] and actions distribution [19, 32, 40, 37]\n",
      "Batch of 128 exps with avg reward -3.9453125 and dist. [126, 0, 2] and actions distribution [51, 23, 32, 22]\n",
      "Batch of 128 exps with avg reward -3.984375 and dist. [126, 0, 2] and actions distribution [68, 24, 27, 9]\n",
      "episode 1520\n",
      "Batch of 128 exps with avg reward -3.40625 and dist. [125, 0, 3] and actions distribution [48, 44, 26, 10]\n",
      "Batch of 128 exps with avg reward -4.234375 and dist. [126, 0, 2] and actions distribution [19, 32, 48, 29]\n",
      "target_model <- acting_model\n",
      "episode 1530\n",
      "Batch of 128 exps with avg reward -3.7109375 and dist. [126, 0, 2] and actions distribution [15, 14, 61, 38]\n",
      "Batch of 128 exps with avg reward -4.7578125 and dist. [127, 0, 1] and actions distribution [22, 17, 49, 40]\n",
      "episode 1540\n",
      "Batch of 128 exps with avg reward 1.59375 and dist. [119, 0, 9] and actions distribution [23, 42, 47, 16]\n",
      "episode 1550\n",
      "Batch of 128 exps with avg reward 1.9921875 and dist. [118, 0, 10] and actions distribution [14, 77, 18, 19]\n",
      "episode 1560\n",
      "Batch of 128 exps with avg reward -1.0234375 and dist. [122, 0, 6] and actions distribution [29, 42, 25, 32]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.5078125 and dist. [13, 0, 115] and actions distribution [24, 17, 50, 37]\n",
      "Batch of 128 exps with avg reward -4.4765625 and dist. [127, 0, 1] and actions distribution [31, 14, 58, 25]\n",
      "Batch of 128 exps with avg reward -4.7109375 and dist. [127, 0, 1] and actions distribution [39, 21, 48, 20]\n",
      "Batch of 128 exps with avg reward -5.8203125 and dist. [21, 0, 107] and actions distribution [22, 41, 43, 22]\n",
      "episode 1570\n",
      "episode 1580\n",
      "Batch of 128 exps with avg reward 2.4140625 and dist. [118, 0, 10] and actions distribution [34, 32, 46, 16]\n",
      "target_model <- acting_model\n",
      "episode 1590\n",
      "Batch of 128 exps with avg reward 1.609375 and dist. [119, 0, 9] and actions distribution [26, 33, 44, 25]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:05:54,318] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001600.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1600\n",
      "Batch of 128 exps with avg reward 5.4296875 and dist. [114, 0, 14] and actions distribution [18, 57, 36, 17]\n",
      "episode 1610\n",
      "Batch of 128 exps with avg reward 1.375 and dist. [119, 0, 9] and actions distribution [16, 54, 24, 34]\n",
      "episode 1620\n",
      "Batch of 128 exps with avg reward 0.5625 and dist. [120, 0, 8] and actions distribution [8, 46, 48, 26]\n",
      "Batch of 128 exps with avg reward -2.5625 and dist. [124, 0, 4] and actions distribution [25, 33, 42, 28]\n",
      "target_model <- acting_model\n",
      "episode 1630\n",
      "Batch of 128 exps with avg reward -4.1171875 and dist. [126, 0, 2] and actions distribution [45, 28, 35, 20]\n",
      "Batch of 128 exps with avg reward -5.234375 and dist. [6, 0, 122] and actions distribution [53, 6, 64, 5]\n",
      "Batch of 128 exps with avg reward -1.453125 and dist. [123, 0, 5] and actions distribution [51, 31, 36, 10]\n",
      "episode 1640\n",
      "Batch of 128 exps with avg reward 0.3359375 and dist. [120, 0, 8] and actions distribution [16, 66, 27, 19]\n",
      "episode 1650\n",
      "Batch of 128 exps with avg reward 1.328125 and dist. [119, 0, 9] and actions distribution [20, 44, 42, 22]\n",
      "target_model <- acting_model\n",
      "episode 1660\n",
      "Batch of 128 exps with avg reward 3.65625 and dist. [116, 0, 12] and actions distribution [15, 55, 29, 29]\n",
      "episode 1670\n",
      "Batch of 128 exps with avg reward -2.15625 and dist. [123, 0, 5] and actions distribution [8, 56, 30, 34]\n",
      "episode 1680\n",
      "Batch of 128 exps with avg reward 3.15625 and dist. [117, 0, 11] and actions distribution [9, 41, 41, 37]\n",
      "episode 1690\n",
      "Batch of 128 exps with avg reward 3.8359375 and dist. [116, 0, 12] and actions distribution [9, 46, 40, 33]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:06:35,826] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001700.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1700\n",
      "episode 1710\n",
      "Batch of 128 exps with avg reward 6.125 and dist. [113, 0, 15] and actions distribution [10, 53, 33, 32]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -2.109375 and dist. [123, 0, 5] and actions distribution [12, 57, 33, 26]\n",
      "episode 1720\n",
      "Batch of 128 exps with avg reward -2.9453125 and dist. [124, 0, 4] and actions distribution [13, 54, 33, 28]\n",
      "Batch of 128 exps with avg reward -2.6640625 and dist. [124, 0, 4] and actions distribution [32, 39, 39, 18]\n",
      "episode 1730\n",
      "Batch of 128 exps with avg reward -1.9609375 and dist. [123, 0, 5] and actions distribution [22, 58, 33, 15]\n",
      "Batch of 128 exps with avg reward -2.6015625 and dist. [124, 0, 4] and actions distribution [41, 43, 35, 9]\n",
      "target_model <- acting_model\n",
      "episode 1740\n",
      "Batch of 128 exps with avg reward -4.890625 and dist. [126, 0, 2] and actions distribution [4, 100, 20, 4]\n",
      "Batch of 128 exps with avg reward -6.015625 and dist. [26, 0, 102] and actions distribution [7, 45, 57, 19]\n",
      "Batch of 128 exps with avg reward -4.96875 and dist. [127, 0, 1] and actions distribution [14, 35, 57, 22]\n",
      "episode 1750\n",
      "Batch of 128 exps with avg reward -4.421875 and dist. [126, 0, 2] and actions distribution [8, 41, 47, 32]\n",
      "Batch of 128 exps with avg reward -1.140625 and dist. [122, 0, 6] and actions distribution [10, 46, 52, 20]\n",
      "target_model <- acting_model\n",
      "episode 1760\n",
      "Batch of 128 exps with avg reward -5.34375 and dist. [127, 0, 1] and actions distribution [9, 43, 46, 30]\n",
      "Batch of 128 exps with avg reward -3.515625 and dist. [125, 0, 3] and actions distribution [18, 32, 55, 23]\n",
      "episode 1770\n",
      "Batch of 128 exps with avg reward 0.0078125 and dist. [121, 0, 7] and actions distribution [21, 30, 52, 25]\n",
      "Batch of 128 exps with avg reward -2.9375 and dist. [124, 0, 4] and actions distribution [13, 53, 41, 21]\n",
      "episode 1780\n",
      "Batch of 128 exps with avg reward -3.5234375 and dist. [125, 0, 3] and actions distribution [14, 49, 47, 18]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -5.7421875 and dist. [19, 0, 109] and actions distribution [20, 25, 47, 36]\n",
      "episode 1790\n",
      "Batch of 128 exps with avg reward -0.8828125 and dist. [122, 0, 6] and actions distribution [24, 28, 49, 27]\n",
      "Batch of 128 exps with avg reward -4.8203125 and dist. [127, 0, 1] and actions distribution [30, 24, 45, 29]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:07:49,943] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001800.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1800\n",
      "Batch of 128 exps with avg reward -1.015625 and dist. [122, 0, 6] and actions distribution [16, 33, 47, 32]\n",
      "episode 1810\n",
      "Batch of 128 exps with avg reward -0.3046875 and dist. [121, 0, 7] and actions distribution [16, 41, 42, 29]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -3.578125 and dist. [125, 0, 3] and actions distribution [11, 44, 46, 27]\n",
      "episode 1820\n",
      "Batch of 128 exps with avg reward -1.1171875 and dist. [122, 0, 6] and actions distribution [9, 47, 49, 23]\n",
      "episode 1830\n",
      "Batch of 128 exps with avg reward -0.625 and dist. [122, 0, 6] and actions distribution [14, 23, 73, 18]\n",
      "Batch of 128 exps with avg reward -1.671875 and dist. [123, 0, 5] and actions distribution [16, 33, 59, 20]\n",
      "episode 1840\n",
      "Batch of 128 exps with avg reward -2.0 and dist. [123, 0, 5] and actions distribution [15, 56, 38, 19]\n",
      "target_model <- acting_model\n",
      "Batch of 128 exps with avg reward -2.8046875 and dist. [124, 0, 4] and actions distribution [38, 58, 24, 8]\n",
      "episode 1850\n",
      "Batch of 128 exps with avg reward -3.6171875 and dist. [125, 0, 3] and actions distribution [18, 45, 35, 30]\n",
      "episode 1860\n",
      "Batch of 128 exps with avg reward 5.3515625 and dist. [114, 0, 14] and actions distribution [9, 59, 37, 23]\n",
      "episode 1870\n",
      "Batch of 128 exps with avg reward 4.6171875 and dist. [115, 0, 13] and actions distribution [7, 49, 46, 26]\n",
      "episode 1880\n",
      "Batch of 128 exps with avg reward 0.40625 and dist. [120, 0, 8] and actions distribution [6, 49, 45, 28]\n",
      "target_model <- acting_model\n",
      "episode 1890\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:08:40,579] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video001900.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 1900\n",
      "Batch of 128 exps with avg reward 3.7734375 and dist. [116, 0, 12] and actions distribution [4, 54, 43, 27]\n",
      "Batch of 128 exps with avg reward -4.3203125 and dist. [126, 0, 2] and actions distribution [4, 39, 48, 37]\n",
      "episode 1910\n",
      "Batch of 128 exps with avg reward -0.765625 and dist. [122, 0, 6] and actions distribution [14, 29, 56, 29]\n",
      "Batch of 128 exps with avg reward -2.15625 and dist. [124, 0, 4] and actions distribution [13, 13, 65, 37]\n",
      "episode 1920\n",
      "Batch of 128 exps with avg reward 2.296875 and dist. [118, 0, 10] and actions distribution [7, 35, 42, 44]\n",
      "target_model <- acting_model\n",
      "episode 1930\n",
      "Batch of 128 exps with avg reward -3.6015625 and dist. [125, 0, 3] and actions distribution [19, 44, 33, 32]\n",
      "episode 1940\n",
      "Batch of 128 exps with avg reward 1.796875 and dist. [118, 0, 10] and actions distribution [9, 71, 21, 27]\n",
      "episode 1950\n",
      "Batch of 128 exps with avg reward 2.078125 and dist. [118, 0, 10] and actions distribution [14, 46, 33, 35]\n",
      "episode 1960\n",
      "Batch of 128 exps with avg reward 2.09375 and dist. [118, 0, 10] and actions distribution [15, 48, 28, 37]\n",
      "episode 1970\n",
      "Batch of 128 exps with avg reward 6.7578125 and dist. [112, 0, 16] and actions distribution [7, 54, 35, 32]\n",
      "target_model <- acting_model\n",
      "episode 1980\n",
      "Batch of 128 exps with avg reward 0.578125 and dist. [120, 0, 8] and actions distribution [4, 44, 36, 44]\n",
      "episode 1990\n",
      "Batch of 128 exps with avg reward 2.2578125 and dist. [118, 0, 10] and actions distribution [5, 44, 32, 47]\n",
      "Batch of 60 exps with avg reward 0.9166666666666666 and dist. [56, 0, 4] and actions distribution [4, 27, 16, 13]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:09:21,687] Starting new video recorder writing to /Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay/openaigym.video.1.23179.video002000.mp4\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "episode 0\n",
      "episode 10\n",
      "episode 20\n",
      "episode 30\n",
      "episode 40\n",
      "episode 50\n",
      "episode 60\n",
      "episode 70\n",
      "episode 80\n",
      "episode 90\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2017-03-12 09:09:33,865] Finished writing results. You can upload them to the scoreboard via gym.upload('/Users/gui/Dev/rl-study/tmp/DoomBasic_dqn_replay')\n",
      "[2017-03-12 09:09:33,873] [DoomBasic-v0] Uploading 2100 episodes of training data\n",
      "[2017-03-12 09:09:39,734] [DoomBasic-v0] Uploading videos of 21 training episodes (2652401 bytes)\n",
      "[2017-03-12 09:10:08,711] [DoomBasic-v0] Creating evaluation object from tmp/DoomBasic_dqn_replay with learning curve and training video\n",
      "[2017-03-12 09:10:09,155] \n",
      "****************************************************\n",
      "You successfully uploaded your evaluation on DoomBasic-v0 to\n",
      "OpenAI Gym! You can find it at:\n",
      "\n",
      "    https://gym.openai.com/evaluations/eval_kOeHaJ2CQxaaQc2nOOLRA\n",
      "\n",
      "****************************************************\n"
     ]
    }
   ],
   "source": [
    "N_BATCHES = 200\n",
    "\n",
    "\n",
    "class DeepQNetworkTrainer:\n",
    "    \n",
    "    MINI_BATCH_SIZE = 128\n",
    "    UPDATE_TARGET_EVERY_N_BACTHES = 5\n",
    "    \n",
    "    def __init__(self, acting_model, target_model, double_q=False, gamma=.99, reward_clip=5):\n",
    "        self.acting_model = acting_model\n",
    "        self.target_model = target_model\n",
    "        self.double_q = double_q\n",
    "        \n",
    "        self.gamma = gamma\n",
    "        self.reward_clip = reward_clip\n",
    "\n",
    "        # counter to periodically set target_model <- acting_model\n",
    "        self.trained_epochs = 0\n",
    "    \n",
    "    def fit_episodes(self, experiences_batch):\n",
    "        prev_frames, target_action_rewards = self._batch_to_input_targets(experiences_batch, double_q=self.double_q)\n",
    "        self.acting_model.fit(x=prev_frames, y=target_action_rewards, batch_size=self.MINI_BATCH_SIZE, nb_epoch=1, verbose=0)\n",
    "        self.trained_epochs += 1\n",
    "\n",
    "        if (self.trained_epochs % self.UPDATE_TARGET_EVERY_N_BACTHES) == 0:\n",
    "            print('target_model <- acting_model')\n",
    "            self.target_model = copy_model(self.acting_model)\n",
    "\n",
    "    def _batch_to_input_targets(self, experiences_batch, double_q=False):\n",
    "        \n",
    "        experiences = list(filter(None, experiences_batch))\n",
    "        random.shuffle(experiences)\n",
    "        n_samples = len(experiences)\n",
    "\n",
    "        prev_frames, actions, rewards, next_frames, is_ends = zip(*experiences)\n",
    "        prev_frames = np.asarray(prev_frames)\n",
    "        next_frames = np.asarray(next_frames)\n",
    "        actions = np.asarray(actions)\n",
    "        rewards = np.asarray(rewards)\n",
    "        is_ends = np.asarray(is_ends)\n",
    "\n",
    "        print('Batch of {} exps with avg reward {} and dist. {} and actions distribution {}'.format(\n",
    "            n_samples,\n",
    "            np.mean(rewards),\n",
    "            np.histogram(rewards, bins=3)[0].tolist(),\n",
    "            np.bincount(actions).tolist()))\n",
    "\n",
    "        clipped_rewards = np.clip(rewards, -np.inf, self.reward_clip)\n",
    "\n",
    "        if double_q:\n",
    "            greedy_actions = self.acting_model.predict(next_frames).argmax(axis=1)\n",
    "            actions_target_values = self.target_model.predict(next_frames)[np.arange(n_samples), greedy_actions]            \n",
    "            targets = clipped_rewards + self.gamma * (1 - is_ends) * actions_target_values\n",
    "        else:\n",
    "            # Transcription of the Q-learning target formula\n",
    "            targets = clipped_rewards + self.gamma * (1 - is_ends) * self.target_model.predict(next_frames).max(axis=1)\n",
    "\n",
    "            target_action_rewards = self.target_model.predict(prev_frames)\n",
    "            target_action_rewards[np.arange(n_samples), actions] = targets\n",
    "\n",
    "            return prev_frames, target_action_rewards\n",
    "\n",
    "directory = 'tmp/DoomBasic_dqn_replay'\n",
    "env = create_env(monitor_directory=directory)\n",
    "\n",
    "trainer = DeepQNetworkTrainer(acting_model, target_model)\n",
    "\n",
    "experiences = (\n",
    "    Experience(previous_s, a, r, next_s, end) \n",
    "    for (previous_s, a, r, end), (next_s, _, _, _) \n",
    "    in pairwise(generate_sares(env, EpsilonGreedyQAgent(acting_model))))\n",
    "\n",
    "#experience_batches = ExperiencesReplay(experiences, memory_size=1000, batch_size=128)\n",
    "experience_batches = grouper(experiences, n=128)\n",
    "\n",
    "for experience_batch in experience_batches:\n",
    "    trainer.fit_episodes(experience_batch)\n",
    "    \n",
    "# final greedy episodes\n",
    "sares = list(generate_sares(env, EpsilonGreedyQAgent(acting_model, epsilon=0), episode_count=100))\n",
    "\n",
    "env.close()\n",
    "gym.upload(directory, api_key='sk_bNZUvCfkTfabQCoKoKbjFA')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Embedding viz\n",
    "\n",
    "See http://projector.tensorflow.org/?config=https://raw.githubusercontent.com/pilipolio/rl-study/master/projectors/doom_v1_projector_config.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "prev_frames, target_action_rewards = sares_to_input_targets(target_model, sares, n_misses=None)\n",
    "\n",
    "n_frames = 1000\n",
    "frames = prev_frames[:n_frames, :, :, :]\n",
    "thumbnails = frames[:, :, 25:-25, :]\n",
    "frame_embeddings = Sequential(acting_model.layers[:-1]).predict(frames)\n",
    "\n",
    "\n",
    "frame_action_rewards = acting_model.predict(frames)\n",
    "frame_metadata = pd.DataFrame.from_dict({\n",
    "    'best_action': np.array(['NOOP', 'SHOOT', 'LEFT', 'RIGHT'])[frame_action_rewards.argmax(1)], \n",
    "    'value': frame_action_rewards.max(1)})\\\n",
    "    .assign(value_quantile=lambda df: np.digitize(df.value, bins=np.percentile(df.value, q=[25, 50, 75])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import scipy.misc\n",
    "\n",
    "def images_to_sprite(data):\n",
    "    \"\"\"Creates the sprite image along with any necessary padding\n",
    "    From https://github.com/tensorflow/tensorflow/issues/6322\n",
    "    Args:\n",
    "      data: NxHxW[x3] tensor containing the images.\n",
    "\n",
    "    Returns:\n",
    "      data: Properly shaped HxWx3 image with any necessary padding.\n",
    "    \"\"\"\n",
    "    if len(data.shape) == 3:\n",
    "        data = np.tile(data[...,np.newaxis], (1,1,1,3))\n",
    "    data = data.astype(np.float32)\n",
    "    min = np.min(data.reshape((data.shape[0], -1)), axis=1)\n",
    "    data = (data.transpose(1,2,3,0) - min).transpose(3,0,1,2)\n",
    "    max = np.max(data.reshape((data.shape[0], -1)), axis=1)\n",
    "    data = (data.transpose(1,2,3,0) / max).transpose(3,0,1,2)\n",
    "    # Inverting the colors seems to look better for MNIST\n",
    "    #data = 1 - data\n",
    "\n",
    "    n = int(np.ceil(np.sqrt(data.shape[0])))\n",
    "    padding = ((0, n ** 2 - data.shape[0]), (0, 0),\n",
    "            (0, 0)) + ((0, 0),) * (data.ndim - 3)\n",
    "    data = np.pad(data, padding, mode='constant',\n",
    "            constant_values=0)\n",
    "    # Tile the individual thumbnails into an image.\n",
    "    data = data.reshape((n, n) + data.shape[1:]).transpose((0, 2, 1, 3)\n",
    "            + tuple(range(4, data.ndim + 1)))\n",
    "    data = data.reshape((n * data.shape[1], n * data.shape[3]) + data.shape[4:])\n",
    "    data = (data * 255).astype(np.uint8)\n",
    "    return data\n",
    "\n",
    "def save_projector_config(frame_embeddings, frame_metadata, thumbnails=None):\n",
    "    gh_root = 'https://raw.githubusercontent.com/pilipolio/rl-study/master'\n",
    "    projector_dir = 'projectors'\n",
    "    embedding_name = 'doom_v1'\n",
    "    \n",
    "    projector_config = {\n",
    "        'embeddings': [\n",
    "        {\n",
    "            'metadataPath': os.path.join(gh_root, projector_dir, embedding_name + '_metadata.tsv'),\n",
    "            'tensorName': 'Frames',\n",
    "            'tensorShape': frame_embeddings.shape,\n",
    "            'tensorPath': os.path.join(gh_root, projector_dir, embedding_name + '.tsv')\n",
    "        }\n",
    "        ]\n",
    "    }\n",
    "    \n",
    "    if thumbnails is not None:\n",
    "        projector_config['embeddings'][0]['sprite'] = {\n",
    "                'imagePath': os.path.join(gh_root, projector_dir, embedding_name + '_sprite.png'),\n",
    "                'singleImageDim': thumbnails.shape}\n",
    "        sprite = images_to_sprite(thumbnails)\n",
    "        scipy.misc.imsave(os.path.join(projector_dir, embedding_name + '_sprite.png'), sprite)\n",
    "        \n",
    "    pd.DataFrame(frame_embeddings).to_csv(os.path.join(projector_dir, embedding_name + '.tsv'),\n",
    "                            sep='\\t', index=None, header=None)\n",
    "    frame_metadata.to_csv(os.path.join(projector_dir, embedding_name + '_metadata.tsv'), sep='\\t', index=None)\n",
    "\n",
    "    with open(os.path.join(projector_dir, embedding_name + ('_with_sprite' if thumbnails is not None else '') + '_projector_config.json'), 'w+') as f:\n",
    "        json.dump(projector_config, f)\n",
    "\n",
    "save_projector_config(frame_embeddings, frame_metadata)\n",
    "\n",
    "save_projector_config(frame_embeddings, frame_metadata, thumbnails)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}