Skip to content

Instantly share code, notes, and snippets.

@yrevar

yrevar/Boltzmann Policy.ipynb

Last active December 16, 2018 23:22
Show Gist options
  • Save yrevar/6c96ebc81870110b82fc6256ec770958 to your computer and use it in GitHub Desktop.
Save yrevar/6c96ebc81870110b82fc6256ec770958 to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
Boltzmann Policy.ipynb
{
"cells": [
{
"cell_type": "code",{
"cells": [
{
"cell_type": "code",
"execution_count": 1,{{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: OpenAI gym not installed.\n"
]
}
],
"source": [
"# Other Imports.\n",
"from simple_rl.tasks import NavigationWorldMDP\n",
"from simple_rl.agents import QLearningAgent\n",
"from simple_rl.planning import ValueIteration\n",
"from simple_rl.tasks.grid_world.GridWorldStateClass import GridWorldState\n",
"from simple_rl.tasks.navigation.NavigationStateClass import NavigationWorldState\n",
"\n",
"# Python Imports.\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"%matplotlib inline\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Given goal MDP"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def value_iter(S, A, R, T, gamma, policy, n_iters):\n",
" \n",
" nS, nA = len(S), len(A)\n",
" V = np.zeros((nS))\n",
" Q = np.zeros((nS, nA))\n",
" Pi = np.ones((nS, nA)) / nA\n",
" s_to_idx = {s: idx for idx, s in enumerate(S)}\n",
" \n",
" for _iter in range(n_iters):\n",
" for si, s in enumerate(S):\n",
" \n",
" if s.is_terminal():\n",
" continue\n",
" for ai, a in enumerate(A):\n",
" s_prime_idx = s_to_idx[T(s,a)]\n",
" Q[si,ai] = R(s,a) + gamma * V[s_prime_idx]\n",
" V[si] = policy(Q[si,:]).dot(Q[si,:])\n",
" return V"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAFyCAYAAAApoxulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFOtJREFUeJzt3X2QZXV95/H3B2aAyMMMhBEDSTtFiljG4SEGiww6AYQSzAbUJJoHn1JlMlVWlk25S6LZyiYmf8QqUrusq7VbO7VVKRUIRqOBEYoncSioUVi0MCvZYkcq2ohAZkAaH+hxGL77xz0jl85gD6dP3/4x9/2qmurzcO/5fk9P96d//Tvn3k5VIUlqyyEr3YAk6V8znCWpQYazJDXIcJakBhnOktQgw1mSGmQ4qwlJ3p7k5mU69kyS7yU5dDmOLy0Hw1kHLMk3kvxLkiPHtv1ekm1LPXZVXVVVb1jqceBHfV4wduzZqjqqqvYOcfwFtar7nKwa27a621Zj27YlmU/y3SRPJvlykg8kOXzsMR9Msqf7QfJEku1JNg7ds14cDGe9UIcCf7jSTTTmO8Abx9bf2G1b6N9W1dHATwH/Afgt4IYkGXvMJ6vqKGAdcCfwmQX7NSUMZ71Qfw1clmTt/nYm+XCSB8dGh5u67ScmeSrJcWOP/YUku7qR5u8muXNs3xuS3J9kLsl/T3J7kt/r9v1sktuSPNY9/6p9/ST5BDADbO1GoH+cZH03wl011st1SR5P8vUkvz9W94NJ/i7Jx7tR7n1Jzlzkc/IJ4F1j6+8CPv58D66q71fVNuASYCPwb/bzmD3Ax4CXAT+5SH0dhAxnvVD3ANuAy55n//8GzgCOA64GPpXkiKr6NvBF4NfHHvs7wKe7IPqRJMcDnwb+hFEw3Q+cPf4Q4EPAicArgZ8BPghQVe8EZoGLu6mMy/fT4zXAt7rn/wbwV0leP7b/ku4xa4HrgI8+z7nu8w/ALydZm+RYYBNw7SLPoapmGX0+Ny3c1013/C7wYFXtWuxYOvgYzurjz4BLk6xbuKOqrqyqx6rq6ar6z8DhwCu63VcDvw3Q/ar+W922hX4FuK+qPlNVTwP/DXhkrMbXq+qWqtpdVTuB/wKccyCNJ/kZ4LXA+6tqvqruBf4Xzx353llVN3Rz1J8ATl/ksPPAVuA3u3/XddsOxLcZ/SDb521JngAeBH4ReMsBHkcHGcNZL1hVfQ34HPCBhfuSXJbk/3bTEU8Aa4Dju91/D2xM8lPALwPPAHfsp8SJjMJpX71iNNLdV+OEJNckeSjJk8CVYzUWcyLweFV9d2zbN4GTxtYfGVv+AXDE+AW/5/FxRgH/Y6c09uMk4PGx9b+rqrVV9dKqen1VffkFHEsHEcNZff058PuMhVo3v/zHwNuAY6tqLTDHaBqCqvoOcDOj0eXvANfU/t8W8WHgp8eOm/F14K+AAk6tqmOAd+yr0flxb7X4beC4JEePbZsBHvpxJ3sA7mB0oe8ERhfyFtWN4n+R/f+A0pQznNVLVX0d+CTw78Y2Hw08DewEViX5M+CYBU+9mtHo8jfY/5QGwPXAqUne3I1Y/4DRhbHxOt8D5pKcBPzRguc/Cpz8PH0/CGwHPpTkiCSnAe9hNPrurfshczFwyfP8wPmRJC9Jcg6jeem7gRuWUlsHJ8NZS/GXwJFj6zcBNwL/j9FUwTxj0xOd64BTgEeq6qv7O2h3AeytwOXAY8DPM7pwtrt7yF8Ar2Y0Kr8e+MyCQ3wI+NPuXuH9Xbj8bWA9o1H0Z4E/r6pbFznXRVXVfVV13495yEeTfJfRD4//ymia56KqemaptXXwiW+2r9YlOYTRnPPbq+oLK92PNAmOnNWkJBd2t6YdDvxHRnPKX1rhtqSJMZzVqo3AA8AuRnO5b66qp1a2JWlynNaQpAY5cpakBi12Y/2SHH/88bV+/frlLCFJLyrf+MY32LVr16JvZrWs4bx+/Xruueee5SwhSS8qZ5652PtojTitIUkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNWjRcE5yYpKvJJlPsmrh+iSalKRpcyAj58eB83n2r1AsXJckDWzRcK6q+e5P2u93faEkm5PsSLJzdnZ2qD4laaoMPudcVVuq6pSqWjczMzP04SVpKnhBUJIaZDhLUoMO5G6N1UluBU4Hbkpy1sL15W5SkqbNorfCVdUe4IIFmxeuS5IG5LSGJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMWDeckJyb5SpL5JKu6bVckuSPJh5e/RUmaPgcycn4cOB/4EkCSVwNHVdUm4LAkr1nG/iRpKi0azlU1X1XfGdv0S8At3fKtwMblaEySplmfOee1wJPd8ly3/hxJLk6yZW5ubim9SdLU6hPOc8Ax3fIxwBMLH1BVW6tq85o1a5bSmyRNrT7h/EVGc9AAF9DNRUuShnMgd2usTnIrcDpwE7AamE9yB7C3qu5e5h4laeqsWuwBVbWH0Qh53F3L044kCXwRiiQ1yXCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJalCvcE6yKsk1Sb6Q5PKhm5Kkadd35PwW4KtVdR7wE0lOH7AnSZp6fcP5ZOAfu+V7gbP37UiyOcmOJDtnZ2eX2p8kTaW+4Xw/cE63fB6wdt+OqtpSVadU1bqZmZml9idJU6lvOG9lNJ3xeWA38OhwLUmSeoVzVe2tqkur6nxgL3DTsG1J0nRb1edJSU4CrgKeAT5eVQ8N2pUkTble4dyF8bnDtiJJ2scXoUhSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1qNdf307yEuBTwJHAHPC2qto9ZGOSNM36jpwvAu6qqnOBu7t1SdJA+obzA4xGzQBrgceGaUeSBP3DeQewMcl9wJnA9vGdSS5OsmVubm6p/UnSVOobzu8GtlbVq4DrgXeM76yqrVW1ec2aNUvtT5KmUt9wDvB4t7wLMIUlaUC97tYArgY+meSdwB7gN4drSZLUK5yr6gngwoF7kSR1fBGKJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqUK9wTnJRkm3dv4eTvHnoxiRpmq3q86SquhG4ESDJXcCtQzYlSdNuSdMaSU4GHq2q7w3UjySJpc85/xrw2fENSTYn2ZFk5+zs7BIPL0nTaanhfDFw3fiGqtpSVadU1bqZmZklHl6SplPvcE7yMuCHVfXYgP1IkljayPlNwLVDNSJJelavuzUAqup/DtmIJOlZvghFkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIa1PsVgtJKSCZfs2ryNSVHzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWQe/I5+Gv/w/o4/Si4ThrIPf2btg02OwcddKdyIdMMNZB783PvLcj9KLQO83PkryLuDdwKHA26vqocG6kpbir++FM594dv2H3bslbZiDL2x7dvs9a+GPzphoa9KB6hXOSU4Czqmq8wfuR1q6q14OG56EI54ZrR9Wz/0IMH8IXPnyyfcmHaC+0xoXAocm+XySjyQ5dHxnkouTbJmbm1t6h9ILde+x8CenjgJ4f+YPgQ+cCl89drJ9SS9A33A+ATisGzn/AHjT+M6q2lpVm9esWbPU/qR+7j0W/uLnYfeCL/Hdh4y2G8xqXN9wngNu75ZvA145TDvSgI56GvYy+jd/yLPLR3lLndrXN5y3A6d1y2cA/zxMO9KAfuWR0bzzA0fBn24YfTziGe/a0ItCrwuCVXVvkqeSbAN2AVcM2pU0hO8fCv/jZ+Hvfxoq8N5j4dcfhNO8FqL29b6VrqouG7IRaXD/6dTnrj8T+NQMfGpl2pFeCF+EIkkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWpQ7xehSCuhavHHSAcDR86S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJalCvcE6yPsmjSbYluXnopiRp2i3lXeluqap3DNaJJOlHljKtcV6SO5K8b7BuJElA/3B+GPg54DzggiSn7duRZHOSHUl2zs7ODtGjJE2dXuFcVbur6vtV9TTwOWDD2L4tVXVKVa2bmZkZqk9Jmip9LwgePbb6WuCBYdqRJEH/aY1NSb6cZDvwUFXdNWRTkjTtet2tUVU3ADcM3IskqeOLUCSpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoOWFM5J3pfkzqGakSSN9A7nJIcDZwzYiySps5SR83uAjw3ViCTpWb3COclq4Nyquu159l+cZMvc3NySmpOkadV35PxO4Orn21lVW6tq85o1a3oeXpKmW99wfgXw3iQ3Aq9KcumAPUnS1FvV50lV9f59y0nurKqPDNeSJGnJ9zlX1euGaESS9CxfhCJJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhrUK5yTbEiyPckdSf4mSYZuTJKmWd+R8/1VdXZVberWzxyqIUlSz3Cuqj1jq7uBB/etJNmcZEeSnbOzs0vtT5KmUu855ySXJPkacALw2L7tVbWlqk6pqnUzMzND9ChJU6d3OFfVdVW1AfgW8KvDtSRJ6ntB8PCx1SeBp4ZpR5IEsKrn8y5K8u+75R3AzQP1I0miZzhX1bXAtQP3Iknq+CIUSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAb1CuckZyXZnuTOJFcM3ZQkTbu+I+dvAq+vqtcBL01y6oA9SdLUW9XnSVX1yNjqHmDvMO1IkmCJc85JTgPWVdU/Ldh+cZItc3NzS2pOkqZV73BOchzwUeA9C/dV1daq2rxmzZql9CZJU6vvBcFVwJXAZQumOCRJA+g7cn4r8Brg8iTbkmwcsCdJmnp9Lwj+LfC3A/ciSer4IhRJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktSgXuGc5MQkX0kyn2TV0E1J0rTrO3J+HDgf+NKAvUiSOr1GvVU1D8wnGbgdSRIsw5xzks1JdiTZOTs7O/ThJWkqDB7OVbWlqk6pqnUzMzNDH16SpoJ3a0hSg/rerbE6ya3A6cBNSc4ati1Jmm59LwjuAS4YuBdJUsdpDUlqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJalCqavkOnuwEvtnjqTPASryl3RpgbgXqrsT5TtO5wsqc7zSdK/h1fKBeXlXrFnvQsoZzX0l2Hkjzy1B3S1VtXoG6Ez/faTrXru7Ez3eazrWr69fxgFqd1nhihepuXaG6K3G+03SusDLnO03nCn4dD6rVcF6JX1OoqpX6j574+U7TucKKne80nSv4dTyoVsN5y0o3MGHTdL6e68Frms532c+1yTlnSZp2rY6cJWmqNRfOSU5M8pUk80l6/TGAHjXPSrI9yZ1JrphEza7uhq7uHUn+JhP8c+ZJ3pfkzgnWW5/k0STbktw8qbpd7Xcl+XxX+6QJ1Luoq7UtycNJ3rzcNbu6L0lyfVf32iSHT6juqiTXJPlCksuXuda/yockV3TfQx+eVN1J5FRz4Qw8DpwPfGmCNb8JvL6qXge8NMmpE6p7f1WdXVWbuvUzJ1G0+6Y9YxK1Frilqs6tqjdMqmAXxudU1fld7YeWu2ZV3djVOpfRvbC3LnfNzkXAXV3du7v1SXgL8NWqOg/4iSSnL2Ot5+RDklcDR3XfQ4clec0k6u5nfXDNhXNVzVfVdyZc85Gqmu9W9wB7J1R3z9jqbuDBSdQF3gN8bEK1xp3XjXDeN8GaFwKHdiPnjyQ5dFKFk5wMPFpV35tQyQeAI7vltcBjE6p7MvCP3fK9wNnLVWg/+fBLwC3d8q3AxknUnURONRfOKynJacC6qvqnCda8JMnXgBOYwDdTktXAuVV123LXWuBh4OeA84ALus/1JJwAHFZV5wM/AN40oboAvwZ8doL1dgAbk9zH6Lew7ROqez9wTrd8HqMfDJOyFniyW56bcO1lZTh3khwHfJTRqHJiquq6qtoAfAv41QmUfCdw9QTqPEdV7a6q71fV08DngA0TKj0H3N4t3wa8ckJ1AS4GrptgvXcDW6vqVcD1wDsmVHcro+mMzzP6DfDRCdWF0f/vMd3yMazcC38GZzgzuqABXAlcVlWPTLDu+AWbJ4GnJlD2FcB7k9wIvCrJpROoSZKjx1Zfy+hX8EnYDuwbpZ8B/PMkiiZ5GfDDqprU1AJAGM2FAuxi9L4Ty66q9lbVpd1vJ3uBmyZRt/NFRnO/ABcw2WtVy6q5cE6yOsmtwOnATUnOmkDZtwKvAS7vrnQvy7zVflyU5PYktzP69XvZ72KoqvdX1YVVdRFwX1V9ZLlrdjYl+XKS7cBDVXXXJIpW1b3AU0m2Mfo//vQk6jKaPrl2QrX2uRp4W3eubweumkTRJCd13ze3AduX86LrwnwAVgPzSe4A9lbV3ZOo293htaw55YtQJKlBzY2cJUmGsyQ1yXCWpAYZzpLUIMNZkhpkOEtSgwxnSWrQ/wcH+j8cfc2HfAAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x432 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"np.random.seed(0)\n",
"nvmdp = NavigationWorldMDP(width=11, height=11, \n",
" nav_cell_types=['white'],\n",
" nav_cell_rewards=[0],\n",
" nav_cell_p_or_locs=[1.],\n",
" goal_cell_locs=[[(6,6)]],\n",
" goal_cell_rewards=[1.],\n",
" goal_cell_types=[\"blue\"],\n",
" slip_prob=0.00, step_cost=0.01, gamma=.99)\n",
"nvmdp.visualize_grid()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"S = nvmdp.get_reachable_states()\n",
"A = nvmdp.get_actions()\n",
"s_to_idx = {s: idx for idx, s in enumerate(S)}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# High Temperature"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjYAAAIVCAYAAADCjrWtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+wrXV9H/r3hwMHIpooQpMIKtieZPxNGgQ1MWJAwZsANqMJbY12ximjN2bumLET096alt7OtGam3k7iTWSmdjRqiDGTeIhEKqKOXqqCBrWQy4DE8MPE8Eu04Dlw9v7cP/bCLrbnnL332nud9ay1Xq+ZZ3h+rOd5Puths/eHz/P9Ud0dAIBFcNSsAwAA2CkSGwBgYUhsAICFIbEBABaGxAYAWBgSGwBgYUhsgMOqqlOrqqvq6FnHArARiQ0sgar6WFVdepD9F1XV30pagEUhsYHl8N4kr62qWrf/l5N8oLsPzCAmgB0nsYHl8KdJnpzkJY/uqKonJfn5JO+rqp+rqr+oqm9X1R1V9W8OdaGq+npVnTu2/W+q6v1j2y+sqmur6ltV9eWqOnsaXwjgYCQ2sAS6+7tJPpTkdWO7fzHJ/9fdX07y4OjYE5P8XJI3VdWrtnqfqjo5yUeT/F9JTkjy1iR/XFUnbe8bAGyOxAaWx3uTvLqqjhttv260L939qe7+anevdvdXkvxBkpdOcI/XJrmyu68cXevjSa5P8r/tQPwAG5LYwJLo7s8muSfJq6rq7yc5M8kHk6SqzqqqT1bV3VX1QJI3Jjlxgts8PclrRq+hvlVV30ry00l+dGe+BcDh6QkBy+V9WavU/HiSq7r7m6P9H0zyO0le2d37qur/zqETmweTPG5s+0fG1u9I8vvd/c93NmyAzVGxgeXyviTnJvnnGb2GGnlCkvtGSc2ZSf7JYa5xQ5KLq+qYqjojyavHjr0/yQVVdV5V7aqq46rq7Ko6ZYe/B8BBSWxgiXT315Ncm+T4JHvHDv3vSS6tqu8keXvWGhofyr9O8veT3J/k32b0Omt0/TuSXJTkXya5O2sVnH8Rv2uAI6S6e9YxAADsCP8XBQAsDIkNALAwJDYAwMKQ2AAAC2Oq49jsrmP7uBw/zVvA9K2fNhKSRL8L5ty+PJiHe38lyXkvO77vvW9lout88Sv7r+ru83c0uG2YamJzXI7PWXXONG/B903WPHA1f0XCOmrOnjFHRK/OWWbTq7OOYOv02p2qz/cnvrd+730r+cJVT5voOrt+9JZJRimfmvn7KwMAcAimVACAJddJVjOHVb2DULEBABaGig0ALL3Oyjy2wzoIFRsAYGGo2ADAkltrY7MYvdBUbACAhaFiAwAsTK8oiQ0ALLlOZ2VBBkT0KgoAWBgqNgCAxsMAAEOjYgMAS66TrKjYAAAMi4oNAKCNDQDA0KjYAMCS68Q4NgAAQ7NhYlNVT6mqL1XVvqo6ev32kQgSAJiu1QmXodlMxea+JOck+dwhtgEABmHDikt370uyr6oOug0AzLdOG8fmUKrqkqq6paru3pcHd/ryAACHtOOJTXdf1t17uvuk43L8Tl8eANhpnaxMuAyNXlEAwMLYTK+oY6rq6iTPT3JVVZ21fnvaQQIA09NZnF5Rm2k8/EiSc9ftXr8NADBzxqEBgKVXWcli9HbWxgYAWBgqNgCw5DrJ6gB7OE1CYgMAeBUFADA0KjYAsOQ6KjYAAIOjYgMAZLVVbAAABkXFBgCWnDY2AAADpGIDAEuuU1lZkFrHYnwLAICo2AAA0SsKAGBwVGzG1RxmqzVfuWnt2jXrELasds3XM06SzNtzXlmZdQRbt7I66wi2pOfwESfz9YzT8zuLpF5RAAADpGIDAEuvstKLUetYjG8BABAVGwBYep1kdUFqHYvxLQCAQaqq86vq5qq6taredpDjb6yqr1bVDVX12ap61tix3xidd3NVnbeZ+6nYAABT6RVVVbuSvCvJy5PcmeS6qtrb3TeNfeyD3f17o89fmOQ/JTl/lOBcnOTZSZ6S5Oqq+rHuw/fxU7EBAKblzCS3dvdt3f1wksuTXDT+ge7+9tjm8Vl7M5bR5y7v7v3d/VdJbh1d77BUbABgyXVvq1fUiVV1/dj2Zd192Wj95CR3jB27M8lZ6y9QVb+S5NeS7E7ys2Pnfm7duSdvFIzEBgDI6uSvou7p7jO2c+/ufleSd1XVP0nyfyZ5/aTX8ioKAJiWu5I8dWz7lNG+Q7k8yasmPDeJxAYAlt7alApHTbRs4Loke6rqtKranbXGwHvHP1BVe8Y2fy7JLaP1vUkurqpjq+q0JHuSfGGjG3oVBQBMRXcfqKo3J7kqya4k7+nuG6vq0iTXd/feJG+uqnOTPJLk/oxeQ40+96EkNyU5kORXNuoRlUhsAIApTqnQ3VcmuXLdvrePrf8fhzn33yf591u5n1dRAMDCULEBgCVnSgUAgAFSsQEAstI7P6XCLKjYAAALQ8UGAJZcpzYzJs1cWIxvAQCQTSQ2VfWUqvpSVe2rqqNH+95ZVZ+pqv88/RABgGlb7aMmWoZmMxHdl+ScjGbYrKp/mOTx3f2SJLur6gVTjA8AYNM2bGPT3fuS7Kv6XmvpFyb5+Gj96iQvytpcEADAHHp0rqhFMMm3eGKSb4/WHxhtP0ZVXVBVlx3Iw9uJDQBgSyZJbB5I8oOj9R9M8q31H+juK7r7kqOzezuxAQBHQKey0pMtQzNJYvPfs9bmJknOzajtDQDArG2mV9QxVXV1kudnbdrxY7LW5uYzSVa6+wtTjhEAmLLVHDXRMjSbaTz8SNYqM+M+P51wAAAmZ+RhAFhy3cnKAMekmYTEBgCWXmU1w2sIPInFSM8AAKJiAwBLr7M4r6IW41sAAETFBgDIck+pAAAwSCo2ALDkOpXVAU6PMAkVGwBgYajYAADa2AAADI2KDQAsuU6yahwbAIBhUbEBgKVXWTFXFADAsKjYAMCSW6Q2NtNPbGqOSls1f/9Sa9euWYewJXXcsbMOYcuOOv5xsw5h6+btOe/bP+sItmz1wYdmHcLWzOEz7pVZR7BVq7MOYGt61gFMh4oNAKCNDQDA0KjYAMCS666FaWOzGN8CACAqNgBAkpUFqdhIbABgyXWSVY2HAQCGRcUGAJZeLcyrqMX4FgAAUbEBgKW3NqWCNjYAAIOiYgMAZGVBah2L8S0AAKJiAwBLr1Pa2AAADI2KDQCQ1QWpdSzGtwAAiIoNACy97mRFGxsAgGFRsQEAlrtXVFUdXVWXV9Unq+odOx0UAMAkJn0V9Y+SfLm7X5bkB6rq+TsYEwBwBK2NY3PURMvQTBrRM5J8ZbR+Q5IX70w4AACTmzSxuTnJS0frL0vyxEcPVNUlVXVLVd29Lw9tNz4A4AhYSU20DM2kic0VWXsF9Ykk+5N889ED3X1Zd+/p7pOOy+N2IkYAgE2ZqFdUd68k+dUkqarLkly1k0EBAEdOZ3F6RU2U2FTVyUk+kGQ1yfu6+64djQoAYAKTVmzuSnL2zoYCAMxGDbKH0yQM0AcAZHWADYEnsRjpGQBAVGwAYOmZBBMAYIAkNgDA1KZUqKrzq+rmqrq1qt52kOO/VlU3VdVXquoTVfX0sWMrVXXDaNm7me/hVRQAMBVVtSvJu5K8PMmdSa6rqr3dfdPYx/4iyRnd/VBVvSnJO5L80ujYd7v79K3cU8UGAJbc2iSYky0bODPJrd19W3c/nOTyJBc95t7dn+zuR+dg+lySU7bzXSQ2AMB2nFhV148tl4wdOznJHWPbd472Hcobkvz52PZxo2t+rqpetZlgvIoCALYzjs093X3Gdu9fVa9Nckb+1yTbSfL07r6rqp6R5Jqq+mp3f+1w11GxAQCm5a4kTx3bPmW07zGq6twk/yrJhd29/9H9j07Z1N23JflUkp/Y6IYSGwBYco9OgjmFNjbXJdlTVadV1e4kFyd5TO+mqvqJJO/OWlLzd2P7n1RVx47WT0zyU0nGGx0flFdRAMBUdPeBqnpzkquS7Erynu6+saouTXJ9d+9N8ltJHp/kj6oqSW7v7guTPDPJu6tqNWuFmP+wrjfVQUlsAICpTYLZ3VcmuXLdvrePrZ97iPOuTfLcrd7PqygAYGGo2ADAsttce5m5oGIDACyM6Vdsan5yp9q1a9YhbFkdd+ysQ9iSo0544qxD2LKHn3birEPYsu88bb5+Lp5w+/6NPzQwu2+/Z9YhbMnqfd+adQhbt2++fi56ZdYRTK6zrXFsBmV+sg4AgA1oYwMAaGMDADA0KjYAsOQeHXl4EUhsAICFSWy8igIAFoaKDQAsuY4B+gAABkfFBgAwQB8AwNCo2ADAsmu9ogAABkfFBgCW3CIN0KdiAwAsDBUbAEDFBgBgaFRsAGDJGXkYAGCAVGwAgPSCVGwmSmyq6nFJ/ijJ8UkeSPKL3b1/JwMDANiqSV9FnZ/k8919dpIvjLYBgDm1mppoGZpJE5uvZa1akyRPTHLvzoQDADC5SdvY3JLkRVV1Y5K/S/Lr4wer6oIkF/xAHr/N8ACAaWtzReX1Sa7o7mcn+WiS144f7O4ruvuSo3PMduMDANi0SSs2leS+0fo9SX5oZ8IBAGZhqXtFJflgkj+sql9O8kiSX9q5kACAI2txBuibKLHp7m8lOW+HYwEA2BYD9AEAC/MqypQKAMDCULEBgCXX0d0bAGBwVGwAYNn12iB9i0DFBgBYGCo2AMAgJ7SchIoNALAwVGwAYMl1jGMDADA4KjYAsPQWZ64oFRsAYGGo2AAAxrEBABgaFRsAQK8oAIChmW7FppI6an4ywNo1f3neUcc/btYhbMnDTztx1iFs2dd/7rhZh7Blt7zud2cdwpbsed+bZh3Clp360fn6Wd69/+FZh7Blq488MusQtqZXZx3B1oyF261iAwAwONrYAAALM46NxAYA0N0bAGBoVGwAAI2HAQCGRsUGAJZcp1RsAACGRsUGAMiCdIpSsQEAFoeKDQAsO1MqAAAMj4oNALAwjWxUbACAhaFiAwBoYwMAMDQSGwAg3ZMtG6mq86vq5qq6taredpDjv1ZVN1XVV6rqE1X19LFjr6+qW0bL6zfzPSQ2AMBUVNWuJO9K8sokz0ryj6vqWes+9hdJzuju5yX5cJJ3jM49IclvJjkryZlJfrOqnrTRPSU2ALDkOmttbCZZNnBmklu7+7bufjjJ5Ukuesy9uz/Z3Q+NNj+X5JTR+nlJPt7d93X3/Uk+nuT8jW4osQEAtuPEqrp+bLlk7NjJSe4Y275ztO9Q3pDkzyc8N8mEvaKq6vwkj74n+/Ekb+ruP53kWgDAjHWSyXtF3dPdZ2w3hKp6bZIzkrx0O9eZqGLT3R/r7rO7++wktye5ejtBAAAL6a4kTx3bPmW07zGq6twk/yrJhd29fyvnrretV1FV9Ywk3+zu/zm275JR6+W7933vlRkAMGRT6hV1XZI9VXVaVe1OcnGSveMfqKqfSPLurCU1fzd26Kokr6iqJ40aDb9itO+wttvG5heS/Mn4ju6+rLv3dPdJx9Xjtnl5AOCI6AmXw12y+0CSN2ctIfnLJB/q7hur6tKqunD0sd9K8vgkf1RVN1TV3tG59yX5d1lLjq5Lculo32Ftd+ThC7KW3AAAfJ/uvjLJlev2vX1s/dzDnPueJO/Zyv0mTmyq6keSPNzd9056DQBgCDbVdXsubOdV1EVJPrJTgQAAbNfEFZvufvdOBgIAzNAmpkeYBwboAwAWxnYbDwMA866jjQ0AwNCo2AAA2tgAAAyNig0AkEQbGwCAQVGxAQC0sQEAGBoVGwBAxQYAYGhUbABg2XUSIw8DAAyLig0AkNbGBgBgWFRsAICF6RUlsRm3a9esI9i6446ddQRb8p2nzVe8SXLL63531iFs2XlPOX3WIWzJLd+Yv2f8wq++cdYhbMmT75i///bm7nfygQOzjmB7NB4GABgWFRsAILUgr6JUbACAhaFiAwDLrrMwjYdVbACAhaFiAwBLr/SKAgAYGhUbAEAbGwCAoVGxAQBUbAAAhkbFBgBQsQEAGBoVGwBYdh3j2AAADI2KDQBgdm8AgKGR2AAbOnD87nz5374yB47fPetQgGnpCZeBkdgAG7r7xafm7pc8I3e/6NRZhwJwWBMnNlX1uqr6RFV9qqpO3smggGH5xvnPXPvnK58540gADm+ixsOjROal3X3ODscDDMAXf+vC3P+TT/3edj28kiT51rN/NFdf8yvf2/+kL96Rn/wXe494fMDOW/bGw+cl2TWq2Px2Ve3ayaCA2TrtA1/MUfse+d527971mH8myVH7Hslp77/+iMcGcDiTJjY/nGT3qGLzUJKLxg9W1QVVddmBfuSgJwPDdsINd+X0f/nRxyQ3447a90hO/40/ywlf/sYRjgyYmq7JloGZNLF5IMmnR+vXJHnMi/fuvqK7Lzm6jtlObMAMnXDDXXnupVflqP0HHrP/qP0H8txLr5LUAIM0aWJzbZLnjdZPT/JXOxMOMCQHHn9samU1WVldq96srKZWVnPg8cfOOjRgJ03a1XuA7XImSmy6+4Yk362qTyV5QZIP72RQwDB845XPzMpxx+QJt92b5//rK/OE2+7NynHH6B0FDNbEUyp091t3MhBgeHY9+HD2/N7/m6f98ZdTnZzwpj/K7b/w/Nz/vB+ddWjAThtg9WUS5ooCDun0t//5Y7ZrtfP0D9+Qp3/4hhlFBHB4EhsAYOnHsQEAGBwVGwBgYdrYqNgAAAtDxQYAULEBABgaFRsAWHLVekUBAAyOig0AMMiZuiehYgMALAwVGwBArygAgKFRsQEAFqZXlMQGAPAqCgBgaFRsxq2szDqCrdu3f9YRbMkTbp+veJNkz/veNOsQtuyWb/zurEPYknl8xqfevm/WIWzNnP2uSDKfv5PnlQH6AAA2VlXnV9XNVXVrVb3tIMd/pqq+VFUHqurV646tVNUNo2XvZu6nYgMATKWNTVXtSvKuJC9PcmeS66pqb3ffNPax25P8syRvPcglvtvdp2/lnhIbAGBazkxya3ffliRVdXmSi5J8L7Hp7q+Pjq3uxA29igIA1io2kyzJiVV1/dhyydhVT05yx9j2naN9m3Xc6Jqfq6pXbeYEFRsAYDvu6e4zpnTtp3f3XVX1jCTXVNVXu/trhztBxQYASPVkywbuSvLUse1TRvs2pbvvGv3ztiSfSvITG50jsQEApuW6JHuq6rSq2p3k4iSb6t1UVU+qqmNH6ycm+amMtc05FIkNADAV3X0gyZuTXJXkL5N8qLtvrKpLq+rCJKmqF1TVnUlek+TdVXXj6PRnJrm+qr6c5JNJ/sO63lQHpY0NADA13X1lkivX7Xv72Pp1WXtFtf68a5M8d6v3k9gAAOaKAgAYGhUbAFh25ooCABgeFRsAQBsbAIChUbEBAFRsAACGRsUGAJZcRa8oAIDBUbEBABamjc1EiU1VnZrk81mb0Orh7n7FDsYEABxJCzRA33YqNh/v7tfuWCQAANu0nTY2L6uqz1TVW8Z3VtUlVXVLVd29rx/aZngAwBHREy4DM2li8zdJfizJy5KcW1XPe/RAd1/W3Xu6+6Tj6nE7ESMAwKZMlNh09/7ufrC7DyT5syTP2dmwAIAjapkrNlX1hLHNn0rytZ0JBwBgcpO+inpJVX2xqq5Ncld3f34ngwIAjqzqyZahmahXVHdfmeTKHY4FAGBbDNAHAAyyvcwkTKkAACwMFRsAWHYD7eE0CRUbAGBhqNgAAIPs4TQJFRsAYGGo2AAA2tgAAAyNig0AoI0NAMDQqNgAANrYAAAMjYoNACy7BRp5WGIDAEuuRssi8CoKAFgYKjYAgFdRm9JJr87Rk1pZnXUEW7b64EOzDmFLdt9+z6xD2LJTP3rirEPYshd+9Y2zDmFLTr1936xD2LJ5+1met98VSdJz9jt5rv7eJQuTyKynYgMAGKAPAGBoVGwAgIV5NaViAwAsDBUbAEDFBgBgaFRsAGDZtV5RAACDo2IDAGhjAwAwNCo2AIA2NgAAQ6NiAwBoYwMAMDQqNgCANjYAAEOjYgMAy66zMG1sJDYAwMIkNl5FAQALY1uJTVW9pao+u1PBAABHXmWt8fAky9BMnNhU1bFJTt/BWAAAtmU7FZs3JHnvTgUCAMxQT7gMzESJTVUdk+Ts7r7mEMcvqKrLDuSRbQUHALAVk1ZsfjnJBw91sLuv6O5Ljs4xE14eADiSqnuiZWgmTWx+PMmbqupjSZ5dVb+6gzEBAExkonFsuvvXH12vqs9292/vXEgAwBE10PYyk9j2ODbd/dM7EQgAwHYZeRgAGOSYNJMw8jAAMDVVdX5V3VxVt1bV2w5y/Geq6ktVdaCqXr3u2Our6pbR8vrN3E/FBgCYShubqtqV5F1JXp7kziTXVdXe7r5p7GO3J/lnSd667twTkvxmkjNG0X1xdO79h7unig0AMC1nJrm1u2/r7oeTXJ7kovEPdPfXu/srSVbXnXteko93932jZObjSc7f6IYqNgDAdtrYnFhV149tX9bdl43WT05yx9ixO5OctcnrHuzckzc6SWIDAGzHPd19xqyDeJRXUQDAtOaKuivJU8e2Txnt24yJzpXYAADTcl2SPVV1WlXtTnJxkr2bPPeqJK+oqidV1ZOSvGK077AkNgCw7Hqtjc0ky2Ev230gyZuzlpD8ZZIPdfeNVXVpVV2YJFX1gqq6M8lrkry7qm4cnXtfkn+XteTouiSXjvYdljY2AMDUdPeVSa5ct+/tY+vXZe0108HOfU+S92zlfhIbAGBh5oqS2ADAkquYUgEAYHBUbACApBejZKNiAwAsDBUbAGBh2thMP7Hp9XNaDVevzDqCCezbP+sItmT1vm/NOoQt273/4VmHsGVPvuPYWYewNXP2c5wkqw8+NOsQtqTn8Bn3ypz9Up6jv3eLTMUGAJbd5qZHmAva2AAAC0PFBgBILcibNBUbAGBhqNgAANrYAAAMjYoNALAw49io2AAAC0PFBgCWXcdcUQAAQ6NiAwBoYwMAMDQqNgCAcWwAAIZGxQYAllxlcdrYSGwAYNl16+4NADA0KjYAwMK8ilKxAQAWhooNALDc3b2r6jlVdW1Vfaaq/mtV1U4HBgCwVZO+irq5u1/c3S8ZbZ+xUwEBAEde9WTL0EyU2HT3I2Ob+5Pc8ehGVV1SVbdU1d378tB24wMA2LSJGw9X1YVV9T+S/HCSex/d392Xdfee7j7puDxuJ2IEAKapk6z2ZMvATJzYdPfe7n5OkjuT/PzOhQQAMJmJekVV1bHdvX+0+e0k3925kACAI254xZeJTNrd+/yq+rXR+i1J/tsOxQMAMLGJEpvu/kiSj+xwLADAjAyxh9MkjDwMACwMIw8DAGb3BgAYGhUbAEAbGwCAoVGxAYBl11mYcWxUbACAhaFiAwBLrpLUgvSKktgAAMnqrAPYGV5FAQALQ8UGAFiYV1EqNgDAwlCxAYBlp7s3AMDwqNgAwNJrk2ACAAzN9Cs2c5UBzl8n/l6ZdQRbtG//rCPYstVHHpl1CFu3a9esI9ialXn7QU56Zb5+X/QcPuP0fD3j+fp79/1MggkAMDDa2AAAc19xepSKDQCwMFRsAGDZdVJz1qTpUFRsAICFoWIDAGhjAwAwNCo2AIC5ogAAhkbFBgBIaWMDADAsKjYAgF5RAMCC6KzNAz3JsoGqOr+qbq6qW6vqbQc5fmxV/eHo+Oer6tTR/lOr6rtVdcNo+b3NfBUVGwBgKqpqV5J3JXl5kjuTXFdVe7v7prGPvSHJ/d39D6rq4iT/MckvjY59rbtP38o9VWwAYMlVOtWTLRs4M8mt3X1bdz+c5PIkF637zEVJ3jta/3CSc6qqJv0uEhsAYDtOrKrrx5ZLxo6dnOSOse07R/tysM9094EkDyR58ujYaVX1F1X16ap6yWaC8SoKANhO4+F7uvuMnQxl5G+SPK27762qn0zyp1X17O7+9uFOUrEBAKblriRPHds+ZbTvoJ+pqqOT/FCSe7t7f3ffmyTd/cUkX0vyYxvdUGIDAKxVbCZZDu+6JHuq6rSq2p3k4iR7131mb5LXj9ZfneSa7u6qOmnU+DhV9Ywke5LcttENJ3oVVVVnJXln1jp6Xdfdb5nkOgDA4uruA1X15iRXJdmV5D3dfWNVXZrk+u7em+S/JPn9qro1yX1ZS36S5GeSXFpVj2Qt33hjd9+30T0nbWPz10l+trv3VdUHquq53f3VCa8FAMzSo+PYTOPS3VcmuXLdvrePre9L8pqDnPfHSf54q/ebKLHp7r8d23wkycok1wEA2Enb6hVVVc9LctK6gXZSVRckueAHcvx2Lg8AHCFLPwlmVZ2Q5HeyNmLgY3T3Fd19ydHZvZ3YAAC2ZNLGw0cneX+St657LQUAzKMlr9i8JskLkryjqj5VVS/awZgAACYyaePhP0jyBzscCwAwE5sak2YuGKAPAFgY5ooCgGXXUbEBABgaFRsAYGojDx9pKjYAwMJQsQEAjDwMADA0KjYAwML0ipLYAMCy6ySri5HYeBUFACwMFRsAWHqmVAAAGBwVGwBAxQYAYGhUbAAAFRsAgKFRsRk3l9nqfM1a1iuzjmACPV/POEly4MCsI1h4PW9jfszjz/Fc/k6eU8axAQAYHhUbAFh6PZ9VvYNQsQEAFoaKDQCwMG2aVGwAgIWhYgMAy06vKACA4VGxAQC0sQEAGBoVGwBAxQYAYGhUbABg6fXCVGwkNgCw7DrJqikVAAAGRcUGAFiYV1EqNgDAwlCxAQBUbAAAhkbFBgCWXi/3JJhV9ZSq+lJV7asqyREAMAiTJiX3JTknyZ/sYCwAwCx00r0Y49hMlNh0974k+6pqh8MBAJjcjjcerqpLquqWqrp7Xx7c6csDANOw2pMtA7PjiU13X9bde7r7pONy/E5fHgDgkDT8BQCWexybqjqmqq5O8vwkV1XVWTsbFgDA1k3aePiRJOfucCwAwCx0m90bAGBotLEBAJa7jQ0AwBCp2AAAaW1sAACGRcUGAJZeL0wbG4kNACy7ziCnR5iEV1EAwMJQsQEAktZ4GABgUFRsAGDJdZLWxgYAYFhUbABg2XVrYwMAMDQSGwAgvdoTLRupqvOr6uaqurWq3naQ48dW1R+Ojn++qk4dO/Zh7vKoAAAF10lEQVQbo/03V9V5m/keEhsAYCqqaleSdyV5ZZJnJfnHVfWsdR97Q5L7u/sfJHlnkv84OvdZSS5O8uwk5yf5f0bXOyyJDQCw1sZmkuXwzkxya3ff1t0PJ7k8yUXrPnNRkveO1j+c5JyqqtH+y7t7f3f/VZJbR9c7rKk2Hv5O7r/n6v7wX0/h0k9LcvsUrjstP5TkgalceXq98zzjR61M5aqJZ3wkeMbTNW/PN/GMxz390ZXv5P6rru4PnzjhdY6rquvHti/r7stG6ycnuWPs2J1Jzlp3/vc+090HquqBJE8e7f/cunNP3iiYqSY23X3SNK5bVXd39xnTuPY0VNVl3X3JrOPYCs94+jzj6fOMp2venm/iGR9Kd58/7XscKfP6Kupbsw5gi66YdQAT8IynzzOePs94uubt+Sae8ZF2V5Knjm2fMtp30M9U1dFZq6rdu8lzv8+8JjbzVEZMd8/bf0iJZ3wkeMbT5xlP11w938QznoHrkuypqtOqanfWGgPvXfeZvUleP1p/dZJrurtH+y8e9Zo6LcmeJF/Y6IbzOkDfZRt/hG3yjKfPM54+z3i6PN/pm+tnPGoz8+YkVyXZleQ93X1jVV2a5Pru3pvkvyT5/aq6Ncl9WUt+Mvrch5LclORAkl/p7g1bPdZaUgQAMP/m9VUUAMD3mavEpqqeUlVfqqp9owZGg1ZVZ1XVtVX12ap656zj2Yyqes4o5s9U1X8djSUweFX1lqr67Kzj2IyqOrWqvllVn6qq/zbreDajql5XVZ8Yxbxhd8tZG410+qnR8jdV9apZx3Q4VfW4qvroKN6PVNWxs47pcKrq6Kq6vKo+WVXvmHU8h3OwvxtV9c7R77j/POv41lsf77z93RuCuUpssvbu7Zw8tl/7kP11kp/t7p9O8veq6rmzDmgTbu7uF3f3S0bbg+/KOfojcPqs49iij3f32d39ilkHspFRIvPS7j5nFPOGvRJmrbs/Nor17KyNAXL1jEPayPlJPj+K9wuj7SH7R0m+3N0vS/IDVfX8WQd0GI/5u1FV/zDJ40e/43ZX1QtmGdxBrP87N29/92ZurhKb7t7X3ffPOo7N6u6/7e59o81HMs2h3nZIdz8ytrk/jx1YaajekP81auW8eNno/xjfMutANuG8JLtGFZvf3syQ5kNRVc9I8s3u/p+zjmUDX0ty/Gj9iVnr6jpkz0jyldH6DUlePMNYDusgfzdemOTjo/Wrk7zoyEd1aOvjnbe/e0MwV4nNvKqq5yU5qbtvmnUsm1FVF1bV/0jywxn4L9iqOibJ2d19zaxj2YK/SfJjSV6W5NzRz8eQ/XCS3d19TpKH8v3DoQ/ZLyT5k1kHsQm3JHlRVd2YtSrptTOOZyM3J3npaP1lWUvG5sUTk3x7tP5A5it2NkFiM2VVdUKS38laVWEudPfe7n5O1oav/vlZx7OBX07ywVkHsRWjeU8e7O4DSf4syXNmHdMGHkjy6dH6NUmeOcNYtuqCfP+YGUP0+iRXdPezk3w0yWtnHM9GrsjaK6hPZK2y+80Zx7MVDyT5wdH6D2b+B8BjHYnNFI0aer0/yVu7+29nHc9mrGu0+O0k351VLJv040neVFUfS/LsqvrVWQe0kap6wtjmT2XtNcSQXZvk0arS6Un+aoaxbFpV/UiSh7t70FXHkcpaW4okuSdrI68OVnevdPevjqp4K1kbo2Re/PestVlJknOj7crCmavEpqqOqaqrkzw/yVVVtX4iraF5TZIXJHnHqLfDoN7lHsL5VfXpqvp01l5BDLrXTnf/enefN5rn5Mbu/u1Zx7QJL6mqL1bVtUnu6u7Pzzqgw+nuG5J8t6o+lbWf5w/PNqJNuyjJR2YdxCZ9MMkvjp7xP03ygdmGc3hVdfLod9o1Sa4dcoPy9X83khyTZF9VfSbJSndvOJLtkXSwv3Nz9ndv5gzQBwAsjLmq2AAAHI7EBgBYGBIbAGBhSGwAgIUhsQEAFobEBgBYGBIbAGBh/P+DHhCXHckVBwAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 1440x1440 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"def get_value_grid(V_array, nvmdp, s_to_idx):\n",
" \"\"\"Returns value over states space grid.\n",
" \"\"\"\n",
" V = np.zeros((nvmdp.height, nvmdp.width), dtype=np.float32)\n",
" for row in range(nvmdp.height):\n",
" for col in range(nvmdp.width):\n",
" x, y = nvmdp._rowcol_to_xy(row, col)\n",
" V[row, col] = V_array[s_to_idx[NavigationWorldState(x, y)]]\n",
" return V\n",
" \n",
"## Greedy Policy\n",
"# v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, lambda Q: np.max(Q), 100)\n",
"## Boltzmann Policy\n",
"boltz_temp = 100\n",
"v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, \n",
" lambda Q, t=boltz_temp: np.exp((1/t)*Q) / (np.exp((1/t)*Q).sum()), 100)\n",
"V = get_value_grid(v, nvmdp, s_to_idx)\n",
"\n",
"plt.figure(figsize=(20,20))\n",
"nvmdp.visualize_grid(V, trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0.01, 0.01, 0.01, 0.01, 0.01, 0. , 0. , 0. ],\n",
" [0. , 0. , 0.01, 0.01, 0.02, 0.03, 0.02, 0.01, 0.01, 0. , 0. ],\n",
" [0. , 0.01, 0.01, 0.03, 0.06, 0.1 , 0.06, 0.03, 0.01, 0.01, 0. ],\n",
" [0. , 0.01, 0.02, 0.06, 0.16, 0.33, 0.16, 0.06, 0.02, 0.01, 0. ],\n",
" [0. , 0.01, 0.03, 0.1 , 0.33, 0. , 0.33, 0.1 , 0.03, 0.01, 0. ],\n",
" [0. , 0.01, 0.02, 0.06, 0.16, 0.33, 0.16, 0.06, 0.02, 0.01, 0. ],\n",
" [0. , 0.01, 0.01, 0.03, 0.06, 0.1 , 0.06, 0.03, 0.01, 0.01, 0. ],\n",
" [0. , 0. , 0.01, 0.01, 0.02, 0.03, 0.02, 0.01, 0.01, 0. , 0. ],\n",
" [0. , 0. , 0. , 0.01, 0.01, 0.01, 0.01, 0.01, 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ]],\n",
" dtype=float32)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V.round(2)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAIVCAYAAAAZNR+PAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X2QZXd5H/jvo5FGb7zIRgIHCQwkcioyQayLF/NmJEsxYm0h4sVeTMDOFllV7OBywbJr1tlQWeLd2pCqUA4miSdZO3ZsII5iFsnICASWA8gWyBhjC0eRjA2Sihe9oAEkjWbU/ewffUdpDdPTfW/36TNH9/OpOqV77r197qOjmb6Pvuf3+53q7gAATMUJYxcAADAPzQsAMCmaFwBgUjQvAMCkaF4AgEnRvAAAk6J5gQmqqqdVVVfViWPXsqiq+suqunj2+Oeq6t+OXRMwDZoXGEFVfbCq3naU5y+rqi+P3ZRU1aur6oaquq+qvjp7/FNVVUN8Xnf/393997Z7nEdDUwdsTvMC4/jVJK89SjPwuiS/0d0PjVBTkqSq/pckv5DknyX5jiRPSvL3k7woyd4NfmbPrhUILD3NC4zj/0vyhCQvOfxEVX1bkh9K8muz/R+sqj+qqq9X1W1V9Y83Otj6SzCz/X9cVb++bv97q+r6qrq3qv64qi7Y4DiPT/K2JD/V3Vd09zd6zR9199/p7gdn7/t3VfWvqurqqrovyYWb1VtVr6uqL1TV3VX1D494bcv1VtV1VfVPquoTVfWNqvpQVZ05e/k/z/55b1V9s6pesNE5A6ZL8wIj6O4Hkvxmkh9f9/SPJvkv3f3Hs/37Zq+fkeQHk/xkVb1y3s+qqrOTfCDJzyf59iRvTvKfquqso7z9BUlOTvL+LRz6NUn+rySPTfLxY9VbVecl+VdZS5aenLXG7Zxt1PuaJP9TkidmLQ168+z575v984zufkx3//4W/j2AidG8wHh+NcmrquqU2f6Pz55LknT3dd39J9292t2fTfKeJC9d4HNem+Tq7r56dqwPJ7kxyX9/lPeemeSu9Zet1iUgD1TV96177/u7+xOzYx7YpN5XJfnt7v7Ps/TmHyVZ3Ua9v9Ld/3VdE/jsuc8KMFmaFxhJd388yV1JXllVfzXJ85K8+/DrVfX8qvrdqrqzqvZnbdzJmUc/2jF9Z5IfmTUg91bVvUlenOSvHOW9dyc5c/2A1+5+YXefMXtt/e+M29b/4Cb1Pnn9+7v7vtnxFq33y+se35/kMRv9ywOPPpoXGNevZS1xeW2Sa7r7K+tee3eSK5M8pbsfn+RfJ9lots99SU5bt/8d6x7fluTfd/cZ67bTu/v/Ocpxfj/Jg0ku20LtR96S/lj1finJUw6/sapOy9qlo6OZp97NagIehTQvMK5fS3Jxkv856y4ZzTw2yT3dfaCqnpe1cR4b+UySV1fVSVX1nKxdpjns15NcWlUvq6o9VXVKVV1QVd8y5qS7703yfyb5l1X1qqp6bFWdUFXPTnL6Jv8ux6r3iiQ/VFUvrqq9WRsUvNHvny3XexR3Zu1y1DO28F5gojQvMKLu/ssk12etMbjyiJd/KsnbquobSd6atbEdG/lHSf5qkq9lrfl4+PJTd9+WtSTl57L25X5bkv81G/z97+63J3lTkv8tyVdm2y8l+dlZrRvZsN7uvinJP5jV9aVZnbdv8Plz1XvEz96ftUHEn5hdcvrezX4GmJ7qlrICANMheQEAJkXzAgBMiuYFAJgUzQsAMCmD3nl1b53cp2w6uxLYcYPc+3lA5g3ArjuQ+3KwH6wkedmFp/fd96wsdJw//OyD13T3JTta3CYGbV5Oyel5fl005EcwNd9yE+UJqOkFlHXCtM5zr06we+mN7m5wnDKzlCPc0B95+PHd96zkk9c8daHj7Pkrtyyy8ve2TO+3MgCw1AZNXgCA418nWd3wXqnHH8kLADApkhcAWHqdlQmN45K8AACTInkBgCW3NuZlOjPSJC8AwKRIXgCASc020rwAwJLrdFYmtJChy0YAwKRIXgAAA3YBAIYieQGAJddJViQvAADDkLwAAMa8AAAMRfICAEuuE+u8AAAMZdPmpaqeXFWfrqoDVXXikfu7USQAMKzVBbcxbCV5uSfJRUn+YIN9AIBds2ly0t0HkhyoqqPuAwDT1unlXuelqi6vqluq6s4DuW+nDw8ALLkdb166e193n9vdZ52S03f68ADATutkZcFtDGYbAQCTspXZRidV1bVJzk9yTVU9/8j9oYsEAIbTmdZso60M2D2U5OIjnj5yHwBgV1inBQCWXmUl05lFbMwLADApkhcAWHKdZHU6y7xoXgCAuGwEADAUyQsALLmO5AUAYDCSFwAgqy15AQAYhOQFAJacMS8AAAOSvADAkutUViaUZ0ynUgCASF4AgJhtBAAwGMnLejWdrvNhNa3+s/bsGbuEudVJ0/trUntPGruEufTBQ2OXMLc+9NDYJcylH5reOU5P6E6BE2e2EQDAgKb3v5QAwA6rrPR08ozpVAoAEMkLACy9TrI6oTxjOpUCAETyAgDEbCMAgMFIXgBgyXVPa7aR5gUAyKrLRgAAw5C8AMCSW7s9wHTyjOlUCgAQyQsA4PYAAADDkbwAwJJzewAAgAFJXgCArLR1XgAABiF5AYAl1ynrvAAADGXT5qWqnlxVn66qA1V14uy5d1TVx6rqF4YvEQAY2mqfsNA2hq186j1JLkryB0lSVd+T5DHd/ZIke6vquQPWBwDwCJuOeenuA0kOVD08Cvl7k3x49vjaJC9I8qlBqgMABrcM9zY6I8nXZ4/3z/Yfoaourap9D+XgdmoDAPgWizQv+5M8bvb4cUnuPfIN3X1Vd19+YvZupzYAYBd0Kiu92DaGRZqX38/aGJgkuTizsTAAALthK7ONTqqqa5Ocn+SaJCdlbQzMx5KsdPcnB64RABjYak5YaBvDVgbsHspawrLeDcOUAwBwbFbYBYAl152sjLRmyyKmUykAMJDK6oLbpkeuuqSqbq6qW6vqLcd43/9QVV1Vz9nsmJoXAGAQVbUnybuSvDzJeUl+rKrOO8r7HpvkZ7LFYSmaFwBYcp21y0aLbJt4XpJbu/vz3X0wyXuTXHaU9/2TJP80yYGt1Kt5AQC248yqunHddvm6185Octu6/dtnzz1sdtuhp3T3B7b6gQbsAgDbuT3AXd296TiVo6mqE5L88yR/d56fk7wAAEO5I8lT1u2fM3vusMcmeWaS66rqL7N2/8QrNxu0K3kBgCXXqawOs9T/p5KcW1VPz1rT8uokr3n4c7v3Jznz8H5VXZfkzd1947EOKnkBAAbR3Q8leUPWVuj/syS/2d03VdXbquoVix5X8gIAbGfMyzF199VJrj7iubdu8N4LtnJMyQsAMCmSFwBYcp1k1e0BAACGIXkBgKVXWdnCfYqOF5IXAGBSJC8AsOSmNuZF87JeTec/3GF10rT+E55w8sljlzC3evzjxi5hbqtPmFbNJ9z99bFLmNvqvfvHLmE+D6yOXcHcemVl7BLm0z12BUtjWt98AMAgjHkBABiI5AUAllx3TWrMy3QqBQCI5AUASLIyoeRF8wIAS66TrBqwCwAwDMkLACy9mtRlo+lUCgAQyQsALL212wMY8wIAMAjJCwCQlQnlGdOpFAAgkhcAWHqdMuYFAGAokhcAIKsTyjOmUykAQCQvALD0upMVY14AAIYheQEAHv2zjarqxKp6b1X9blW9faeLAgDYyKKXjf52kj/u7guTnFpV5+9gTQDALlpb5+WEhbYxLPqpz0jy2dnjzyR54c6UAwBwbIs2Lzcneens8YVJzjj8QlVdXlW3VNWdB3LfdusDAHbBSmqhbQyLNi9XZe1y0UeSPJjkK4df6O593X1ud591Sk7fiRoBAB620Gyj7l5J8tNJUlX7klyzk0UBALunM63ZRgs1L1V1dpLfSLKa5Ne6+44drQoAYAOLJi93JLlgZ0sBAMZRo80cWoRF6gCArI40+HYR02mzAAAieQGApefGjAAAA5K8AACTGrA7nUoBACJ5AYClt3ZjRmNeAAAGIXkBAKzzAgAwFMkLACy5qd2YUfICAEyK5AUAsM4LAMBQJC8AsOzaOi8AAIMZPnmp6XRytWfP2CXM7YSTTx67hLnUE75t7BLmdv93nTV2CXP74sunFao+9XdOG7uEuZ32p6tjlzCX1UOHxi5hbr3aY5cwn14Zu4KFdazzAgAwmGn97xkAMAhjXgAABiJ5AYAlN7UVdjUvAMCkmheXjQCASZG8AMCS61ikDgBgMJIXAMAidQAAQ5G8AMCya7ONAAAGI3kBgCU3tUXqJC8AwKRIXgAAyQsAwFAkLwCw5KywCwAwIMkLAJCeUPKyUPNSVacl+Y9JTk+yP8mPdveDO1kYAMDRLHrZ6JIkN3T3BUk+OdsHACZqNbXQNoZFm5c/z1rqkiRnJLl7Z8oBADi2Rce83JLkBVV1U5KvJvnZ9S9W1aVJLj314f4GADhe9ZLc2+gnklzV3d+d5ANJXrv+xe6+qrsvPzF7t1sfAMAjLJq8VJJ7Zo/vSvL4nSkHABjDo362UZJ3J/kPVfW6JIeS/I87VxIAsLumtUjdQs1Ld9+b5GU7XAsAwKYsUgcATOqykdsDAACTInkBgCXXWY6p0gAAo5C8AMCy67WF6qZC8gIATIrkBQAY7SaLi5C8AACTInkBgCXXsc4LAMBgJC8AsPSmdW8jyQsAMCmSFwDAOi8AAEORvAAAZhsBACRJVV1SVTdX1a1V9ZajvP73q+pPquozVfXxqjpvs2MOn7zUdPqjOml6QVQ9/nFjlzCX+7/rrLFLmNsXXrFn7BLm9poXfWLsEubyW3e9ZOwS5va0vzh17BLmc+/0/hwnh8YuYGl0D5O8VNWeJO9K8reS3J7kU1V1ZXd/bt3b3t3d/3r2/lck+edJLjnWcafTWQAAU/O8JLd29+e7+2CS9ya5bP0buvvr63ZPz9qaecc0vagBANhx21jn5cyqunHd/r7u3jd7fHaS29a9dnuS5x95gKr6B0nelGRvku/f7AM1LwDAdqZK39Xdz9neZ/e7kryrql6T5P9I8hPHer/LRgDAUO5I8pR1++fMntvIe5O8crODal4AgHTXQtsmPpXk3Kp6elXtTfLqJFeuf0NVnbtu9weT3LLZQV02AgAG0d0PVdUbklyTZE+SX+7um6rqbUlu7O4rk7yhqi7O2vSyr2WTS0aJ5gUAll5nSynKYsfuvjrJ1Uc899Z1j39m3mO6bAQATIrkBQDYfHGV44jkBQCYFMkLACy7gW4PMBTJCwAwKZIXAGBSg14kLwDApEheAABjXgAAhiJ5AQC2c1fpXSd5AQAmRfICAEuuY8wLAMBgFmpequqSqrputn2pql6504UBALukk3Qtto1goctG3f3BJB9Mkqq6Icm1O1kUAMBGtnXZqKqekeQr3f3Ndc9dXlW3VNWdB3L/tgsEAIbXvdg2hu2OefnhJO9b/0R37+vuc7v7rFNy2jYPDwDsil5wG8F2m5dLk1y5E4UAAGzFwlOlq+o7khzs7rt3sB4AYNfV0kyVvizJ+3eqEACArVg4eenuX9rJQgCAEbk9AADAMNweAACWXbs9AADAYCQvAIAxLwAAQ5G8AABJjHkBABiE5AUAMOYFAGAokhcAQPICADAUyQsALLtOYoVdAIBhSF4AgLQxLwAAw5C8AACTmm00bPNSSZ0wnQFAtfeksUuY2+oTHjd2CXP54sun1y+/5kWfGLuEuf38E/9k7BLm8lvPPn/sEua28qHTxi5hLnu+Or2/e/XgdL4/kqRXx65gmwzYBQAYxvRacQBgx9WELhtJXgCASZG8AMCy60xqwK7kBQCYFMkLACy9MtsIAGAokhcAwJgXAIChSF4AAMkLAMBQJC8AgOQFAGAokhcAWHYd67wAAAxF8gIAuKs0AMBQNC/ApvavnJofvv0ns3/l1LFLAYbSC24j0LwAm7rym+fn/d/8nlz1zWeNXQrA4s1LVf14VX2kqq6rqrN3sijg+PIr+1/0iH8CjGmhAbuzZuWl3X3RDtcDHAd+4ItvzEfuP+/h/b05lCS5/v6/lj3/5d88/PxFp30uH3rqO3a9PmDnLcOA3Zcl2TNLXt5ZVXt2sihgXP/7E67OafXgw/sHc9Ij/pkkp9WD+bknfGDXawNYtHl5UpK9s+Tl/iSXrX+xqi6tqn0P9aHt1geM4MLTb86V57zzEQ3MeqfVg7nqnH+RC07/r7tcGTCYrsW2ESzavOxP8nuzxx9N8jfWv9jdV3X35SfWSd/yg8A0XHj6zXnP2ftySh18xPOn1MG85+x9GhdgNIs2L9cnOTzt4NlJ/mJnygGOJ/eunJoTs5oTspJT62BOyEpOzGruNWUaHl0WnSY9panS3f2ZJA9U1XVJnpvkip0sCjg+/PL+F+f+3ptnnXxH3nfOL+ZZJ9+R+3uvWUfAqBa+PUB3v3knCwGOP48/4YG8/YlX5Ge+7dqcUJ3vf9rP5xfuuTgfe+DcsUsDdtqEZhu5txGwofed8y8fsb+nOm96wofzpnx4pIoANC8AQJZjnRcAgFFIXgCASY15kbwAAJMieQEAJC8AAEORvADAkqs22wgAYDCSFwBgtDtEL0LyAgBMiuQFADDbCABgKJIXAGBSs400LwCAy0YAAEMZNnnppFen08r1wUNjlzC3E+7++tglzOWpv3Pa2CXM7bfuesnYJcztt559/tglzOVx73vM2CXMbc/Xvjp2CXPpldWxS5jblL4/Js8idQAAwzHmBQAw5gUAYCiSFwBA8gIAMBTJCwBgthEAwFA0LwDApGheAIBJMeYFADDbCABgKJIXAFh27m0EALCmqi6pqpur6taqestRXn9TVX2uqj5bVR+pqu/c7JiaFwBgbczLItsxVNWeJO9K8vIk5yX5sao674i3/VGS53T3s5JckeTtm5WqeQEAhvK8JLd29+e7+2CS9ya5bP0buvt3u/v+2e4fJDlns4NqXgCAQZKXJGcnuW3d/u2z5zby+iS/s9lBDdgFALbjzKq6cd3+vu7eN+9Bquq1SZ6T5KWbvVfzAgBLrrKt2UZ3dfdzNnjtjiRPWbd/zuy5R35+1cVJ/mGSl3b3g5t9oMtGAMBQPpXk3Kp6elXtTfLqJFeuf0NV/XdJfinJK7r7q1s5qOQFABhkhd3ufqiq3pDkmiR7kvxyd99UVW9LcmN3X5nknyV5TJL/WFVJ8sXufsWxjrtQ81JVT0tyQ5I/S3Kwu39gkeMAAMeBARep6+6rk1x9xHNvXff44nmPuZ3k5cPd/dpt/DwAwNy2M+blwqr6WFW9cf2TVXV5Vd1SVXceyP0b/SwAcDwZZqr0IBZtXr6U5LuSXJjk4qp61uEXuntfd5/b3WedktN2okYAgIct1Lx094PdfV93P5Tkt5M8c2fLAgB21aM9eamqx67bfVGSP9+ZcgAAjm3Ry0Yvqao/rKrrk9zR3TfsZFEAwO6qXmwbw0KzjY427QkAYDdYpA4AGG38yiLcHgAAmBTJCwAsuxFnDi1C8gIATIrkBQAYbebQIiQvAMCkSF4AAGNeAACGInkBAIx5AQAYiuQFADDmBQBgKJIXAFh2E1thV/MCAEuuZttUuGwEAEyK5AUAcNnoEXp18I/YKX3oobFLmNvqvfvHLmEup/3pdP48HPa0vzh17BLmtvKh08YuYS57vvbVsUuY3z33jl3BXPrgwbFLmN+Evj/YXZIXAMAidQAAQ5G8AACTGvMieQEAJkXyAgBIXgAAhiJ5AYBl12YbAQAMRvICABjzAgAwFMkLAGDMCwDAUCQvAIAxLwAAQ5G8AADGvAAADEXyAgDLrjOpMS+aFwBgUs2Ly0YAwKRsq3mpqjdW1cd3qhgAYPdV1gbsLrKNYeHmpapOTvLsHawFAGBT20leXp/kV3eqEABgRL3gNoKFmpeqOinJBd390Q1ev7Sq9j2Ug9sqDgDgSIsmL69L8u6NXuzuq7r78hOzd8HDAwC7qboX2sawaPPy15P8ZFV9MMl3V9VP72BNAAAbWmidl+7+2cOPq+rj3f3OnSsJANhVE1ukbtvrvHT3i3eiEACArbDCLgDgxowAAEORvAAAyzXmBQBgN0leAABjXgAAhiJ5AQCMeQEAGIrkBQCWXRvzAgAwGMkLADCpMS+aFwBYchWXjQAABiN5AQCSnk70InkBACZF8gIATGrMy/DNy4RiqH7o0NglzO+B1bErmMvqoQme43v3jF3B3PZ8dVr/X9Ir0/pznCR98ODYJcylDz00dgmwY6b1Gw4A2HmdSU2VNuYFAJgUyQsAkJrQ1VvJCwAwKZIXAMCYFwCAoUheAIBJrfMieQEAJkXyAgDLrjOpRWUlLwDApEheAABjXgAAhiJ5AQCs8wIAMBTJCwAsucq0xrxoXgBg2XWbKg0AMBTJCwAwqctGkhcAYFIkLwDAo3+qdFU9s6qur6qPVdWvVFXtdGEAAEez6GWjm7v7hd39ktn+c3aqIABg91Uvto1hoealuw+t230wyW2Hd6rq8qq6paruPJD7tlsfAMAjLDxgt6peUVV/muRJSe4+/Hx37+vuc7v7rFNy+k7UCAAMqZOs9mLbCBZuXrr7yu5+ZpLbk/zQzpUEALCxhWYbVdXJ3f3gbPfrSR7YuZIAgF03odlGi06VvqSq3jR7fEuSD+1QPQAAx7TogN33d/dLZ9vf6+7VnS4MANg9Q802qqpLqurmqrq1qt5ylNe/r6o+XVUPVdWrtlKrFXYBgEFU1Z4k70ry8iTnJfmxqjrviLd9McnfTfLurR7XCrsAwFB3lX5eklu7+/NJUlXvTXJZks/9t4/tv5y9tuWrOJIXAGA7zqyqG9dtl6977eysWwsuazOUz97uB0peAIDtrJZ7V3fv6kr7khcAYCh3JHnKuv1zZs9ti+YFAJZdb2M7tk8lObeqnl5Ve5O8OsmV2y1X8wIADKK7H0ryhiTXJPmzJL/Z3TdV1duq6hVJUlXPrarbk/xIkl+qqps2O64xLwCw5CpJDTPbKN19dZKrj3juresefyprl5O2TPMCACQTWm7WZSMAYFIkLwDAYJeNhiB5AQAmRfICAMtua9OejxuSFwBgUiQvALD0eqgbMw5C8gIATIrkZb0JdZ2H9crK2CXMpVend46TQ2MXMLd6sMYuYS6T/HPRE1oUY6om+Dt5yrZxY8ZdJ3kBACZF8gIATCrpkrwAAJMieQGAZddJTWgYl+QFAJgUyQsAYMwLAMBQJC8AgHsbAQAMRfICAKSMeQEAGIbkBQCY1GwjzQsALLtOYpE6AIBhSF4AYMlV2oBdAIChSF4AgEkN2JW8AACTInkBAB79yUtVPb+qrq+qj1fVO3a6KACAjSx62egLSb6/u1+c5IlV9Td3sCYAYDcdXudlkW0EC1026u4vr9s9lGRlZ8oBADi2bY15qapnJTmruz93xPOXJrn01Jy+ncMDALtkKdZ5qapvT/KLSV5/5GvdfVV3X35i9m6nNgCAb7FQ8lJVJyb59SRvPuISEgAwRUuQvPxIkucmeXtVXVdVL9jBmgAANrTogN33JHnPDtcCAIyilyJ5AQAYhRV2AWDZdSQvAABDkbwAAKOtlrsIyQsAMCmSFwBgOVbYBQAYg+QFAJjUbCPNCwAsu06yOp3mxWUjAGBSJC8AsPTcHgAAYDCSFwBA8gIAMBTJCwAgeQEAGIrkZeom1CknSXpl7AqWQk/oBmvAccA6LwAAw5G8AMDS60lFtpIXAGBSJC8AwKTGUEpeAIBJkbwAwLIz2wgAYDiSFwDAmBcAgKFIXgAAyQsAwFAkLwCw9HpSyYvmBQCWXSdZdXsAAIBBSF4AgEldNpK8AACTInkBACQvAABDkbwAwNLrR/+NGavqyVX16ao6UFUaIABg1yzaeNyT5KIk79vBWgCAMXTSPZ11XhZqXrr7QJIDVbXD5QAAHNuOD9itqsur6paquvNA7tvpwwMAQ1jtxbYR7Hjz0t37uvvc7j7rlJy+04cHAJacwbYAwKN/nZeqOqmqrk1yfpJrqur5O1sWAMDRLTpg91CSi3e4FgBgDN3uKg0AMBRjXgCAR/+YFwCAsUheAIC0MS8AAMOQvADA0utJjXnRvADAsuuMttT/Ilw2AgAmRfICACRtwC4AwCAkLwCw5DpJG/MCADAMyQsALLtuY14AAIaieQEA0qu90LaZqrqkqm6uqlur6i1Hef3kqvoPs9dvqKqnbXZMzQsAMIiq2pPkXUlenuS8JD9WVecd8bbXJ/lad/+1JO9I8k83O67mBQBYG/OyyHZsz0tya3d/vrsPJnlvksuOeM9lSX519viKJBdVVR3roIMO2P1GvnbXtX3FFwY49FOTfHGA4w7l8Un2j13EnJzj4TnHw3OOhzW185s4x+t95+EH38jXrrm2rzhzweOcUlU3rtvf1937Zo/PTnLbutduT/L8I37+4fd090NVtT/JE5LctdEHDtq8dPdZQxy3qu7s7ucMcewhVNW+7r587Drm4RwPzzkennM8rKmd38Q53kh3XzL0Z+ykqV42unfsAuZ01dgFLMA5Hp5zPDzneFhTO7+Jc7zb7kjylHX758yeO+p7qurErKVjdx/roFNtXqYU+aW7p/aXJXGOd4NzPDzneFiTOr+JczyCTyU5t6qeXlV7k7w6yZVHvOfKJD8xe/yqJB/t7mNOY5rqInX7Nn8L2+QcD885Hp5zPCznd3iTPsezMSxvSHJNkj1Jfrm7b6qqtyW5sbuvTPL/Jvn3VXVrknuy1uAcU23S3AAAHFemetkIAFhSk2pequrJVfXpqjowG9RzXKuq51fV9VX18ap6x9j1bEVVPXNW88eq6lc2m2t/vKiqN1bVx8euYyuq6mlV9ZWquq6qPjR2PVtRVT9eVR+Z1Xz22PVsZrai53Wz7UtV9cqxazqWqjqtqj4wq/f9VXXy2DUdS1WdWFXvrarfraq3j13PsRzte6Oq3jH7HfcLY9d3pCPrndr33m6ZVPOStWthFyX5g7EL2aIvJPn+7n5xkidW1d8cu6AtuLm7X9jdL5ntH/fTIGe/6J89dh1z+nB3X9DdPzB2IZuZNSsv7e6LZjUfOVPguNPdH5z/2lzvAAAC+0lEQVTVekHW1si4duSSNnNJkhtm9X5ytn88+9tJ/ri7L0xyalWdP3ZBx/CI742q+p4kj5n9jttbVc8ds7ijOPJ7bmrfe7tiUs1Ldx/o7q+NXcdWdfeXu/vAbPdQkpUx69mK7j60bvfBPHJxoePV6/PfVmecigtn/+f3xrEL2YKXJdkzS17eOVvuexKq6hlJvtLd3xy7lk38eZLTZ4/PyCbTRI8Dz0jy2dnjzyR54Yi1HNNRvje+N8mHZ4+vTfKC3a9qY0fWO7Xvvd0yqeZlqqrqWUnO6u7PjV3LVlTVK6rqT5M8Kcf5L9GqOinJBd390bFrmcOXknxXkguTXDz783E8e1KSvd19UZL7861Lex/PfjjJ+8YuYgtuSfKCqropa2nn9SPXs5mbk7x09vjCrDVcU3FGkq/PHu/PtGpnRvMysKr69iS/mLV0YBK6+8rufmbWlnH+obHr2cTrkrx77CLm0d0Pdvd93f1Qkt9O8syxa9rE/iS/N3v80SR/Y8Ra5nVpvnVNiePRTyS5qru/O8kHkrx25Ho2c1XWLhd9JGsJ7VdGrmce+5M8bvb4cZn+InBLSfMyoNngql9P8ubu/vLY9WzFEQMFv57kgbFq2aK/nuQnq+qDSb67qn567II2U1WPXbf7oqxdMjieXZ/kcDr07CR/MWItW1ZV35HkYHcf1+nhTGVtbEOydj+Xx49Yy6a6e6W7f3qWxq1kbQ2Pqfj9rI0hSZKLYyzJJE2qeamqk6rq2iTnJ7mmqo68udPx5keSPDfJ22ezCI6ra6sbuKSqfq+qfi9rlwuO69kw3f2z3f2y2X05burud45d0xa8pKr+sKquT3JHd98wdkHH0t2fSfJAVV2XtT/PV4xb0ZZdluT9YxexRe9O8qOzc/x3kvzGuOUcW1WdPfud9tEk1x/Pg7iP/N5IclKSA1X1sSQr3f3JUQs8wtG+5yb2vbcrLFIHAEzKpJIXAADNCwAwKZoXAGBSNC8AwKRoXgCASdG8AACTonkBACbl/weMaLzOJ/MarAAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 1440x1440 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"V_dx = np.vstack((np.diff(V, axis=0), np.zeros(nvmdp.width)))\n",
"V_dy = np.hstack((np.diff(V, axis=1), np.zeros((nvmdp.height,1))))\n",
"\n",
"plt.figure(figsize=(20,20))\n",
"nvmdp.visualize_grid(np.sqrt(V_dx**2+V_dy**2), trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value Gradient\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0. , 0. , 0. , 0. , 0. , 0.01, 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0.01, 0.01, 0.02, 0.01, 0.01, 0. , 0. , 0. ],\n",
" [0. , 0. , 0.01, 0.02, 0.04, 0.07, 0.04, 0.02, 0.01, 0. , 0. ],\n",
" [0. , 0.01, 0.02, 0.05, 0.1 , 0.24, 0.1 , 0.04, 0.01, 0. , 0. ],\n",
" [0. , 0.01, 0.04, 0.1 , 0.25, 0.38, 0.2 , 0.05, 0.02, 0. , 0. ],\n",
" [0.01, 0.02, 0.07, 0.24, 0.38, 0.47, 0.29, 0.08, 0.02, 0.01, 0. ],\n",
" [0. , 0.01, 0.04, 0.1 , 0.2 , 0.29, 0.14, 0.05, 0.02, 0.01, 0. ],\n",
" [0. , 0.01, 0.02, 0.04, 0.05, 0.08, 0.05, 0.02, 0.01, 0. , 0. ],\n",
" [0. , 0. , 0.01, 0.01, 0.02, 0.02, 0.02, 0.01, 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0.01, 0.01, 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ]])"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.sqrt(V_dx**2+V_dy**2).round(2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Low Temperature"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAIVCAYAAAAZNR+PAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+wZGd5H/jvM6MZaUCIMZawhQQIEtkFiB/riJ82hRQpQSSW5HhtL3gxzhZl1XoDtYUXl0k2YR3W/4TULpu1qSSzFSfGGBNMymFkFGQEyIHI/BC2wAivVjLGICHDgDRSBmY0M/c++8ftka8Gzdy5PX3mzFF/PtQp9enue/qZrkv3c7/nfd9T3R0AgKnYMnYBAACboXkBACZF8wIATIrmBQCYFM0LADApmhcAYFI0L8BxVdVFVdVVdcbYtQAkmhdYClX1oap626Pcf21V/aXGBJgSzQssh99I8tqqqqPu/5kkv9Xdh0eoCWAumhdYDv8xyfcmefmRO6rqe5L8aJJ3VdXfrao/rqoHq+qrVfXLxzpQVX25qq5ct//LVfXudfsvqapbqmpvVX2uqi4b4h8ELC/NCyyB7t6f5H1JXrfu7p9K8v929+eSfHv22M4kfzfJz1fVj232darqgiQfTPIrSZ6U5M1J/kNVnXdy/wKAv6J5geXxG0l+oqrOmu2/bnZfuvvm7v6T7l7t7s8n+e0kr5jjNV6b5IbuvmF2rA8nuTXJ31lA/QBJNC+wNLr7E0m+meTHquqvJXlRkvckSVW9uKo+VlV7quqBJP9jknPneJmnJ/nJ2SmjvVW1N8mPJDl/Mf8KgMQMA1gu78pa4vKDSW7s7q/P7n9Pkl9L8qruPlBV/1eO3bx8O8nj1u1//7rbX03ym939c4stG+CvSF5gubwryZVJfi6zU0YzT0hy36xxeVGSnz7OMW5L8uqq2lZVlyb5iXWPvTvJ1VX1yqraWlVnVdVlVXXhgv8dwBLTvMAS6e4vJ7klyeOT7F730P+U5G1V9V+TvDVrg3uP5Z8k+WtJ7k/yTzM79TQ7/leTXJvkHyXZk7Uk5hfjswZYoOrusWsAADhh/hoCACZF8wIADKaqfr2qvlFVXzjG41VV/3dV3VVVn6+qH9romJoXAGBI/y7JVcd5/FVJLp5t1yX5lxsdUPMCAAymu/9zkvuO85Rrk7yr13wyyc6qOu7aUIOu87L9jMf1ju07h3yJheotR1+z7vQ3tZr7jGnVmySrW8euYPN6Yis41QQvC7llZewKNqcOT29yRq1Oq+ap1bv/4N4cPPydSpJXXv74/tZ98/1Sf/bzD92e5MC6u3Z1965NHOKCrM1MPOLu2X33HusHBv2I27F9Z17y118/5Ess1OrZ28cuYdMOnb1t7BI25eDOiX2rJjmwc3oB5f7zptUk7tgzrQ/9JDlr7+rYJWzK9r3T6xC37Ts0dgmbsmXfwbFL2JRP3vVvHr79rftW8ukbnzbXcbaef+eB7r50UXWdiOl9KgMAjyX3JHnquv0LZ/cdk+YFAJZcJ1md838LsDvJ62azjl6S5IHuPuYpo8S1jQCAAVXVbye5LMm5VXV3kv8tybYk6e5/leSGrF15/q4k30nyP2x0TM0LACy9zkoPM46ru1+zweOd5B9s5phOGwEAkyJ5AYAltzbmZTqz/iQvAMCkSF4AgEXNHDolNC8AsOQ6nZV22ggAYBCSFwDAgF0AgKFIXgBgyXWSFckLAMAwJC8AgDEvAABDkbwAwJLrxDovAABD2bB5qaqnVNUfVdWBqjrj6P1TUSQAMKzVObcxnEjycl+SK5J88hj7AACnzIbJSXcfSHKgqh51HwCYtk4v9zovVXVdVd1ZVXv2H3xg0YcHAJbcwpuX7t7V3Rd393k7tj9x0YcHABatk5U5tzGYbQQATMqJzDbaVlU3JXl+khur6sVH7w9dJAAwnM60ZhudyIDdQ0muPOruo/cBAE4J67QAwNKrrGQ6s4iNeQEAJkXyAgBLrpOsTmeZF80LABCnjQAAhiJ5AYAl15G8AAAMRvICAGS1JS8AAIOQvADAkjPmBQBgQJIXAFhyncrKhPKM6VQKABDJCwAQs40AAAYzaPLSWyqrZ28f8iUW6tDZ28YuYdMO7pxWeHZg5/T65f3nTeevkSP2n78ydgmbtHXsAuYwtd/laX1WTNHUvkF6y199tpltBAAwIK04ACy9ykpPJ8+YTqUAAJG8AMDS6ySrE8ozplMpAEAkLwBAzDYCABiM5AUAllz3tGYbaV4AgKw6bQQAMAzJCwAsubXLA0wnz5hOpQAAkbwAAC4PAAAwHMkLACw5lwcAABiQ5AUAyEpb5wUAYBCSFwBYcp2yzgsAwFA2bF6q6ilV9UdVdaCqzpjd946q+nhV/YvhSwQAhrbaW+baxnAir3pfkiuSfDJJquqHkpzd3S9Psr2qXjhgfQAAj7DhmJfuPpDkQNXDo5BfkuTDs9s3JXlpks8MUh0AMLhluLbRziQPzm4/MNt/hKq6uqp2HT584GRqAwD4LvM0Lw8kOWd2+5wke49+Qndf393XnXHGWSdTGwBwCnQqKz3fNoZ5mpc/zNoYmCS5MrOxMAAAp8KJzDbaVlU3JXl+khuTbMvaGJiPJ1np7k8PXCMAMLDVbJlrG8OJDNg9lLWEZb1PDVMOAMDxWWEXAJZcd7Iy0pot89C8AMDSq6zGhRkBAAYheQGAJdeZ1mmj6VQKABDJCwCQx/7lAQAARiN5AYAl16msjrTU/zwkLwDApEheAABjXgAAhiJ5AYAl10lWrfMCADAMyQsALL3KimsbAQAMQ/ICAEtuamNeBm1eekvl0NnbhnyJhTq4c3q93IGd0/llS5L9500nljxi//krY5ewaWc/7cGxS9iUfTln7BLmsHXsAjZpWp8Va6b3mTwlvWV6n8dH+M0AAIx5AQAYiuQFAJZcd01qzMt0KgUAiOQFAEiyMqHkRfMCAEuuk6wasAsAkFTVVVV1R1XdVVVveZTHn1ZVH6uqP66qz1fV39nomJIXAFh6Nchpo6ramuSdSf5WkruTfKaqdnf3F9c97R8neV93/8uqenaSG5JcdLzjSl4AgKG8KMld3f2l7j6Y5L1Jrj3qOZ08vFLlE5N8baODSl4AYMmtXR5g7jEv51bVrev2d3X3rtntC5J8dd1jdyd58VE//8tJfr+q3pjk8Umu3OgFNS8AwMn4ZndfehI//5ok/667/4+qemmS36yqS7p79Vg/oHkBALIyzEiSe5I8dd3+hbP71nt9kquSpLv/sKrOSnJukm8c66DGvAAAQ/lMkour6hlVtT3Jq5PsPuo5X0lyRZJU1bOSnJVkz/EOKnkBgCXXqZMZ83Ls43Yfrqo3JLkxa5di//Xuvr2q3pbk1u7eneR/SfL/VNWbsjb85u93dx/vuJoXAGAw3X1D1qY/r7/vretufzHJD2/mmJoXACCrExpJMp1KAQAieQGApdedrAww5mUokhcAYFIkLwDAILONhjJX8lJVZ1TVe2dXgXz7oosCADiWeU8b/b0kn+vuy5PsqKrnL7AmAOAUWlvnZctc2xjmfdVnJvn87PZtSV62mHIAAI5v3ubljiSvmN2+PMnOIw9U1XVVdWdV7XnowN6TrQ8AOAVWUnNtY5i3ebk+a6eLPpLkoSRfP/JAd+/q7ou7+7wzz9p5zAMAAMxjrtlG3b2S5I1JUlW7snbNAgBggjrTmm00V/NSVRck+a0kq0ne1d1HX94aAGAQ8yYv9yS5bLGlAADjqNFmDs3DInUAQFZHGnw7j+m0WQAAkbwAwNJzYUYAgAFJXgCASQ3YnU6lAACRvADA0lu7MKMxLwAAg5C8AADWeQEAGIrkBQCW3NQuzCh5AQAmRfICAFjnBQBgKJIXAFh2bZ0XAIDBDJq89BmVgzunE+4c2Dm9Xm7/edPplJNk//krY5ewaWc/7cGxS9i0ay76wtglbMruXDJ2CZu2L+eMXcImbR27gDlM7TN5Ot93ydp39MO3Y50XAIDBTKtNBAAGYcwLAMBAJC8AsOSmtsKu5gUAmFTz4rQRADApkhcAWHIdi9QBAAxG8gIAWKQOAGAokhcAWHZtthEAwGAkLwCw5Ka2SJ3kBQCYFMkLACB5AQAYiuQFAJacFXYBAAYkeQEA0hNKXuZqXqrqcUl+J8njkzyQ5Ke6+6FFFgYA8GjmPW10VZJPdfdlST492wcAJmo1Ndc2hnmblz/LWuqSJDuTfGsx5QAAHN+8Y17uTPLSqro9yTeS/NL6B6vq6iRXn/n47z3J8gCAofWSXNvoZ5Nc393PSfLBJK9d/2B3X9/d152xfcfJ1gcA8AjzJi+V5L7Z7W8meeJiygEAxvCYn22U5D1J/n1V/UySQ0n+u8WVBACcWtNapG6u5qW79yZ55YJrAQDYkEXqAIBJnTZyeQAAYFIkLwCw5DrLMVUaAGAUkhcAWHa9tlDdVEheAIBJkbwAAKNdZHEekhcAYFIkLwCw5DrWeQEAGIzkBQCW3rSubSR5AQAmRfICAFjnBQBgKJIXAMBsIwCAoQyavKxuTQ7snE5/tP+86XSdR+w/f2XsEjbl7Kc9OHYJm3bNRV8Yu4RN+5Un/8nYJTzm7c4lY5ewKftyztglzGHr2AVs0nS+75K17+gjuiUvAACDMeYFAJjUOi+aFwDAVGkAgKFIXgAAA3YBAIYieQGAJdcpyQsAwFAkLwBAJjTZSPICAEyL5AUAlp3LAwAADEfyAgBMatCL5AUAmBTJCwBgzAsAQJJU1VVVdUdV3VVVbznGc36qqr5YVbdX1Xs2OqbkBQAY5KrSVbU1yTuT/K0kdyf5TFXt7u4vrnvOxUn+YZIf7u77q+rJGx1X8gIADOVFSe7q7i9198Ek701y7VHP+bkk7+zu+5Oku7+x0UE1LwCw5DprY17m2ZKcW1W3rtuuW3foC5J8dd3+3bP71vuBJD9QVf+lqj5ZVVdtVK/TRgDAyfhmd196Ej9/RpKLk1yW5MIk/7mqntvde4/1A3MlL7PBNzfPtnur6sfmKhcAGF8n6ZpvO757kjx13f6Fs/vWuzvJ7u4+1N1/nuT/y1ozc0xzNS/d/aHuvqy7L0vylSQ3zXMcAOAx7TNJLq6qZ1TV9iSvTrL7qOf8x6ylLqmqc7N2GulLxzvoSY15qapnJvl6d+9bd991VXVnVe05tO/+kzk8AHCKdM+3Hf+YfTjJG5LcmORPk7yvu2+vqrdV1TWzp92Y5FtV9cUkH0vyi939reMd92THvPx4kt89qtBdSXYlyePOe+qEFhsGgCU20Dd2d9+Q5Iaj7nvrutud5Bdm2wk52dlGV+e74x8AgMHMnbxU1fcnObhRtAMAnO4envY8CSeTvFyb5AOLKgQA4ETMnbx0979eZCEAwIgmNErVCrsAwKRYYRcAll1naca8AACccpIXAMCYFwCAoUheAIAkxrwAAAxC8gIAGPMCADAUyQsAIHkBABiK5AUAll0nscIuAMAwJC8AQNqYFwCAYUheAIBJzTYatHnpM5L9501nAND+81fGLmHTzn7ag2OXsCnXXPSFsUvYtF958p+MXcKmvfIpLxi7hE258Wu3jV3CY97uXDJ2CZu2L+eMXcImbR27gE3pozsAA3YBAIbhtBEAkJrQaSPJCwAwKZIXAFh2nUkN2JW8AACTInkBgKVXZhsBAAxF8gIAGPMCADAUyQsAIHkBABiK5AUAkLwAAAxF8gIAy65jnRcAgKFIXgAAV5UGABiK5gXY0OHHb8/n/umrcvjx28cuBRhKz7mNQPMCbGjPyy7Knpc/M3teetHYpQDM37xU1euq6iNVdXNVXbDIooDTy9euetbaf1/1rJErAZhzwO6sWXlFd1+x4HqA08Bn//k1uf9vPPXh/Tq4kiTZ+5zzc9NH/8HD93/PZ7+av/GLu095fcDiLcOA3Vcm2TpLXn61qrYusihgXM/4rc9my4FDD+/39q2P+G+SbDlwKM94962nvDaAeZuX70uyfZa8fCfJtesfrKqrq2rXykMHTrY+YARPuu2evOAfffARDcx6Ww4cygv+4e/lSZ/72imuDBhM13zbCOZtXh5I8gez2x9N8ogT4d19fXdft/XMs06mNmBET7rtnjz3bTdmy0OHH3H/locO57lvu1HjAoxm3ublliTPm91+QZI/X0w5wOnk8NlnplZWk5XVtRRmZTW1sprDZ585dmnAIs07TXpKU6W7+7Yk+6vq5iQvTPL+RRYFnB6+9qpnZeWsbXnCl76V5/+TG/KEL30rK2dtM+sIGNXclwfo7jcvshDg9LP12wdz8b/6L3naf/hcqpMn/fzv5Cs//vzc/7zzxy4NWLQJzTZybSPgmF7w1v/0iP1a7Tz9/bfl6e+/baSKADQvAECWY50XAIBRSF4AgEmNeZG8AACTInkBACQvAABDkbwAwJKrNtsIAGAwkhcAYLQrRM9D8gIATIrkBQAw2wgAYCiSFwBgUrONNC8AgNNGAABDGTR5qcPJjj0TauWydewCNm1fzhm7hE3ZnUvGLmEp3Pi128YuYVP+8TeeO3YJm7b7y9P6Xd73lWl9ViTJjnun9Zk8re+7te/oh1mkDgBgOMa8AADGvAAADEXyAgBIXgAAhiJ5AQDMNgIAGIrmBQCYFM0LADApxrwAAGYbAQAMRfICAMvOtY0AAIYjeQEAjHkBABiK5AUAkLwAAAxF8gIAS65ithEAwGAkLwDApMa8zNW8VNVFST6V5E+THOzuv73AmgCAU2lii9SdTPLy4e5+7cIqAQA4AScz5uXyqvp4Vb1p/Z1VdV1V3VlVew7tu/8kywMATomecxvBvM3LvUl+IMnlSa6squcdeaC7d3X3xd193razv2cRNQIAPGyu5qW7H+rub3f34SS/l+SSxZYFAJxSAyUvVXVVVd1RVXdV1VuO87z/tqq6qi7d6JhzNS9V9YR1uz+c5M/mOQ4A8NhVVVuTvDPJq5I8O8lrqurZj/K8JyT5n7M2GWhD8542enlVfbaqbklyT3ef0IsBAKen6vm2DbwoyV3d/aXuPpjkvUmufZTn/e9J/lmSAydS61yzjbr7hiQ3zPOzAMBjyrlVdeu6/V3dvWt2+4IkX1332N1JXrz+h6vqh5I8tbs/WFW/eCIvaJE6AOBkZg59s7s3HKfyaKpqS5L/M8nf38zPuTwAADCUe5I8dd3+hbP7jnhC1ib93FxVX07ykiS7Nxq0K3kBgGU33Jotn0lycVU9I2tNy6uT/PTDL9v9QJJzj+xX1c1J3tzdt+Y4JC8AwCBmS6q8IcmNWbuk0Pu6+/aqeltVXTPvcSUvAMBg1zZ6tEk+3f3WYzz3shM5puQFAJgUyQsAMNp1iuYheQEAJkXyAgAMNuZlCJIXAGBSJC8AgDEvAABDkbwAwLIbboXdQWheAGDJ1WybCqeNAIBJkbwAAE4bHbFlJTlr7+qQL7FgUwyito5dwKbsyzljl7Bpu3PJ2CU85u3+8vTe431fmdbv8o57p/VZkSQ79kzo2zRT+75b+46eKskLAGCROgCAoUheAIBJjXmRvAAAkyJ5AQAkLwAAQ5G8AMCya7ONAAAGI3kBAIx5AQAYiuQFADDmBQBgKJIXAMCYFwCAoUheAABjXgAAhiJ5AYBl15nUmBfNCwAwqebFaSMAYFJOqnmpqjdV1ScWVQwAcOpV1gbszrONYe7mparOTPKCBdYCALChk0leXp/kNxZVCAAwop5zG8FczUtVbUtyWXd/9BiPX11Vuw4f3H9SxQEAHG3e5OVnkrznWA929/Xdfd0Z23fMeXgA4FSq7rm2MczbvPxgkp+vqg8leU5VvXGBNQEAHNNc67x09y8duV1Vn+juX11cSQDAKTWxRepOep2X7v6RRRQCAHAirLALALgwIwDAUCQvAMByjXkBADiVJC8AgDEvAABDkbwAAMa8AAAMRfICAMuujXkBABiM5AUAmNSYF80LACy5itNGAACDkbwAAElPJ3qRvAAAkyJ5AQAmNeZl0OalDne27z085Ess2BR7uamFZ1vHLmDT9uWcsUvYtN25ZOwSNmXfV6b3Hu+4d1q/yzv2TOibaeasvatjl7Ap0/q+W/uOnqopflsDAIvUmdRU6an92Q4ALDnJCwCQmtBZOskLADApkhcAwJgXAIChSF4AgEmt8yJ5AQAmRfICAMuu49pGAABDkbwAAMa8AAAMRfICAFjnBQBgKJIXAFhylWmNedG8AMCy6zZVGgBgKJIXAGBSp40kLwDApEheAIDH/lTpqrqkqm6pqo9X1b+tqlp0YQAAj2be00Z3dPfLuvvls/1LF1UQAHDqVc+3jWGu5qW7D63bfSjJV4/sVNV1VXVnVe156MDek60PAOAR5h6wW1XXVNUXknxfkm8dub+7d3X3xd193pln7VxEjQDAkDrJas+3jWDu5qW7d3f3JUnuTvKjiysJAODY5pptVFVndvdDs90Hk+xfXEkAwCk3odlG806VvqqqfmF2+84kv7+gegAAjmuu5qW7P5DkAwuuBQAYiRV2AQAGYoVdAMBVpQEAhiJ5AQCMeQEASJKquqqq7qiqu6rqLY/y+C9U1Rer6vNV9ZGqevpGx9S8AMCy65PYjqOqtiZ5Z5JXJXl2ktdU1bOPetofJ7m0u5+X5P1J3r5RuZoXAGAoL0pyV3d/qbsPJnlvkmvXP6G7P9bd35ntfjLJhRsd1JgXAFhylaSGmW10QdZdvDlrlxR68XGe//ok/2mjg2peAIBkde6fPLeqbl23v6u7d232IFX12iSXJnnFRs/VvAAAJ+Ob3X3pMR67J8lT1+1fOLvvEarqyiT/a5JXrLt24jFpXgCAoU4bfSbJxVX1jKw1La9O8tOPeN2q/ybJv05yVXd/40QOasAuADCI7j6c5A1Jbkzyp0ne1923V9Xbquqa2dP+eZKzk/xOVd1WVbs3Oq7kBQCW3QlMe5770N03JLnhqPveuu72lZs9puQFAJgUyQsALL12YUYAgKEMmrzUamfbvkNDvgSTC8+m2C9vHbuATduXc8YuYVN23Du993jHnun8lZokZ+2dfxGPsWzfe3jsEjZlat93tfrI32EXZgQAGMjU/mwHAIZgzAsAwDAkLwCw7DqpCQ2LkrwAAJMieQEAjHkBABiK5AUAGOzaRkOQvAAAkyJ5AQBSxrwAAAxD8gIATGq2keYFAJZdJ7FIHQDAMCQvALDkKm3ALgDAUCQvAMCkBuxKXgCASZG8AACP/eSlql5cVbdU1Seq6h2LLgoA4FjmPW30F0n+Znf/SJInV9VzF1gTAHAqHVnnZZ5tBHOdNuruv1y3eyjJymLKAQA4vpMa81JVz0tyXnd/8aj7r05y9Y4zn3QyhwcATpGlWOelqp6U5NeSvP7ox7r7+u6+7owzzjqZ2gAAvstcyUtVnZHk3UnefNQpJABgipYgefnJJC9M8vaqurmqXrrAmgAAjmneAbu/neS3F1wLADCKXorkBQBgFFbYBYBl15G8AAAMRfICAIy2Wu48JC8AwKRIXgCA5VhhFwBgDJIXAGBSs400LwCw7DrJ6nSaF6eNAIBJkbwAwNJzeQAAgMFIXgAAyQsAwFAkLwCA5AUAYCiDJi+12tmy7+CQL7FQ28YuYClMMeybYo+/dewCNmXHnun8xXfEWXsndBW7JNv3Hh67hE3btu/Q2CVsypS+75K17+iHWecFAGA4U/wzGABYqE56Ommi5AUAmBTJCwBgthEAwFAkLwCw7Mw2AgAYjuQFADDmBQBgKJIXAEDyAgAwFMkLACy9nlTyonkBgGXXSVZdHgAAYBCSFwBgUqeNJC8AwKRIXgAAyQsAwFAkLwCw9Pqxf2HGqnpKVf1RVR2oKg0QAHDKzNt43JfkiiS/u8BaAIAxdNI9nXVe5mpeuvtAkgNVteByAACOb+EDdqvquqq6s6r27D/4wKIPDwAMYbXn20aw8Oalu3d198Xdfd6O7U9c9OEBgCVnsC0A8Nhf56WqtlXVTUmen+TGqnrxYssCAHh08w7YPZTkygXXAgCModtVpQEAhmLMCwDw2B/zAgAwFskLAJA25gUAYBiSFwBYej2pMS+aFwBYdp3Rlvqfh9NGAMCkSF4AgKQN2AUAGITkBQCWXCdpY14AAIYheQGAZddtzAsAwFA0LwBAerXn2jZSVVdV1R1VdVdVveVRHj+zqv797PFPVdVFGx1T8wIADKKqtiZ5Z5JXJXl2ktdU1bOPetrrk9zf3X89yTuS/LONjqt5AQDWxrzMsx3fi5Lc1d1f6u6DSd6b5NqjnnNtkt+Y3X5/kiuqqo530EEH7D64/95v/v6f/MpfDHDopyX5ygDHHcoTkzwwdhGb5D0envd4eN7jYU3t/U28x+s9/ciN/5r7b7yp33/unMc5q6puXbe/q7t3zW5fkOSr6x67O8mLj/r5h5/T3Yer6oEk35vkm8d6wUGbl+4+b4jjVtWe7r50iGMPoap2dfd1Y9exGd7j4XmPh+c9HtbU3t/Ee3ws3X3V0K+xSFM9bbR37AI26fqxC5iD93h43uPheY+HNbX3N/Een2r3JHnquv0LZ/c96nOq6oyspWPfOt5Bp9q8TCnyS3dP7f8siff4VPAeD897PKxJvb+J93gEn0lycVU9o6q2J3l1kt1HPWd3kp+d3f6JJB/t7uNOY5rqInW7Nn4KJ8l7PDzv8fC8x8Py/g5v0u/xbAzLG5LcmGRrkl/v7tur6m1Jbu3u3Un+TZLfrKq7ktyXtQbnuGqD5gYA4LQy1dNGAMCSmlTzUlVPqao/qqoDs0E9p7WqenFV3VJVn6iqd4xdz4moqktmNX+8qv7tRnPtTxdV9aaq+sTYdZyIqrqoqr5eVTdX1e+PXc+JqKrXVdVHZjVfMHY9G5mt6HnzbLu3qn5s7JqOp6oeV1UfnNX7gao6c+yajqeqzqiq91bVx6rq7WPXczyP9r1RVe+Yfcb9i7HrO9rR9U7te+9UmVTzkrVzYVck+eTYhZygv0jyN7v7R5I8uaqeO3ZBJ+CO7n5Zd798tn/aT4OcfdC/YOw6NunD3X1Zd//tsQvZyKxZeUV3XzGr+eiZAqed7v7QrNbLsrZGxk0jl7SRq5J8albvp2f7p7O/l+Rz3X15kh1V9fwAWABDAAAC2ElEQVSxCzqOR3xvVNUPJTl79hm3vapeOGZxj+Lo77mpfe+dEpNqXrr7QHffP3YdJ6q7/7K7D8x2DyVZGbOeE9Hdh9btPpRHLi50unp9/mp1xqm4fPaX35vGLuQEvDLJ1lny8quz5b4noaqemeTr3b1v7Fo28GdJHj+7vTMbTBM9DTwzyednt29L8rIRazmuR/neeEmSD89u35Tkpae+qmM7ut6pfe+dKpNqXqaqqp6X5Lzu/uLYtZyIqrqmqr6Q5Ptymn+IVtW2JJd190fHrmUT7k3yA0kuT3Ll7PfjdPZ9SbZ39xVJvpPvXtr7dPbjSX537CJOwJ1JXlpVt2ct7bxl5Ho2ckeSV8xuX561hmsqdiZ5cHb7gUyrdmY0LwOrqicl+bWspQOT0N27u/uSrC3j/KNj17OBn0nynrGL2Izufqi7v93dh5P8XpJLxq5pAw8k+YPZ7Y8medaItWzW1fnuNSVORz+b5Prufk6SDyZ57cj1bOT6rJ0u+kjWEtqvj1zPZjyQ5JzZ7XMy/UXglpLmZUCzwVXvTvLm7v7Lses5EUcNFHwwyf6xajlBP5jk56vqQ0meU1VvHLugjVTVE9bt/nDWThmczm5JciQdekGSPx+xlhNWVd+f5GB3n9bp4UxlbWxDsnY9lyeOWMuGunulu984S+NWsraGx1T8YdbGkCTJlTGWZJIm1bxU1baquinJ85PcWFVHX9zpdPOTSV6Y5O2zWQSn1bnVY7iqqv6gqv4ga6cLTuvZMN39S939ytl1OW7v7l8du6YT8PKq+mxV3ZLknu7+1NgFHU9335Zkf1XdnLXf5/ePW9EJuzbJB8Yu4gS9J8lPzd7j/z7Jb41bzvFV1QWzz7SPJrnldB7EffT3RpJtSQ5U1ceTrHT3p0ct8CiP9j03se+9U8IidQDApEwqeQEA0LwAAJOieQEAJkXzAgBMiuYFAJgUzQsAMCmaFwBgUv5/WEH1p732V5gAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 1440x1440 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"boltz_temp = 0.01\n",
"v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, \n",
" lambda Q, t=boltz_temp: np.exp((1/t)*Q) / (np.exp((1/t)*Q).sum()), 30)\n",
"V = get_value_grid(v, nvmdp, s_to_idx)\n",
"\n",
"plt.figure(figsize=(20,20))\n",
"nvmdp.visualize_grid(V, trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0.13, 0.17, 0.21, 0.26, 0.33, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13],\n",
" [0.17, 0.21, 0.26, 0.33, 0.41, 0.51, 0.41, 0.33, 0.26, 0.21, 0.17],\n",
" [0.21, 0.26, 0.33, 0.41, 0.51, 0.64, 0.51, 0.41, 0.33, 0.26, 0.21],\n",
" [0.26, 0.33, 0.41, 0.51, 0.64, 0.8 , 0.64, 0.51, 0.41, 0.33, 0.26],\n",
" [0.33, 0.41, 0.51, 0.64, 0.8 , 1. , 0.8 , 0.64, 0.51, 0.41, 0.33],\n",
" [0.41, 0.51, 0.64, 0.8 , 1. , 0. , 1. , 0.8 , 0.64, 0.51, 0.41],\n",
" [0.33, 0.41, 0.51, 0.64, 0.8 , 1. , 0.8 , 0.64, 0.51, 0.41, 0.33],\n",
" [0.26, 0.33, 0.41, 0.51, 0.64, 0.8 , 0.64, 0.51, 0.41, 0.33, 0.26],\n",
" [0.21, 0.26, 0.33, 0.41, 0.51, 0.64, 0.51, 0.41, 0.33, 0.26, 0.21],\n",
" [0.17, 0.21, 0.26, 0.33, 0.41, 0.51, 0.41, 0.33, 0.26, 0.21, 0.17],\n",
" [0.13, 0.17, 0.21, 0.26, 0.33, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13]],\n",
" dtype=float32)"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V.round(2)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAIVCAYAAADPkMXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuwZWd5H+jfq1a3LiBQjDAGCQwkcsoyBkKJOwTJ0hjhIMv2YA/YgJ3CUQ0OVApCBmJPwIOTqTGpGcbB+NLJYIwxEIcMRjIKAgEyYIFAtoFYJDIyvkgaLrogAZJa3X3OO3+c3fJR091nn33O6tWL/TzUKu2199prv9q0znn7t77vW9XdAQCYkuPGLgAAYLM0MADA5GhgAIDJ0cAAAJOjgQEAJkcDAwBMjgYGJqiqHllVXVXHj13Loqrqr6rq/Nnjn6+q/zB2TcB0aGBgBFX1/qp6/SGev6iqvjx2Y1JVz6+qq6vqzqr66uzxz1VVDfF53f2/d/fPbvU83w6NHTAfDQyM47eTvPAQDcGLkvxud+8foaYkSVX98yS/kuTfJvmuJA9J8j8neXqSXYd5z46jViBANDAwlt9P8qAkzzzwRFX9nSTPTfK22f4/qqo/raqvV9UNVfWLhzvZ+ssxs/1frKq3r9t/SlVdVVW3V9Vnq+qcw5zngUlen+Tnuvvd3f2NXvOn3f1T3X3P7Li3VtWvV9VlVXVnknM3qreqXlRVf11Vt1bVLxz02tz1VtWVVfVLVfVHVfWNqvpAVZ02e/mjs3/eXlXfrKqnHu47A6ZNAwMj6O67k/xekheve/onkvz37v7sbP/O2eunJvlHSV5aVT+y2c+qqtOTvC/Jv07yHUleleQ/V9WDD3H4U5OckOS9c5z6J5P8mySnJPn4keqtqrOS/HrWEqaHZa15O2ML9f5kkn+c5Duzlgq9avb8P5z989Tuvn93f2KOfw9ggjQwMJ7fTvK8qjpxtv/i2XNJku6+srv/a3evdvfnkrwzybMW+JwXJrmsuy+bneuDSa5J8kOHOPa0JLesv4S1Lgm5u6r+4bpj39vdfzQ7554N6n1ekj/o7o/OUpx/lWR1C/X+Vnf/+bpG8PGb/laASdPAwEi6++NJbknyI1X1d5M8Kck7DrxeVU+uqo9U1c1VdUfWxqGcduizHdF3J/nxWRNye1XdnuQZSR56iGNvTXLa+kGw3f207j519tr6nxk3rH/jBvU+bP3x3X3n7HyL1vvldY/vSnL/w/3LA9+eNDAwrrdlLXl5YZLLu/sr6157R5JLkjy8ux+Y5DeSHG4W0J1JTl63/13rHt+Q5He6+9R12/26+/84xHk+keSeJBfNUfvBt7I/Ur1fSvLwAwdW1clZu4x0KJupd6OagG9TGhgY19uSnJ/kn2Td5aOZU5Lc1t17qupJWRv3cTifSfL8qtpZVWdn7ZLNAW9PcmFVPbuqdlTViVV1TlV9yxiU7r49yf+W5Neq6nlVdUpVHVdVj09yvw3+XY5U77uTPLeqnlFVu7I2UPhwP3/mrvcQbs7apalHz3EsMGEaGBhRd/9Vkquy1hxcctDLP5fk9VX1jSSvzdpYj8P5V0n+bpKvZa0BufdSVHffkLVE5eez9gv+hiT/Iof577+735DklUn+lyRfmW2/meTVs1oP57D1dve1Sf7prK4vzeq88TCfv6l6D3rvXVkbWPxHs8tPT9noPcA0VbfEFQCYFgkMADA5GhgAYHI0MADA5GhgAIDJGfSOrbvqxD7puAmtLzXIfXYHNszNgQc0tXozwe8406t5ipMJplbz1OpNplfzxOq9u+/M3t5TSfLsc+/Xt962stB5/vhz91ze3Rdsa3FzGLSBOem4++cpJz93yI/YVrVjgjfU3Tno/4Xb7/iJ1Zukdu4cu4TNO35if5b3jXbz7YX13n1jl7A5+yf4HU+t5n3T+jPxyT2X3fv41ttW8qnLH7HQeXY89AuLrBC+ZS4hAQCDqaq3VNVXq+rPNjjuiVW1v6qed6TjDtDAAMCS6ySrC/5vDm9NcsRLTFW1I8kvJ/nAvDVrYACAwXT3R5PctsFhL0/yn5N8dd7zTm9AAgCwzTorPVeaciinVdU16/Z3d/fued9cVacn+dEk5yZ54rzv08AAAFtxS3efvYX3/99JXt3dq7WJGZQaGABYcmtjYEabBn52knfNmpfTkvxQVe3v7t8/0ps0MADAaLr7UQceV9Vbk/zBRs1LooEBAJJ5ZxRtWlW9M8k5WRsrc2OS1yXZmSTd/RuLnlcDAwBLrtNZGWgl4e5+wSaO/Zl5jzWNGgCYHAkMADDmIN6FSGAAgMmRwADAkuskKxIYAIBhSWAAAGNgAACGJoEBgCXXyWDrwAxFAgMATM6GDUxVPayq/qSq9lTV8QfvH40iAYBhrS64jWWeBOa2JOcl+eRh9gEAjqoNE5Tu3pNkz+w219+yDwBMW6etA1NVF1fVF6rq5rtXv7ndpwcA2P4Gprt3d/eZ3f3gk467/3afHgDYbp2sLLiNxSwkAGBy5pmFtLOqrkjyuCSXV9WTD94fukgAYDid6c1CmmcQ774k5x/09MH7AABHjXVcAGDpVVYyrdnFxsAAAJMjgQGAJddJVqe1DIwGBgCIS0gAAEOTwADAkutIYAAABieBAQCy2hIYAIBBSWAAYMkZAwMAcBRIYABgyXUqKxPLNKZVLQBAJDAAQMxCAgAY3LAJTCW1Y8egH7Gtdk4wkDp+WjXXzp1jl7BpfeKusUvYvCn9d5dMbO7DmlpdHbuETZnYffqSTO/PRa+sjF3C5tTffsNmIQEAHAXT+us7ADCAykpPK9OYVrUAAJHAAMDS6ySrE8s0plUtAEAkMABAzEICABicBAYAllz39GYhaWAAgKy6hAQAMCwJDAAsubVbCUwr05hWtQAAkcAAAG4lAAAwPAkMACw5txIAADgKJDAAQFbaOjAAAIOSwADAkuuUdWAAAIa2YQNTVQ+rqj+pqj1VdfzsuTdW1ceq6leGLxEAGNpqH7fQNpZ5Pvm2JOcl+WSSVNUTkty/u5+ZZFdVPXHA+gAAvsWGY2C6e0+SPVX3jk5+SpIPzh5fkeSpST49SHUAwOCW5V5Ipyb5+uzxHbP9+6iqC6tq977eu5XaAAAOaZEG5o4kD5g9fkCS2w8+oLsv7e6Ld9aurdQGABwFncpKL7aNZZEG5hNZGxOTJOdnNjYGAOBomWcW0s6quiLJ45JcnmRn1sbEfCzJSnd/auAaAYCBrea4hbaxzDOId1/Wkpb1rh6mHADg20lVvSXJc5N8tbsfc4jXfyrJq5NUkm8keWl3f3aj805ryDEAsO26k5U+bqFtDm9NcsERXv/LJM/q7u9P8ktJds9zUrcSAIClV1nNMANyu/ujVfXII7x+1brdTyY5Y57zamAAgK04raquWbe/u7vnSlEO4SVJ/ss8B2pgAGDJdTLv5aBDuaW7z95qDVV1btYamGfMc7wGBgAYVVU9Nsl/SPKc7r51nvdoYACA0W4lUFWPSPL/JnlRd//5vO/TwAAAg6mqdyY5J2tjZW5M8rqsrSmX7v6NJK9N8qAkvza77+L+eS5JaWAAYMl1KqsD3Ragu1+wwes/m+RnN3te68AAAJMjgQEARhsDs6hpVQsAEAkMACy9TrK6+Dowo5hWtQAAkcAAAKmsDHQvpKFIYACAyZHAAMCSm+IYmGEbmKpk54R6pOMnVOtM7dw5dgmb0ifuGruEzTthejWv7prWn+Vp/dhcM62wPansHbuETeuxC9is/fvHrmCpTOunHAAwCGNgAAAGJoEBgCXXXZMbAzOtagEAIoEBAJKsTCyB0cAAwJLrJKsG8QIADEsCAwBLryZ3CWla1QIARAIDAEtv7VYCxsAAAAxKAgMAZGVimca0qgUAiAQGAJZep4yBAQAYmgQGAMjqxDKNaVULABAJDAAsve5kxRgYAIBhSWAAgOWYhVRVx1fVu6rqI1X1hu0uCgDgSBa9hPSjST7b3ecmOamqHreNNQEAR9HaOjDHLbSNZdFPfnSSz80efybJ07anHACAjS3awFyX5Fmzx+cmOfXAC1V1cVV9oapuvnvlm1utDwA4ClZSC21jWbSBuTRrl44+lOSeJF858EJ37+7uM7v7wSftuP921AgAcB8LzULq7pUkL0+Sqtqd5PLtLAoAOHo605uFtFADU1WnJ/ndJKtJ3tbdN21rVQAAR7BoAnNTknO2txQAYBw16oyiRVjIDgDI6ogDchcxrXYLACASGABYem7mCABwFEhgAIDJDeKdVrUAAJHAAMDSW7uZozEwAACDksAAANaBAQAYmgQGAJbcFG/mKIEBACZHAgMAWAcGAGBoEhgAWHZtHRgAgMENnMBUcvx0Qp7auXPsEjatT9w1dgmbc8LE6k2yevL0al45YcfYJXzbm9rf/qb1d+s1ddz+sUvYnH17x65gc+pv/1R0rAMDADC46cQjAMBgjIEBABiYBAYAltwUV+LVwAAAk2tgXEICACZHAwMAS66ztpDdIttGquotVfXVqvqzw7xeVfXvqur6qvpcVT1hnpo1MADAkN6a5IIjvP6cJGfOtouT/Po8J9XAAABZTS20baS7P5rktiMcclGSt/WaTyY5taoeutF5NTAAwJhOT3LDuv0bZ88dkVlIALDsekuzkE6rqmvW7e/u7t3bUNURaWAAgK24pbvP3sL7b0ry8HX7Z8yeOyKXkABgyR1YyG6IWUhzuCTJi2ezkZ6S5I7u/tJGb5LAAACDqap3Jjkna5eabkzyuiQ7k6S7fyPJZUl+KMn1Se5K8o/nOa8GBgAYbCXe7n7BBq93kn+62fO6hAQATI4EBgCW3IGVeKdEAgMATI4EBgBITyyBWaiBqaqTk/ynJPdLckeSn+jue7azMACAw1n0EtIFSa7u7nOSfCpHvkkTAHCMG+peSENZtIH5i6ylL0lyapJbt6ccAICNLToG5gtJnlpV1yb5apJXr3+xqi5McuFJOx6wxfIAgKH11u6FNIpFE5ifTnJpd39fkvcleeH6F7v70u6+eGedsNX6AAC+xaIJTCW5bfb4liQP3J5yAIAxLMUspCTvSPIfq+pFSfYl+Z+2ryQA4Oia3kJ2CzUw3X17kmdvcy0AAHOxkB0AMLlLSG4lAABMjgQGAJZcZ3mmUQMAjEYCAwDLrtcWs5sSCQwAMDkSGABg1BszLkICAwBMjgQGAJZcxzowAACDk8AAwNKb3r2QJDAAwORIYAAA68AAAAxNAgMAmIUEADC0YROYqtTOnYN+xHbqE3eNXcLmnTCtmldPnla9SbLvlOnVfMrrbhy7hE258xceOnYJ3/am+LfVaeUBSY6f2EWNdV9wtwQGAGBwE2sXAYAhTG0dGA0MAGAaNQDA0CQwAIBBvAAAQ5PAAMCS65QEBgBgaBIYACATm4QkgQEApkcCAwDLzq0EAACGJ4EBACY3CEYCAwBMjgQGADAGBgBgaBIYAMDdqAEAhiaBAYAl1zEGBgBgcAs1MFV1QVVdOdu+VFU/st2FAQBHSSfpWmwbyUKXkLr7/UnenyRVdXWSK7azKACAI9nSJaSqenSSr3T3N9c9d3FVfaGqbt6z8o0tFwgADK97sW0sWx0D82NJ3rP+ie7e3d1ndveDT9xxyhZPDwAcFb3gNpKtNjAXJrlkOwoBAJjXwtOoq+q7kuzt7lu3sR4A4KirpZpGfVGS925XIQAA81o4genu39zOQgCAEbmVAADAsNxKAACWXbuVAADA4CQwAIAxMAAAB8zun3hdVV1fVa85xOuPqKqPVNWfVtXnquqH5jmvBgYASFILbkc4Y9WOJG9O8pwkZyV5QVWdddBh/2uS3+vuf5Dk+Ul+bZ5qNTAAwFCelOT67v5id+9N8q6srSO3Xid5wOzxA5P8f/Oc2BgYAGArY2BOq6pr1u3v7u7ds8enJ7lh3Ws3JnnyQe//xSQfqKqXJ7lfkvPn+VANDACwFbd099lbeP8Lkry1u//Pqnpqkt+pqsd09+qR3qSBAQCGmoV0U5KHr9s/Y/bcei9JckGSdPcnqurEJKcl+eqRTmwMDAAwlE8nObOqHlVVu7I2SPeSg475myTnJUlVfW+SE5PcvNGJJTAAsOw6yQAr8Xb3/qp6WZLLk+xI8pbuvraqXp/kmu6+JMk/T/Lvq+oVs0p+prs3zIM0MADAYLr7siSXHfTca9c9/nySp2/2vBoYACAbZx7HFmNgAIDJkcAAAJO7F9KwDUxVcvyOQT9iW+2YUK0zq7um1YOunDC97/iU1904dgmb9vtnXj52CZty3q6XjF3Cpu24Z2XsEjZnSj+LZ3plejVP2gCDeIfkEhIAMDnT+us7ADCImtglJAkMADA5EhgAWHadyQ3ilcAAAJMjgQGApVdmIQEADE0CAwAYAwMAMDQJDAAggQEAGJoEBgCQwAAADE0CAwDLrmMdGACAoUlgAAB3owYAGJoGBtjQHSsn5cdufGnuWDlp7FKAofSC20g0MMCGLvnm4/Lebz4hl37zsWOXApBkCw1MVb24qj5UVVdW1enbWRRwbPmtO55+n38CjG2hQbyzhuVZ3X3eNtcDHAN+8G9ekQ/ddda9+7uyL0ly1V1/Lzv++7+/9/nzTv58PvCINx71+oDttyyDeJ+dZMcsgXlTVe3YzqKAcf3LB12Wk+uee/f3Zud9/pkkJ9c9+fkHve+o1waQLN7APCTJrlkCc1eSi9a/WFUXVtXu/at7tlofMIJz73ddLjnjTfdpYtY7ue7JpWf8u5xzvz8/ypUBg+labBvJog3MHUn+cPb4w0m+d/2L3X1pd198/HEnbqU2YETn3u+6vPP03Tmx9t7n+RNrb955+m7NCzCqRRuYq5IcmI7w+CR/uT3lAMeS21dOyvFZzXFZyUm1N8dlJcdnNbebTg3fXhadQj21adTd/Zkkd1fVlUmemOTd21kUcGx4yx3PyF29K4894aa854xfzWNPuCl39S6zkYDRLXwrge5+1XYWAhx7Hnjc3XnDd747/+zvXJHjqvMDj/zX+ZXbzs/H7j5z7NKA7TaxWUjuhQQc1nvO+LX77O+ozisf9MG8Mh8cqSKANRoYAGBp1oEBABiNBAYAmNwYGAkMADA5EhgAQAIDADA0CQwALLlqs5AAAAYngQEARr2z9CIkMADA5EhgAACzkAAAhiaBAQAmNwtJAwMAuIQEADC0YROY7mTf/kE/YjtNawLZGh3o8O78hYeOXcKmnbfrJWOXsCk7v7F37BI27bi7JlbzPROrN0ntmVbNvX86v++S3DdxsZAdAMDwjIEBAIyBAQAYmgQGAJDAAAAMTQIDAJiFBAAwNA0MADA5GhgAYHKMgQEAzEICABiaBAYAlp17IQEA/K2quqCqrquq66vqNYc55ieq6vNVdW1VvWOe80pgAIBBxsBU1Y4kb07yPyS5Mcmnq+qS7v78umPOTPIvkzy9u79WVd85z7klMADAUJ6U5Pru/mJ3703yriQXHXTMP0ny5u7+WpJ091fnObEGBgBYS2AW2ZLTquqaddvF6856epIb1u3fOHtuve9J8j1V9UdV9cmqumCecl1CAgC24pbuPnsL7z8+yZlJzklyRpKPVtX3d/ftR3qTBAYAllxlbRbSItsGbkry8HX7Z8yeW+/GJJd0977u/sskf561huaINDAAwFA+neTMqnpUVe1K8vwklxx0zO9nLX1JVZ2WtUtKX9zoxBoYAGArY2AOf8ru/UleluTyJP8tye9197VV9fqq+uHZYZcnubWqPp/kI0n+RXffulG5C42BqapHJrl6Vsze7v7BRc4DABwDBlzIrrsvS3LZQc+9dt3jTvLK2Ta3rQzi/WB3v3AL7wcAWMhWLiGdW1Ufq6pXrH+yqi6uqi9U1c17Vr6xxfIAgKNigEtIQ1q0gflS1gbZnJvk/Kp67IEXunt3d5/Z3Q8+cccp21EjAMB9LNTAdPc93X3nbHDOHyR5zPaWBQAcVcuQwFTV+mjl6Un+YnvKAQDY2KKXkJ5ZVX9cVVcluam7r97OogCAo2ughewGs9AspENNiQIAOFrcCwkAGHU8yyKsxAsATI4EBgCW3cgzihYhgQEAJkcCAwCMOqNoERIYAGByJDAAgDEwAABDk8AAAMbAAAAMTQIDABgDAwAwNAkMACy7Ca7Eq4EBgCVXs21KXEICACZHAgMAuIR0H93pvfsG/YjtVKurY5ewaSI/DmXHPStjl7Apx921d+wSNu+eadVce6ZVb5L0vun8/kiSXpnY75CeWMdyEAkMAGAhOwCAoUlgAIDJjYGRwAAAkyOBAQAkMAAAQ5PAAMCya7OQAAAGJ4EBAIyBAQAYmgQGADAGBgBgaBIYAMAYGACAoUlgAABjYAAAhiaBAYBl15ncGBgNDAAwuQbGJSQAYHK21MBU1Suq6uPbVQwAcPRV1gbxLrKNZeEGpqpOSPL4bawFAGAuW0lgXpLkt7erEABgRL3gNpKFGpiq2pnknO7+8GFev7Cqdu/re7ZUHADAoSyawLwoyTsO92J3X9rdF++sExY8PQBwNFX3QttYFm1g/n6Sl1bV+5N8X1W9fBtrAgA4ooXWgenuVx94XFUf7+43bV9JAMBRNcGF7La8Dkx3P2M7CgEAmJeVeAEAN3MEABiaBAYAWL4xMAAAR5sEBgAwBgYAYGgSGADAGBgAgKFJYABg2bUxMAAA96qqC6rquqq6vqpec4Tj/seq6qo6e57zamAAgL+9H9JmtyOoqh1J3pzkOUnOSvKCqjrrEMedkuSfJbl63nI1MACw5Cprl5AW2TbwpCTXd/cXu3tvkncluegQx/1Skl9OsmfemjUwAMBWnFZV16zbLl732ulJbli3f+PsuXtV1ROSPLy737eZDzWIFwBIeuFRvLd091zjVg5WVccl+b+S/Mxm3yuBAQCGclOSh6/bP2P23AGnJHlMkiur6q+SPCXJJfMM5JXAAABDTaP+dJIzq+pRWWtcnp/kJw+82N13JDnt3hqqrkzyqu6+ZqMTD9vAdCf79w/6EdtpYlPgkySVvWOXsCk1dgELmGRMefyOsSvYnHum9ec4SWrPtGruffvGLmHzJvT7I0mysjJ2Bcec7t5fVS9LcnmSHUne0t3XVtXrk1zT3Zcsem4JDAAsuzmmRC986u7Lklx20HOvPcyx58x73kn+5RIAWG4SGAAgtTp2BZsjgQEAJkcCAwBMbiaLBAYAmBwJDAAw1Dowg5HAAACTI4EBgGXX2cq9kEYhgQEAJkcCAwAYAwMAMDQJDABgHRgAgKFJYABgyVWmNwZGAwMAy67bNGoAgKFJYACAyV1CksAAAJMjgQEAlmMadVU9pqquqqqPVdVvVVVtd2EAAIez6CWk67r7ad39zNn+2dtVEABw9FUvto1loQamu/et270nyQ0Hdqrq4qr6QlXdfPfqN7daHwDAt1h4EG9V/XBV/VmShyS59cDz3b27u8/s7gefdNz9t6NGAGBInWS1F9tGsnAD092XdPdjktyY5LnbVxIAwJEtNAupqk7o7ntmu19Pcvf2lQQAHHUTm4W06DTqC6rqlbPHX0jygW2qBwBgQws1MN393iTv3eZaAICRWIkXAGBgVuIFANyNGgBgaBIYAMAYGACAoUlgAGDZdSa3DowEBgCYHAkMACy5SlITm4WkgQEAktWxC9gcl5AAgMmRwAAAk7uEJIEBACZHAgMAy840agCA4UlgAGDptZs5AgAMbdgEpju9f/+gH7GdauwCFjCtfjmp46bz5+GASf65WNkxdgmbUnv2jl3CpvW+fWOXsDkT+ll8r30Tq3l1YgupHJS4uJkjAMDAjIEBAIyBAQAYmgQGAJZdJzWxITwSGABgciQwAIAxMAAAQ5PAAACTW1hMAgMATI4EBgBIGQMDADAsCQwAMLlZSBoYAFh2ncRCdgAAw5LAAMCSq7RBvAAAQ9PAAABrg3gX2TZQVRdU1XVVdX1VveYQr7+yqj5fVZ+rqg9V1XfPU64GBgAYRFXtSPLmJM9JclaSF1TVWQcd9qdJzu7uxyZ5d5I3zHNuDQwAMFQC86Qk13f3F7t7b5J3Jbnovh/bH+nuu2a7n0xyxjzlLtTAVNWTq+qqqvp4Vb1xkXMAAN8WTquqa9ZtF6977fQkN6zbv3H23OG8JMl/medDF52F9NdJfqC791TV71bV93f3f13wXADAmLa2Dswt3X32VkuoqhcmOTvJs+Y5fqEGpru/vG53X5KVRc4DAHxbuynJw9ftnzF77j6q6vwkv5DkWd19zzwn3tI6MFX12CQP7u7PH/T8hUkuPKlO2crpAYCjZKB1YD6d5MyqelTWGpfnJ/nJ+3xu1T9I8ptJLujur8574oUH8VbVdyT51axdr7qP7r60uy/eWbsWPT0AMHHdvT/Jy5JcnuS/Jfm97r62ql5fVT88O+zfJrl/kv9UVZ+pqkvmOfdCCUxVHZ/k7UleddDlJABgigZaibe7L0ty2UHPvXbd4/MXOe+iCcyPJ3likjdU1ZVV9dQFzwMAsGmLDuJ9Z5J3bnMtAMAo5ltV91hiITsAYHLcjRoAll1HAgMAMDQJDACwlZV4RyGBAQAmRwIDAAy1Eu9gJDAAwORIYACAyc1C0sAAwLLrJKvTamBcQgIAJkcCAwBLz60EAAAGJ4EBACQwAABDk8AAABIYAIChDZvAdCf79g36EdupV1bGLmHz9u8fu4LN2bd37Ao273hB5dB6an+Ok/TKxO58N8Wfb6vT+o5X9+wZu4RN6fWJi3VgAACG56+WALD0OulpJV4SGABgciQwAIBZSAAAQ5PAAMCyMwsJAGB4EhgAwBgYAIChSWAAAAkMAMDQJDAAsPR6cgmMBgYAll1ncjfPdAkJAJgcCQwAMLlLSBIYAGByJDAAgAQGAGBoEhgAWHq9HDdzrKqHVdWfVNWeqtIEAQBH1aLNx21Jzkvynm2sBQAYQyfd01oHZqEGprv3JNlTVdtcDgDAxrZ9EG9VXVxVX6iqm+/uO7f79ADAEFZ7sW0k297AdPfu7j6zux98Ut1vu08PAGAWEgCQ5VgHpqp2VtUVSR6X5PKqevL2lgUAcHiLDuLdl+T8ba4FABhDt7tRAwAMzRgYAGA5xsAAAIxJAgMApI0uGMfNAAAF8UlEQVSBAQAYlgQGAJZeT24MjAYGAJZdZ9TbAizCJSQAYHIkMABA0gbxAgAMSgIDAEuuk7QxMAAAw5LAAMCy6zYGBgBgaBoYACC92gttG6mqC6rquqq6vqpec4jXT6iq/zh7/eqqeuQ89WpgAIBBVNWOJG9O8pwkZyV5QVWdddBhL0nyte7+e0nemOSX5zm3BgYAWBsDs8h2ZE9Kcn13f7G79yZ5V5KLDjrmoiS/PXv87iTnVVVtdOJBB/F+vW+75QN3v/2vBzj1I5L8zQDnHcoDk9wxdhGb5Dsenu94eL7jYU3t+018x+t994EH38jXLr+i333aguc5saquWbe/u7t3zx6fnuSGda/dmOTJB73/3mO6e39V3ZHkQUluOdKHDtrAdPeDhzhvVd3c3WcPce4hVNXu7r547Do2w3c8PN/x8HzHw5ra95v4jg+nuy8Y+jO221QvId0+dgGbdOnYBSzAdzw83/HwfMfDmtr3m/iOj7abkjx83f4Zs+cOeUxVHZ+1lOzWjU481QZmSvFfuntq/8EkvuOjwXc8PN/xsCb1/Sa+4xF8OsmZVfWoqtqV5PlJLjnomEuS/PTs8fOSfLi7N5zeNNWF7HZvfAhb5Dsenu94eL7jYfl+hzfp73g2puVlSS5PsiPJW7r72qp6fZJruvuSJP9Pkt+pquuT3Ja1JmdDNUeTAwBwTJnqJSQAYIlNqoGpqodV1Z9U1Z7ZQJ9jWlU9uaquqqqPV9Ubx65nHlX1mFnNH6uq35pnLv6xoKpeUVUfH7uOeVTVI6vqK1V1ZVV9YOx65lFVL66qD81qPn3sejYyW/nzytn2par6kbFrOpKqOrmq3jer971VdcLYNR1JVR1fVe+qqo9U1RvGrudIDvV7o6reOPsZ9ytj13ewg+ud2u+9o2lSDUzWro2dl+STYxcyp79O8gPd/Ywk31lV3z92QXO4rruf1t3PnO0f81MkZz/sHz92HZv0we4+p7t/cOxCNjJrWJ7V3efNaj54BsExp7vfP6v1nKytoXHFyCVt5IIkV8/q/dRs/1j2o0k+293nJjmpqh43dkFHcJ/fG1X1hCT3n/2M21VVTxyzuEM4+Pfc1H7vHTWTamC6e093f23sOubV3V/u7j2z3X1JVsasZx7dvW/d7j257wJEx6qX5G9XcZyKc2d/A3zF2IXM4dlJdswSmDfNlgafhKp6dJKvdPc3x65lA3+R5H6zx6dmjimkI3t0ks/NHn8mydNGrOWIDvF74ylJPjh7fEWSpx79qg7v4Hqn9nvvaJpUAzNVVfXYJA/u7s+PXcs8quqHq+rPkjwkx/gP0qrameSc7v7w2LVswpeSfE+Sc5OcP/vzcSx7SJJd3X1ekrvyrcuAH8t+LMl7xi5iDl9I8tSqujZrqedVI9ezkeuSPGv2+NysNV1TcWqSr88e35Fp1c46GpiBVdV3JPnVrKUEk9Ddl3T3Y7K25PNzx65nAy9K8o6xi9iM7r6nu+/s7v1J/iDJY8auaQN3JPnD2eMPJ/neEWvZrAvzrWtOHIt+Osml3f19Sd6X5IUj17ORS7N26ehDWUtqvzJyPZtxR5IHzB4/INNfKG5paWAGNBtw9fYkr+ruL49dzzwOGjz49SR3j1XLnP5+kpdW1fuTfF9VvXzsgjZSVaes23161i4fHMuuSnIgJXp8kr8csZa5VdV3Jdnb3cd0ijhTWRvrkKzd/+WBI9ayoe5e6e6Xz1K5layt8TEVn8jamJIkOT/GlkzWpBqYqtpZVVckeVySy6vq4BtCHWt+PMkTk7xhNrvgmLrWehgXVNUfVtUfZu3SwTE9S6a7X93dz57dx+Pa7n7T2DXN4ZlV9cdVdVWSm7r76rELOpLu/kySu6vqyqz9eX73uBXN7aIk7x27iDm9I8lPzL7jn0ryu+OWc2RVdfrsZ9qHk1x1LA/sPvj3RpKdSfZU1ceSrHT3p0Yt8CCH+j03sd97R42F7ACAyZlUAgMAkGhgAIAJ0sAAAJOjgQEAJkcDAwBMjgYGAJgcDQwAMDn/P2qDDxi6W9mhAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 1440x1440 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"V_dx = np.vstack((np.diff(V, axis=0), np.zeros(nvmdp.width)))\n",
"V_dy = np.hstack((np.diff(V, axis=1), np.zeros((nvmdp.height,1))))\n",
"\n",
"plt.figure(figsize=(20,20))\n",
"nvmdp.visualize_grid(np.sqrt(V_dx**2+V_dy**2), trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value Gradient\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0.05, 0.06, 0.07, 0.09, 0.12, 0.13, 0.11, 0.08, 0.07, 0.06, 0.04],\n",
" [0.06, 0.07, 0.09, 0.12, 0.14, 0.16, 0.13, 0.1 , 0.08, 0.07, 0.04],\n",
" [0.07, 0.09, 0.12, 0.14, 0.18, 0.2 , 0.16, 0.13, 0.1 , 0.08, 0.05],\n",
" [0.09, 0.12, 0.14, 0.18, 0.23, 0.26, 0.2 , 0.16, 0.13, 0.11, 0.07],\n",
" [0.12, 0.14, 0.18, 0.23, 0.28, 1.02, 0.26, 0.2 , 0.16, 0.13, 0.08],\n",
" [0.13, 0.16, 0.2 , 0.26, 1.02, 1.41, 0.28, 0.23, 0.18, 0.14, 0.08],\n",
" [0.11, 0.13, 0.16, 0.2 , 0.26, 0.28, 0.23, 0.18, 0.14, 0.12, 0.07],\n",
" [0.08, 0.1 , 0.13, 0.16, 0.2 , 0.23, 0.18, 0.14, 0.12, 0.09, 0.05],\n",
" [0.07, 0.08, 0.1 , 0.13, 0.16, 0.18, 0.14, 0.12, 0.09, 0.07, 0.04],\n",
" [0.06, 0.07, 0.08, 0.11, 0.13, 0.14, 0.12, 0.09, 0.07, 0.06, 0.04],\n",
" [0.04, 0.04, 0.05, 0.07, 0.08, 0.08, 0.07, 0.05, 0.04, 0.04, 0. ]])"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.sqrt(V_dx**2+V_dy**2).round(2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "irl_python3",
"language": "python",
"name": "irl_python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: OpenAI gym not installed.\n"
]
}
],
"source": [
"# Other Imports.\n",
"from simple_rl.tasks import NavigationWorldMDP\n",
"from simple_rl.agents import QLearningAgent\n",
"from simple_rl.planning import ValueIteration\n",
"from simple_rl.tasks.grid_world.GridWorldStateClass import GridWorldState\n",
"from simple_rl.tasks.navigation.NavigationStateClass import NavigationWorldState\n",
"\n",
"# Python Imports.\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"%matplotlib inline\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Given goal MDP"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def value_iter(S, A, R, T, gamma, policy, n_iters):\n",
" \n",
" nS, nA = len(S), len(A)\n",
" V = np.zeros((nS))\n",
" Q = np.zeros((nS, nA))\n",
" Pi = np.ones((nS, nA)) / nA\n",
" s_to_idx = {s: idx for idx, s in enumerate(S)}\n",
" \n",
" for _iter in range(n_iters):\n",
" for si, s in enumerate(S):\n",
" \n",
" if s.is_terminal():\n",
" continue\n",
" for ai, a in enumerate(A):\n",
" s_prime_idx = s_to_idx[T(s,a)]\n",
" Q[si,ai] = R(s,a) + gamma * V[s_prime_idx]\n",
" V[si] = policy(Q[si,:]).dot(Q[si,:])\n",
" return V"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAFyCAYAAAApoxulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFNVJREFUeJzt3Xus53V95/HnC2aAymUG6oiF7dHQUKNyWxlDB6WAEMFuB7Vb7cVLm9hOarrsxl1a7aZpbf+oCc0u62r2MtmkUYFi6+rCCOEy4hDIKBQMdqUbdiTVgwjsDMjBC2cchvf+8fsO8+N08Azf8z2/84Hf85GcnO/l9/u+399zeZ3P+Xx/l1QVkqS2HLLSDUiS/inDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazmpDkPUluXqZjzyT5QZJDl+P40nIwnHXQknwryf9LcuTYtt9Jsm2px66qq6rqrUs9Djzb54Vjx56tqqOqau8Qx19Qq7qvyaqxbau7bTW2bVuS+STfT/JkknuSfCTJ4WO3+WiSPd0fkieSbE+yYeie9eJgOOuFOhT4NyvdRGO+B7xtbP1t3baF/lVVHQ38DPDvgF8HbkiSsdt8tqqOAtYBdwCfX7BfU8Jw1gv1l8BlSdYeaGeSjyd5cGx0eE63/YQkTyU5buy2/zzJrm6k+dtJ7hjb99Yk9yeZS/JfktyW5He6fT+X5NYkj3X3v2pfP0k+A8wAW7oR6B8meXU3wl011st1SR5P8s0kvztW96NJ/ibJp7tR7n1J1i/yNfkM8P6x9fcDn36+G1fVD6tqG3AJsAH4Fwe4zR7gU8ArgZ9epL5eggxnvVB3A9uAy55n/98BZwDHAVcDf5vkiKr6LvAV4F+O3fY3gc91QfSsJC8HPgf8EaNguh84e/wmwMeAE4DXAj8LfBSgqt4HzAIbu6mMyw/Q4zXAd7r7/yrwF0neMrb/ku42a4HrgE8+z7nu87+AX0yyNsmxwDnAtYvch6qaZfT1PGfhvm6647eBB6tq12LH0kuP4aw+/gS4NMm6hTuq6sqqeqyqnq6q/wAcDrym23018BsA3b/qv95tW+iXgPuq6vNV9TTwn4FHxmp8s6puqardVbUT+I/AuQfTeJKfBd4EfLiq5qvqXuB/8NyR7x1VdUM3R/0Z4PRFDjsPbAF+rfu4rtt2ML7L6A/ZPu9O8gTwIHAm8M6DPI5eYgxnvWBV9Q3gi8BHFu5LclmS/9NNRzwBrAFe3u3+n8CGJD8D/CLwDHD7AUqcwCic9tUrRiPdfTWOT3JNkoeSPAlcOVZjMScAj1fV98e2fRs4cWz9kbHlHwFHjF/wex6fZhTwP3FK4wBOBB4fW/+bqlpbVa+oqrdU1T0v4Fh6CTGc1defAr/LWKh188t/CLwbOLaq1gJzjKYhqKrvATczGl3+JnBNHfhlER8G/tnYcTO+DvwFUMCpVXUM8N59NTo/6aUWvwscl+TosW0zwEM/6WQPwu2MLvQdz+hC3qK6UfyZHPgPlKac4axequqbwGeBfz22+WjgaWAnsCrJnwDHLLjr1YxGl7/Kgac0AK4HTk3yjm7E+vuMLoyN1/kBMJfkROAPFtz/UeCk5+n7QWA78LEkRyQ5DfgAo9F3b90fmY3AJc/zB+dZSV6W5FxG89J3ATcspbZemgxnLcWfA0eOrd8E3Aj8X0ZTBfOMTU90rgNOBh6pqq8f6KDdBbB3AZcDjwGvY3ThbHd3kz8D3sBoVH498PkFh/gY8MfdY4UPdOHyN4BXMxpFfwH406rausi5Lqqq7quq+37CTT6Z5PuM/nj8J0bTPBdX1TNLra2Xnvhi+2pdkkMYzTm/p6q+vNL9SJPgyFlNSnJR99C0w4F/z2hO+asr3JY0MYazWrUBeADYxWgu9x1V9dTKtiRNjtMaktQgR86S1KDFHli/JMnLa3RRfLLOPHPiJSXpoHzrW99i165di76Y1bKG8yiY717eEgdw9+RLStJBWb9+sdfRGnFaQ5IaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktSg9sL5yKfhz//36LMkTan2wvnsXXDOY7DBNxyWNL3aC+e3PfLcz5I0hRZ9bY0kJzB6p+XXAUcBrxhf7966vr+/vBfWP7F//cfd64GcMgdf3rZ/+91r4Q/OWFIpSXqxOJiR8+PABex/F4qF60tz1atgfqyNw+q5n2G0/8pXDVJOkl4MFg3nqprv3tL+gOsLJdmUZEeSnTC7eAf3Hgt/dOpzA3rc/CHwkVPh68cufixJeokYfM65qjZX1clVtQ5mDu5O9x4Lf/Y62L2gnd2HjLYbzJKmTDsXBI96GvYy+pg/ZP/yUT6kTtL0aSecf+kROOIZeOAo+ONTRp+PeMZHbUiaSouGc5LVSbYCpwM3JTlr4fognfzwUPivPwe/dybccxx88Ez4byfBjw4d5PCS9GKyrO++nayvlXibKt9QXFKr1q9fz913373oewi2M60hSXqW4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoOWNZzPPHP0hJBJf0jSi50jZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1aNFwTnJCkq8lmU+yqtt2RZLbk3x8+VuUpOlzMCPnx4ELgK8CJHkDcFRVnQMcluSNy9ifJE2lRcO5quar6ntjm34BuKVb3gpsWI7GJGma9ZlzXgs82S3PdevPkWRjks1zc3NL6U2SplafcJ4DjumWjwGeWHiDqtpSVZvWrFmzlN4kaWr1CeevMJqDBriQbi5akjScg3m0xuokW4HTgZuA1cB8ktuBvVV11zL3KElTZ9ViN6iqPYxGyOPuXJ52JEngk1AkqUmGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDeoVzklVJrkny5SSXD92UJE27viPndwJfr6rzgZ9KcvqAPUnS1OsbzicBf98t3wucvW9Hkk1JdiTZOTs7u9T+JGkq9Q3n+4Fzu+XzgbX7dlTV5qo6uarWzczMLLU/SZpKfcN5C6PpjC8Bu4FHh2tJktQrnKtqb1VdWlUXAHuBm4ZtS5Km26o+d0pyInAV8Azw6ap6aNCuJGnK9QrnLozPG7YVSdI+PglFkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUG93n07ycuAvwWOBOaAd1fV7iEbk6Rp1nfkfDFwZ1WdB9zVrUuSBtI3nB9gNGoGWAs8Nkw7kiToH847gA1J7gPWA9vHdybZmGTz3NzcUvuTpKnUN5x/C9hSVa8HrgfeO76zqrZU1aY1a9YstT9Jmkp9wznA493yLsAUlqQB9Xq0BnA18Nkk7wP2AL82XEuSpF7hXFVPABcN3IskqeOTUCSpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoN6hXOSi5Ns6z4eTvKOoRuTpGm2qs+dqupG4EaAJHcCW4dsSpKm3ZKmNZKcBDxaVT8YqB9JEkufc/4V4AvjG5JsSrIjyc7Z2dklHl6SptNSw3kjcN34hqraXFUnV9W6mZmZJR5ekqZT73BO8krgx1X12ID9SJJY2sj57cC1QzUiSdqv16M1AKrqvw/ZiCRpP5+EIkkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGtQ7nJO8P8mXkmxLcuKQTUnStFvV505dGJ9bVRcM3I8kif4j54uAQ7uR8yeSHDq+M8nGJJvn5uaW3qEkTaG+4Xw8cFg3cv4R8PbxnVW1pao2rVmzZqn9SdJU6hvOc8Bt3fKtwGuHaUeSBP3DeTtwWrd8BvCPw7QjSYKeFwSr6t4kTyXZBuwCrhi0K0macr3CGaCqLhuyEUnSfj4JRZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBvcI5yauTPJpkW5Kbh25KkqbdqiXc95aqeu9gnUiSnrWUaY3zk9ye5EODdSNJAvqH88PAzwPnAxcmOW3fjiSbkuxIsnN2dnaIHiVp6vQK56raXVU/rKqngS8Cp4zt21xVJ1fVupmZmaH6lKSp0veC4NFjq28CHhimHUkS9J/WOCfJPUm2Aw9V1Z1DNiVJ067XozWq6gbghoF7kSR1fBKKJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lq0JLCOcmHktwxVDOSpJHe4ZzkcOCMAXuRJHWWMnL+APCpoRqRJO3XK5yTrAbOq6pbn2f/xiSb5+bmltScJE2rviPn9wFXP9/OqtpSVZvWrFnT8/CSNN36hvNrgA8muRF4fZJLB+xJkqbeqj53qqoP71tOckdVfWK4liRJS36cc1W9eYhGJEn7+SQUSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUoF7hnOSUJNuT3J7kr5Jk6MYkaZr1HTnfX1VnV9U53fr6oRqSJPUM56raM7a6G3hw30qSTUl2JNk5Ozu71P4kaSr1nnNOckmSbwDHA4/t215Vm6vq5KpaNzMzM0SPkjR1eodzVV1XVacA3wF+ebiWJEl9LwgePrb6JPDUMO1IkgBW9bzfxUn+bbe8A7h5oH4kSfQM56q6Frh24F4kSR2fhCJJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUoF7hnOSsJNuT3JHkiqGbkqRp13fk/G3gLVX1ZuAVSU4dsCdJmnqr+typqh4ZW90D7B2mHUkSLHHOOclpwLqq+ocF2zcm2Tw3N7ek5iRpWvUO5yTHAZ8EPrBwX1VtqapNa9asWUpvkjS1+l4QXAVcCVy2YIpDkjSAviPndwFvBC5Psi3JhgF7kqSp1/eC4F8Dfz1wL5Kkjk9CkaQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDeoVzklOSPK1JPNJVg3dlCRNu74j58eBC4CvDtiLJKnTa9RbVfPAfJKB25EkwTLMOSfZlGRHkp2zs7NDH16SpsLg4VxVm6vq5KpaNzMzM/ThJWkq+GgNSWpQ30drrE6yFTgduCnJWcO2JUnTre8FwT3AhQP3IknqOK0hSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNSlUt38GTncC3e9x1BliJl7RbA8ytQN2VON9pOldYmfOdpnMFf44P1quqat1iN1rWcO4ryc6DaX4Z6m6uqk0rUHfi5ztN59rVnfj5TtO5dnX9OR5Qq9MaT6xQ3S0rVHclzneazhVW5nyn6VzBn+NBtRrOK/FvClW1Ut/oiZ/vNJ0rrNj5TtO5gj/Hg2o1nDevdAMTNk3n67m+dE3T+S77uTY55yxJ067VkbMkTbXmwjnJCUm+lmQ+Sa83A+hR86wk25PckeSKSdTs6p7S1b09yV9lgm9nnuRDSe6YYL1XJ3k0ybYkN0+qblf7/Um+1NU+cQL1Lu5qbUvycJJ3LHfNru7Lklzf1b02yeETqrsqyTVJvpzk8mWu9U/yIckV3e/QxydVdxI51Vw4A48DFwBfnWDNbwNvqao3A69IcuqE6t5fVWdX1Tnd+vpJFO1+ac+YRK0Fbqmq86rqrZMq2IXxuVV1QVf7oeWuWVU3drXOY/RY2K3LXbNzMXBnV/eubn0S3gl8varOB34qyenLWOs5+ZDkDcBR3e/QYUneOIm6B1gfXHPhXFXzVfW9Cdd8pKrmu9U9wN4J1d0ztrobeHASdYEPAJ+aUK1x53cjnA9NsOZFwKHdyPkTSQ6dVOEkJwGPVtUPJlTyAeDIbnkt8NiE6p4E/H23fC9w9nIVOkA+/AJwS7e8FdgwibqTyKnmwnklJTkNWFdV/zDBmpck+QZwPBP4ZUqyGjivqm5d7loLPAz8PHA+cGH3tZ6E44HDquoC4EfA2ydUF+BXgC9MsN4OYEOS+xj9F7Z9QnXvB87tls9n9IdhUtYCT3bLcxOuvawM506S44BPMhpVTkxVXVdVpwDfAX55AiXfB1w9gTrPUVW7q+qHVfU08EXglAmVngNu65ZvBV47oboAG4HrJljvt4AtVfV64HrgvROqu4XRdMaXGP0H+OiE6sLo+3tMt3wMK/fEn8EZzowuaABXApdV1SMTrDt+weZJ4KkJlH0N8MEkNwKvT3LpBGqS5Oix1Tcx+hd8ErYD+0bpZwD/OImiSV4J/LiqJjW1ABBGc6EAuxi97sSyq6q9VXVp99/JXuCmSdTtfIXR3C/AhUz2WtWyai6ck6xOshU4HbgpyVkTKPsu4I3A5d2V7mWZtzqAi5PcluQ2Rv9+L/ujGKrqw1V1UVVdDNxXVZ9Y7pqdc5Lck2Q78FBV3TmJolV1L/BUkm2Mvsefm0RdRtMn106o1j5XA+/uzvU9wFWTKJrkxO735lZg+3JedF2YD8BqYD7J7cDeqrprEnW7R3gta075JBRJalBzI2dJkuEsSU0ynCWpQYazJDXIcJakBhnOktQgw1mSGvT/AbUacozvcInGAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x432 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"np.random.seed(0)\n",
"nvmdp = NavigationWorldMDP(width=11, height=11, \n",
" nav_cell_types=['white'],\n",
" nav_cell_rewards=[0],\n",
" nav_cell_p_or_locs=[1.],\n",
" goal_cell_locs=[[(1,11)]],\n",
" goal_cell_rewards=[1.],\n",
" goal_cell_types=[\"blue\"],\n",
" slip_prob=0.00, step_cost=0.01, gamma=.99)\n",
"nvmdp.visualize_grid()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"S = nvmdp.get_reachable_states()\n",
"A = nvmdp.get_actions()\n",
"s_to_idx = {s: idx for idx, s in enumerate(S)}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# High Temperature"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjYAAAIVCAYAAADCjrWtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X2sZXV97/H3h4EBRS1Q0CogD+3YCApYEeozCMp4LWAbbbFFaWJK5IppNDalrcWW3iatTeptLL06SWmsllLFWIc6dQoiFi4VGXTQgpcw4gOMqDwJiswwc873/nEWuDnOnId19j57nbXfr2SF9bDXWt+9Vc7X7+8pVYUkSVIf7DHuACRJkobFxEaSJPWGiY0kSeoNExtJktQbJjaSJKk3TGwkSVJvmNhImlOSw5NUkj3HHYskzcfERpoAST6T5KJdnD8zyXdNWiT1hYmNNBk+DJydJLPOvxn4p6raOYaYJGnoTGykyfCvwM8CL3/sRJL9gV8B/jHJ65J8OclDSe5M8ie7e1CSbyY5deD4T5J8dOD4l5Ncn+QHSW5OctIovpAk7YqJjTQBquoR4GPAWwZO/zrw/6rqZuDh5tp+wOuA85K8frHvSXIw8GngfwEHAO8GPpHkoKV9A0laGBMbaXJ8GHhDkn2a47c056iqa6rqq1U1XVVfAf4ZeGWLd5wNbKiqDc2zrgQ2Af9jCPFL0rxMbKQJUVXXAfcCr0/y88AJwKUASU5M8rkk9yR5EHgbcGCL1xwGvLFphvpBkh8ALwOeOZxvIUlzcySENFn+kZlKzS8CG6vqe835S4G/BV5bVduS/G92n9g8DDx54PjnBvbvBD5SVb8z3LAlaWGs2EiT5R+BU4HfoWmGajwVuL9Jak4AfnOOZ2wGzkqyV5LjgTcMXPsocHqS05KsSrJPkpOSHDLk7yFJu2RiI02QqvomcD2wL7B+4NL/BC5K8kPgQmY6Gu/OHwM/DzwA/ClNc1bz/DuBM4E/BO5hpoLze/jvGknLJFU17hgkSZKGwv8XJUmSesPERpIk9YaJjSRJ6g0TG0mS1Bsjncdmdfaufdh3lK8Yqucc8+Nxh7Bot35nZc1Uv9cPV+Baizt2jDuCRauaHncIi+MYBmnZbeNhHq3tATjt5H3rvvunWj3npq9s31hVa4ca3BKMNLHZh305MaeM8hVDtXHj5nGHsGjHX3jeuENYlKf/5/fHHcKi1d0rMOZHHhl3CItSU+3+hTpWjijVCndDffbx/fvun+KLG5/d6jmrnnl7m1nKR8amKEmS1BsuqSBJ0oQrYJoV1oS9G1ZsJElSb1ixkSRp4hVTK23QwW5YsZEkSb1hxUaSpAk308emHyP9rNhIkqTesGIjSZJ6MyrKxEaSpAlXFFM9mXTSpihJktQbVmwkSZKdhyVJkrrGio0kSROugCkrNpIkSd1ixUaSJNnHRpIkqWtWXGKzc9/V3Pynr2XnvqvHHYokSb1QwFRVq61rVlxic89LDueelx/JPS8+fNyhSJKkjpk3sUnyrCRfSrItyZ6zj5cjyEHfWfvcmX++9rnL/WpJknpruuXWNQtJTO4HTgE+uZvjkbrpr87ggRce+vhxHp0C4AdHP5Orrn774+f3v+lOXvh765cjJEmS1FHzVmyqaltVPbC741E74p9uYo9tO34Sz+pVT/gnwB7bdnDERzctV0iSJPVKUUy13Lpm6H1skpyb5PYk92zj4SU/74DNWznuDz/9hORm0B7bdnDcH/wbB9z8nSW/S5IkrWxDT2yqal1Vramqg/Zh36E884DNW3n+RRvZY/vOJ5zfY/tOnn/RRpMaSZKWomCq5dY1K2ZU1M6n7E2mpmFqeqZ6MzVNpqbZ+ZS9xx2aJEnqiIWMitoryVXAscDGJCfOPh51kDAzCmpqn7146h33cewfb+Cpd9zH1D57OTpKkqQlKiZoVFRV7QBOnXV69vHIrXr4UdZ88P/y7E/cTAoOOO/jfPvXjuWBY5653KFIkqSOWjFrRR134b8/4TjTxWGXb+awyzePKSJJkvoiTJFxBzEUK6aPjSRJ0nxWTMVGkiSNRgHTHRzh1IaJjSRJsilKkiSpa6zYSJI04QorNpIkSZ1jxUaSJDFdVmwkSZI6xYqNJEkTzj42kiRJHWTFRpKkCVeEqZ7UOvrxLSRJkrBiI0mScFSUJElS54y0YvOcY37Mxo2bR/mKoVrzkfPGHcKiHbZl+7hDWJQ8/Mi4Q1i0mpoadwiLVittNbtaYfFKPeOoKEmSpA6yj40kSRMvTFU/ah39+BaSJElYsZEkaeIVMN2TWkc/voUkSRJWbCRJEo6KkiRJ6hwTG0mSJlzVzKioNtt8kqxNcluSLUku2MX1tyX5apLNSa5LclRz/vAkjzTnNyf54EK+i01RkiSJ6RE0RSVZBVwMvBq4C7gxyfqqunXgY5dW1Qebz58B/DWwtrn29ao6bjHvtGIjSZJG5QRgS1XdUVWPApcBZw5+oKoeGjjcl5lBWq1ZsZEkacLNLKnQutZxYJJNA8frqmpds38wcOfAtbuAE2c/IMnbgXcBq4FXDVw6IsmXgYeA91TVtfMFY2IjSZKW4t6qOn4pD6iqi4GLk/wm8B7gHOBu4NlVdV+SFwL/muToWRWen2JiI0nSxBvZkgpbgUMHjg9pzu3OZcD/Aaiq7cD2Zv+mJF8HngNs2v3t9rGRJEmjcyOwJskRSVYDZwHrBz+QZM3A4euA25vzBzWdj0lyJLAGuGO+F1qxkSRpwo1qSYWq2pnkfGAjsAq4pKpuSXIRsKmq1gPnJzkV2AE8wEwzFMArgIuS7ACmgbdV1f3zvdPERpIkjUxVbQA2zDp34cD+7+7mvk8An1js+0xsJEkSU+WSCpIkSZ1ixUaSpAlXZCnz2HRKP76FJEkSC0hskjwryZeSbEuyZ3Pu/UmuTfI3ow9RkiSN2nTt0WrrmoVEdD9wCvAFgCS/BDylql4OrE7yohHGJ0mStGDz9rGpqm3AtuTx3tK/DFzZ7F8FvJiZCXgkSdIKtMS1ojqlzbfYj5nFqAAebI6fIMnpSdb94KHppcQmSZK0KG0SmweBpzX7TwN+MPsDVXVFVZ2739P6kf1JktRnRZiqdlvXtMk8/ouZPjcAp9L0vZEkSRq3hYyK2ivJVcCxzKz1sBczfW6uBaaq6osjjlGSJI3YNHu02rpmIZ2HdzBTmRl0w2jCkSRJas+ZhyVJmnBVMNXBOWnaMLGRJGnihWm61xG4jX6kZ5IkSVixkSRp4hX9aYrqx7eQJEnCio0kSWKyl1SQJEnqJCs2kiRNuCJMd3B5hDas2EiSpN6wYiNJkuxjI0mS1DVWbCRJmnAFTDuPjSRJUrdYsZEkaeKFKdeKkiRJ6hYrNpIkTbg+9bEZaWJz63cO4vgLzxvlK4bqsC3bxx3Cou39rfvGHcKi1A9/NO4QFq127Bx3CItX0+OOQJLGwoqNJEmyj40kSVLXWLGRJGnCVaU3fWz68S0kSZKwYiNJkoCpnlRsTGwkSZpwBUzbeViSJKlbrNhIkjTx0pumqH58C0mSJKzYSJI08WaWVLCPjSRJUqdYsZEkSUz1pNbRj28hSZKEFRtJkiZeEfvYSJIkdY0VG0mSxHRPah39+BaSJElYsZEkaeJVwZR9bCRJkrrFio0kSZrsUVFJ9kxyWZLPJXnfsIOSJElqo21T1K8CN1fVycCTkhw7xJgkSdIympnHZo9WW9e0jehI4CvN/mbgJcMJR5Ikqb22ic1twCub/ZOB/R67kOTcJLcnuWfHjx5YanySJGkZTJFWW9e0TWyuYKYJ6rPAduB7j12oqnVVtaaqDtrrKfsPI0ZJkrRCJVmb5LYkW5JcsIvrb0vy1SSbk1yX5KiBa3/Q3HdbktMW8r5WiU1VTVXVO6rqFGAK2NjmOZIkafyKmVFRbba5JFkFXAy8FjgKeNNg4tK4tKqeX1XHAe8D/rq59yjgLOBoYC3wd83z5tR2VNTBSa5JcjVwfVVtbfMcSZLUaycAW6rqjqp6FLgMOHPwA1X10MDhvszkWTSfu6yqtlfVN4AtzfPm1GoemyaROanNvZIkqWuylBFOBybZNHC8rqrWNfsHA3cOXLsLOPGn3p68HXgXsBp41cC9X5h178HzBeMEfZIkien2HYHvrarjl/LuqroYuDjJbwLvAc5p+6zuDUCXJEl9sRU4dOD4kObc7lwGvL7lvYCJjSRJE++xRTDbbPO4EViT5Igkq5npDLx+8ANJ1gwcvg64vdlfD5yVZO8kRwBrgC/O90KboiRJ0khU1c4k5zMzenoVcElV3ZLkImBTVa0Hzk9yKrADeICmGar53MeAW4GdwNuramq+d5rYSJKkkS2PUFUbgA2zzl04sP+7c9z758CfL+Z9NkVJkqTesGIjSdKEm1kEs3vLI7RhxUaSJPWGFRtJkrSUeWw6xYqNJEnqDSs2kiRNuMcWwewDKzaSJKk3rNhIkqSRzWOz3PrxLSRJkrBiI0mSynlsJEmSOmekFZu9friTp//n90f5iqHKw4+MO4RFqx/+aNwhLMr09u3jDmHRamreNdckaUUrnMdGkiSpc+xjI0mS7GMjSZLUNVZsJEmacH2aedjERpIk9SaxsSlKkiT1hhUbSZImXOEEfZIkSZ1jxUaSJDlBnyRJUtdYsZEkadKVo6IkSZI6x4qNJEkTrk8T9FmxkSRJvWHFRpIkWbGRJEnqGis2kiRNOGceliRJ6iArNpIkiepJxaZVYpPkycDHgX2BB4Ffr6rtwwxMkiRpsdo2Ra0Fbqiqk4AvNseSJGmFmiattq5pm9h8nZlqDcB+wH3DCUeSJKm9tn1sbgdenOQW4PvA7w9eTHI6cPqT9tpvieFJkqRRK9eK4hzgiqo6Gvg0cPbgxaq6oqrO3WuPfZYanyRJ0oK1rdgEuL/Zvxf4meGEI0mSxmGiR0UBlwL/kuTNwA7gN4YXkiRJWl79maCvVWJTVT8AThtyLJIkSUviBH2SJKk3TVEuqSBJknrDio0kSROucLi3JElS51ixkSRp0tXMJH19YMVGkiT1hhUbSZLUyQUt27BiI0mSesOKjSRJE65wHhtJkqTOsWIjSdLE689aUVZsJElSb1ixkSRJzmMjSZLUNSY2kiSJqrTa5pNkbZLbkmxJcsEurr8rya1JvpLks0kOG7g2lWRzs61fyPewKUqSJI1EklXAxcCrgbuAG5Osr6pbBz72ZeD4qvpxkvOA9wG/0Vx7pKqOW8w7R5vY7NhB3f39kb5imGpqatwhLFrt2DnuEBZlJf7G1PS4I1i8vjSWS1oWVSObx+YEYEtV3QGQ5DLgTODxxKaqPjfw+S8AZy/lhTZFSZKkUTkYuHPg+K7m3O68Ffj3geN9kmxK8oUkr1/IC22KkiRJS5nH5sAkmwaO11XVusU+JMnZwPHAKwdOH1ZVW5McCVyd5KtV9fW5nmNiI0mSltKCfW9VHb+ba1uBQweOD2nOPUGSU4E/Al5ZVdt/ElNtbf55R5JrgBcAcyY2NkVJkqRRuRFYk+SIJKuBs4AnjG5K8gLgQ8AZVfX9gfP7J9m72T8QeCkDfXN2x4qNJEkaSefhqtqZ5HxgI7AKuKSqbklyEbCpqtYDfwU8Bfh4EoBvV9UZwHOBDyWZZqYQ8xezRlPtkomNJEkamaraAGyYde7Cgf1Td3Pf9cDzF/s+ExtJkiZcsbDJ9lYC+9hIkqTesGIjSZLoy7SeVmwkSVJvWLGRJGnSjW5JhWVnxUaSJPWGFRtJktSbTjZWbCRJUm9YsZEkSfaxkSRJ6horNpIkaSmre3eKFRtJktQbVmwkSZpwhX1sJEmSOqdVYpNkbZJrmu3uJK8fdmCSJGmZFFBpt3VMq6aoqvoM8BmAJDcAVw0zKEmSpDaW1BSV5Ejge1X1o4Fz5ya5Pck9j9TDSw5QkiSNXlW7rWuW2sfm14BPDp6oqnVVtaaqDnpS9l3i4yVJ0rKollvHLDWxOR1YP4xAJEmSlqr1cO8kPwc8WlX3DTEeSZK07OJwb+BM4FPDCkSSJGmpWldsqupDwwxEkiSNUQf7y7ThBH2SJKk3XFJBkqRJVy6pIEmS1DlWbCRJkn1sJEmSusaKjSRJAuxjI0mS1ClWbCRJkn1sJEmSusaKjSRJsmIjSZLUNVZsJEmadAU487AkSVK3WLGRJEmUfWwkSZK6xYqNJEnqzaiokSY2VdPUI4+M8hVDVdMr8D/Vmh53BP3Xl/qsJM3FzsOSJEndYlOUJEkiPSlOW7GRJEm9YcVGkqRJV/Sm87AVG0mS1BtWbCRJmnhxVJQkSVLXWLGRJEn2sZEkSeoaKzaSJMmKjSRJUtdYsZEkSVZsJEmSusaKjSRJk65wHhtJkqSuMbGRJEmk2m3zPjdZm+S2JFuSXLCL6+9KcmuSryT5bJLDBq6dk+T2ZjtnId/DxEaSJI1EklXAxcBrgaOANyU5atbHvgwcX1XHAJcD72vuPQB4L3AicALw3iT7z/dOExtJkvSTFb4Xu83tBGBLVd1RVY8ClwFnPuG1VZ+rqh83h18ADmn2TwOurKr7q+oB4Epg7XwvNLGRJElLcWCSTQPbuQPXDgbuHDi+qzm3O28F/r3lvcASRkUleQtwDrAK+K2q2tr2WZIkacW6t6qOX+pDkpwNHA+8cinPaZXYJDkYeGVVnbKUl0uSpG5YSEfgFrYChw4cH9Kce+K7k1OBP2Imt9g+cO9Js+69Zr4Xtm2KOg1Y1fRe/kDTOUiSJGnQjcCaJEckWQ2cBawf/ECSFwAfAs6oqu8PXNoIvCbJ/k2n4dc05+bUNrF5BrC6qdj8mFkdgZKcnmTdztrR8vGSJGlZVdptcz2yaidwPjMJydeAj1XVLUkuSnJG87G/Ap4CfDzJ5iTrm3vvB/6MmeToRuCi5tyc2vaxeRD4fLN/NTNtYoNf5ArgiqftccDvtHy+JEnqgaraAGyYde7Cgf1T57j3EuCSxbyvbcXmeuCYZv844BstnyNJksat7VDvDi6c2apiU1WbkzyS5BrgXuD9Q41KkiSphdbDvavq3cMMRJIkjVEHqy9tOEGfJEnqjdYVG0mS1B8jmsdm2VmxkSRJvWHFRpIk2cdGkiSpa6zYSJIkKzaSJEldY8VGkqQJl3JUlCRJUudYsZEkSfOu1L1SWLGRJEm9YcVGkiQ5KkqSJKlrrNhIkqTejIoysZEkSTZFSZIkdc1oKzYFNTU10lcMVfUkXZUkaTGcoE+SJKl77GMjSZLsYyNJktQ1VmwkSZIVG0mSpK6xYiNJkhwVJUmS1DUmNpIkqTdMbCRJUm/Yx0aSJDkqSpIkqWus2EiSNOlcK0qSJKl7rNhIkiT72EiSJHWNFRtJkmTFRpIkqWus2EiSNOGCo6IkSZI6x4qNJEnqTR+bVolNksOBG4CvAY9W1WuGGJMkSVpOPZqgbykVmyur6uyhRSJJkrRES+ljc3KSa5O8c/BkknOT3J7knm38eInhSZKkZVEtt45pm9jcDTwHOBk4Nckxj12oqnVVtaaqDtqHJw8jRkmSpAVpldhU1faqeriqdgL/BjxvuGFJkqRlNckVmyRPHTh8KfD14YQjSZLUXtumqJcnuSnJ9cDWqrphmEFJkqTllWq3dU2rUVFVtQHYMORYJEmSlsQJ+iRJUif7y7ThkgqSJKk3rNhIkjTpOjrCqQ0rNpIkqTes2EiSpE6OcGrDio0kSRqZJGuT3JZkS5ILdnH9FUm+lGRnkjfMujaVZHOzrV/I+6zYSJKkkfSxSbIKuBh4NXAXcGOS9VV168DHvg38NvDuXTzikao6bjHvNLGRJEmjcgKwparuAEhyGXAm8HhiU1XfbK5ND+OFNkVJkqSlzDx8YJJNA9u5A489GLhz4Piu5txC7dM88wtJXr+QG6zYSJKkpbi3qo4f0bMPq6qtSY4Erk7y1aqac31KKzaSJGlUq3tvBQ4dOD6kObewkKq2Nv+8A7gGeMF895jYSJKkUbkRWJPkiCSrgbOABY1uSrJ/kr2b/QOBlzLQN2d3TGwkSZp0bas181RsqmoncD6wEfga8LGquiXJRUnOAEjyoiR3AW8EPpTklub25wKbktwMfA74i1mjqXbJPjaSJE24NNsoVNUGYMOscxcO7N/ITBPV7PuuB56/2PdZsZEkSb1hxUaSJPVmEczRJzbVk19KkiR1nhUbSZLkIpiSJEldY8VGkiT1po+NFRtJktQbVmwkSZIVG0mSpK6xYiNJ0qQrR0VJkiR1jhUbSZJkHxtJkqSusWIjSZLsYyNJktQ1VmwkSZJ9bCRJkrrGio0kSbKPjSRJUtdYsZEkadIVveljY2IjSZJ6k9jYFCVJknpjSYlNkncmuW5YwUiSpOUXZjoPt9m6pnVik2Rv4LghxiJJkrQkS6nYvBX48LACkSRJY1Qtt45pldgk2Qs4qaqu3s3105Os28mjSwpOkiRpMdpWbN4MXLq7i1V1RVWduyerWz5ekiQtp1S12rqmbWLzi8B5ST4DHJ3kHUOMSZIkqZVW89hU1e8/tp/kuqr6wPBCkiRJy6qj/WXaWPI8NlX1smEEIkmStFTOPCxJkjo5J00bzjwsSZJ6w4qNJEmyj40kSVLXWLGRJEn2sZEkSeoaKzaSJMk+NpIkSV1jxUaSpElX9rGRJEnqHCs2kiSpN31sTGwkSZpwwaYoSZKkzrFiI0mSoPpRsrFiI0mSesOKjSRJso+NJElS11ixkSRp0hW9Ge5txUaSJPWGFRtJkkSmxx3BcFixkSRJvWHFRpIk2cdGkiSpa0xsJEkSqXbbvM9N1ia5LcmWJBfs4vorknwpyc4kb5h17ZwktzfbOQv5HiY2kiRpJJKsAi4GXgscBbwpyVGzPvZt4LeBS2fdewDwXuBE4ATgvUn2n++dJjaSJE26YmatqDbb3E4AtlTVHVX1KHAZcOYTXl31zar6CjB7XNZpwJVVdX9VPQBcCayd74UmNpIkaSkOTLJpYDt34NrBwJ0Dx3c15xai1b2OipIkSUtZK+reqjp+iKEsiRUbSZI0KluBQweOD2nOjexeExtJkvST9aIWu83tRmBNkiOSrAbOAtYvMKKNwGuS7N90Gn5Nc25OJjaSJGkkqmoncD4zCcnXgI9V1S1JLkpyBkCSFyW5C3gj8KEktzT33g/8GTPJ0Y3ARc25OdnHRpKkCReW1MdmTlW1Adgw69yFA/s3MtPMtKt7LwEuWcz7TGwkSZp0Cxu6vSLYFCVJknrDio0kSRpZU9Rys2IjSZJ6w4qNJElayNDtFaFVxSbJ85Jcn+TaJP+QJMMOTJIkabHaNkXdVlUvqaqXN8edmUpZkiQtXqrd1jWtEpuq2jFwuJ2BRaqSnJvk9iT3bOPhpcYnSZK0YK07Dyc5I8l/A88A7nvsfFWtq6o1VXXQPuw7jBglSdIoFTBd7baOaZ3YVNX6qnoeM8uI/8rwQpIkSWqn1aioJHtX1fbm8CHgkeGFJEmSll33ii+ttB3uvTbJu5r924H/GFI8kiRJrbVKbKrqU8CnhhyLJEkaky6OcGrDmYclSVJvOPOwJElydW9JkqSusWIjSZLsYyNJktQ1VmwkSZp0RW/msbFiI0mSesOKjSRJEy5AejIqysRGkiTB9LgDGA6boiRJUm9YsZEkSb1pirJiI0mSesOKjSRJk87h3pIkSd1jxUaSpIlXLoIpSZLUNVZsJEmSi2BKkiR1jRUbSZJkHxtJkqSusWIjSdKkK4hrRUmSJHWLFRtJkmQfG0mSpK6xYiNJklwrSpIkqWus2EiSJGIfG0mSpG6xYiNJknozKsrERpKkSVeAE/RJkiR1ixUbSZImXCg7D0uSJHWNFRtJktSbzsNWbCRJUm9YsZEkSZNdsUlyYpLrk1yX5P3DDkqSJKmNtk1R3wJeVVUvA56e5PlDjEmSJC2nx+axabN1TKumqKr67sDhDmBqOOFIkiS1t6Q+NkmOAQ6qqltnnT8dOP1J7LuUx0uSpGUy8fPYJDkA+FvgrbOvVdUVVXXunqxeSmySJGmFS7I2yW1JtiS5YBfX907yL831G5Ic3pw/PMkjSTY32wcX8r5WFZskewIfBd49q1lKkiStRCOo2CRZBVwMvBq4C7gxyfpZLT1vBR6oql9Ichbwl8BvNNe+XlXHLeadbSs2bwReBLwvyTVJXtzyOZIkqb9OALZU1R1V9ShwGXDmrM+cCXy42b8cOCVJ2r6wbefhfwb+ue1LJUlSl9RSKjYHJtk0cLyuqtY1+wcDdw5cuws4cdb9j3+mqnYmeRD42ebaEUm+DDwEvKeqrp0vGCfokyRJS3FvVR0/gufeDTy7qu5L8kLgX5McXVUPzXWTSypIkjTpipmKTZttbluBQweOD2nO7fIzTR/enwHuq6rtVXUfQFXdBHwdeM58LzSxkSRJo3IjsCbJEUlWA2cB62d9Zj1wTrP/BuDqqqokBzWdj0lyJLAGuGO+F9oUJUmSRjKLcNNn5nxgI7AKuKSqbklyEbCpqtYDfw98JMkW4H5mkh+AVwAXJdnRRPe2qrp/vnea2EiSpJGpqg3AhlnnLhzY38bMaOvZ930C+MRi32diI0mSnHlYkiSpa6zYSJKkkcw8PA4mNpIkTboCpvuR2NgUJUmSesOKjSRJE29JSyp0ihUbSZLUG1ZsJEmSFRtJkqSusWIjSZKs2EiSJHWNFRtJkiad89hIkiR1jxUbSZImXkFNjzuIobBiI0mSesOKjSRJclSUJElS11ixkSRp0jkqSpIkqXus2EiSJPvYSJIkdY0VG0mSZMVGkiSpa6zYSJI08ao3FRsTG0mSJl0B0y6pIEmS1ClWbCRJUm+aoqzYSJKk3rBiI0mSrNhIkiR1jRUbSZImXk32IphJnpXkS0m2JTE5kiRJndA2KbkfOAX45BBjkSRJ41BQ1Y95bFolNlW1DdiWZMjhSJIktTf0zsNJzk1ye5J7tvHwsB8vSZJGYbrabR0z9MSmqtZV1ZqqOmgf9h324yVJknbLjr+SJGmy57FJsleSq4BjgY1JThxuWJIkSYvXtvPwDuDUIcciSZLGocrVvSVJkrrGPjaSJGmy+9hIkiR1kRUbSZJE2cdGkiSpW6zYSJI08ao3fWxMbCRJmnRFJ5dHaMOmKEmulf6SAAAF3UlEQVSS1BtWbCRJEpSdhyVJkjrFio0kSROugLKPjSRJUrdYsZEkadJV2cdGkiSpa0xsJEkSNV2ttvkkWZvktiRbklywi+t7J/mX5voNSQ4fuPYHzfnbkpy2kO9hYiNJkkYiySrgYuC1wFHAm5IcNetjbwUeqKpfAN4P/GVz71HAWcDRwFrg75rnzcnERpIkzfSxabPN7QRgS1XdUVWPApcBZ876zJnAh5v9y4FTkqQ5f1lVba+qbwBbmufNaaSdh3/IA/deVZd/awSPfjbw7RE8d1R+Bnhw3EEskr/x6Pkbj56/8WittN8X/I0HHfbYzg95YONVdfmBLZ+zT5JNA8frqmpds38wcOfAtbuAE2fd//hnqmpnkgeBn23Of2HWvQfPF8xIE5uqOmgUz01yT1UdP4pnj0KSdVV17rjjWAx/49HzNx49f+PRWmm/L/gb705VrR31O5bLSm2K+sG4A1ikK8YdQAv+xqPnbzx6/sajtdJ+X/A3Xm5bgUMHjg9pzu3yM0n2ZKaqdt8C7/0pKzWxWUllRKpqpf0PCfyNl4O/8ej5G4/Wivp9wd94DG4E1iQ5IslqZjoDr5/1mfXAOc3+G4Crq6qa82c1o6aOANYAX5zvhSt1gr51839ES+RvPHr+xqPnbzxa/r6jt6J/46bPzPnARmAVcElV3ZLkImBTVa0H/h74SJItwP3MJD80n/sYcCuwE3h7VU3N987MJEWSJEkr30ptipIkSfopKyqxSfKsJF9Ksq3pYNRpSU5Mcn2S65K8f9zxLESS5zUxX5vkH5q5BDovyTuTXDfuOBYiyeFJvpfkmiT/Me54FiLJW5J8tol53uGW49bMdHpNs92d5PXjjmkuSZ6c5NNNvJ9Ksve4Y5pLkj2TXJbkc0neN+545rKrvxtJ3t/8O+5vxh3fbLPjXWl/97pgRSU2zLS9ncITx7V32beAV1XVy4CnJ3n+uANagNuq6iVV9fLmuPNDOZs/AseNO45FurKqTqqq14w7kPk0icwrq+qUJuZ5RyWMW1V9pon1JGbmALlqzCHNZy1wQxPvF5vjLvtV4OaqOhl4UpJjxx3QHJ7wdyPJLwFPaf4dtzrJi8YZ3C7M/ju30v7ujd2KSmyqaltVPTDuOBaqqr5bVduawx3AvJ2exq2qdgwcbueJEyt11Vv5yayVK8XJzf9jfOe4A1mA04BVTcXmAwuZ0rwrkhwJfK+qfjTuWObxdWDfZn8/Zoa6dtmRwFea/c3AS8YYy5x28Xfjl4Erm/2rgBcvf1S7NzvelfZ3rwtWVGKzUiU5Bjioqm4ddywLkeSMJP8NPIOO/ws2yV7ASVV19bhjWYS7gecAJwOnNv/96LJnAKur6hTgx/z0dOhd9mvAJ8cdxALcDrw4yS3MVEmvH3M887kNeGWzfzIzydhKsR/wULP/ICsrdi2Aic2IJTkA+FtmqgorQlWtr6rnMTN99a+MO555vBm4dNxBLEaz7snDVbUT+DfgeeOOaR4PAp9v9q8GnjvGWBbrdH56zowuOge4oqqOBj4NnD3meOZzBTNNUJ9lprL7vTHHsxgPAk9r9p/Gyp8AT7OY2IxQ09Hro8C7q+q7445nIWZ1WnwIeGRcsSzQLwLnJfkMcHSSd4w7oPkkeerA4UuZaYbosuuBx6pKxwHfGGMsC5bk54BHq6rTVcdGmOlLAXAvMzOvdlZVTVXVO5oq3hQzc5SsFP/FTJ8VgFOx70rvrKjEJsleSa4CjgU2Jpm9kFbXvBF4EfC+ZrRDp9pyd2Ntks8n+TwzTRCdHrVTVb9fVac165zcUlUfGHdMC/DyJDcluR7YWlU3jDuguVTVZuCRJNcw89/ny8cb0YKdCXxq3EEs0KXArze/8W8B/zTecOaW5ODm32lXA9d3uUP57L8bwF7AtiTXAlNVNe9MtstpV3/nVtjfvbFzgj5JktQbK6piI0mSNBcTG0mS1BsmNpIkqTdMbCRJUm+Y2EiSpN4wsZEkSb1hYiNJknrj/wNu6ITtT0uj6gAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 1440x1440 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"def get_value_grid(V_array, nvmdp, s_to_idx):\n",
" \"\"\"Returns value over states space grid.\n",
" \"\"\"\n",
" V = np.zeros((nvmdp.height, nvmdp.width), dtype=np.float32)\n",
" for row in range(nvmdp.height):\n",
" for col in range(nvmdp.width):\n",
" x, y = nvmdp._rowcol_to_xy(row, col)\n",
" V[row, col] = V_array[s_to_idx[NavigationWorldState(x, y)]]\n",
" return V\n",
" \n",
"## Greedy Policy\n",
"# v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, lambda Q: np.max(Q), 100)\n",
"## Boltzmann Policy\n",
"boltz_temp = 100\n",
"v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, \n",
" lambda Q, t=boltz_temp: np.exp((1/t)*Q) / (np.exp((1/t)*Q).sum()), 100)\n",
"V = get_value_grid(v, nvmdp, s_to_idx)\n",
"\n",
"plt.figure(figsize=(20,20))\n",
"nvmdp.visualize_grid(V, trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0. , 0.39, 0.13, 0.04, 0.01, 0.01, 0. , 0. , 0. , 0. , 0. ],\n",
" [0.39, 0.19, 0.08, 0.03, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0.13, 0.08, 0.04, 0.02, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0.04, 0.03, 0.02, 0.01, 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0.01, 0.01, 0.01, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0.01, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ]],\n",
" dtype=float32)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V.round(2)"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAIVCAYAAAAZNR+PAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+QbnV9J/j3hwtXEH8wEaIRSNAMmRpi1M2iRqMRhIlkEyCTMVmT0ZgtZ9lJYjal664ms2PNMjNbM07VWK462dzNJmsmUZJhy/USWYlocDQkCDHGDToMxEkClD/4ISjgvRe6P/tHP5dp2tu3u5/u0+center6tQ953lOn+fDU3D7w/v7Pd9T3R0AgKk4buwCAAC2QvMCAEyK5gUAmBTNCwAwKZoXAGBSNC8AwKRoXmCCquqsquqqOn7sWuZVVX9ZVRfO9n+5qn5t7JqAadC8wAiq6sNVdfkRXr+0qr40dlNSVa+uqhuq6sGq+sps/+eqqob4vO7+X7v7H2z3Oo+Hpg7YmOYFxvHeJK85QjPw2iS/3d2PjFBTkqSq/ock70zyr5I8I8nTk/zDJN+fZO86P7Nn1woEFp7mBcbx/yR5WpKXHX6hqv5Gkh9J8puz4x+uqj+tqq9V1e1V9U/Wu9jqIZjZ8T+pqt9adfx9VXV9Vd1XVX9WVeetc52nJrk8yc9195Xd/fVe8afd/fe7++DsvP+rqn6lqq6uqgeTnL9RvVX12qr6q6q6p6r+0Zr3Nl1vVV1XVf+0qv6wqr5eVb9fVafO3v73sz/vq6oHqurF631nwHRpXmAE3f2NJL+b5KdXvfwTSf5Dd//Z7PjB2funJPnhJD9bVT+61c+qqtOTfCjJP0vyLUnenOT/rqrTjnD6i5M8IckHN3Hpn0ryz5M8Ocknj1ZvVZ2T5Feykiw9MyuN2xnbqPenkvw3Sb41K2nQm2ev/8Dsz1O6+0nd/Ueb+OcAJkbzAuN5b5JXVdWJs+Ofnr2WJOnu67r7/+vu5e7+bJL3J3n5HJ/zmiRXd/fVs2t9JMlNSf6rI5x7apK7Vw9brUpAvlFVP7Dq3A929x/Ornlgg3pfleT3uvvfz9Kbf5xkeRv1/kZ3/8dVTeDzt/ytAJOleYGRdPcnk9yd5Eer6juTvDDJ+w6/X1Uvqqo/qKq7qur+rMw7OfXIVzuq70jy47MG5L6qui/JS5N82xHOvSfJqasnvHb3S7r7lNl7q//OuH31D25Q7zNXn9/dD86uN2+9X1q1/1CSJ633Dw88/mheYFy/mZXE5TVJrunuL696731J9ic5s7ufmuR/T7Le3T4PJnniquNnrNq/Pcm/7e5TVm0nd/e/OMJ1/ijJwSSXbqL2tY+kP1q9X0xy5uETq+qJWRk6OpKt1LtRTcDjkOYFxvWbSS5M8t9m1ZDRzJOT3NvdB6rqhVmZ57GezyR5dVWdUFXnZmWY5rDfSnJxVb2yqvZU1YlVdV5VfdOck+6+L8n/kuTfVNWrqurJVXVcVT0/yckb/LMcrd4rk/xIVb20qvZmZVLwen//bLreI7grK8NRz97EucBEaV5gRN39l0muz0pjsH/N2z+X5PKq+nqSt2Vlbsd6/nGS70zy1aw0H48OP3X37VlJUn45K7/cb0/yP2ad//67++1J3pTkf0ry5dn2q0neMqt1PevW2903J/n5WV1fnNV5xzqfv6V61/zsQ1mZRPyHsyGn79voZ4DpqW4pKwAwHZIXAGBSNC8AwKRoXgCASdG8AACTMuiTV0/9lj191pknDPkRO+rPH1hv2Ylj196vjF3B1tSBQ2OXsGW9tN5CsMcwE/GBDRzIgznUBytJXnn+yX3PvUtzXedPPnvwmu6+aEeL28CgzctZZ56QT11z5sYnHiO+6+OvG7uELTvr3eutWXZsOv6W2zc+6RizfP/Xxy5hy/qRh8cuYWs0W7DrbuiPPrp/z71L+dQ13z7XdfZ8263zrPy9LYaNAIBJGTR5AQCOfZ1ked1npR57JC8AwKRIXgBg4XWWWvICADAIyQsALLiVOS/TuetP8gIATIrkBQCY1N1GmhcAWHCdztKEFos0bAQATIrkBQAwYRcAYCiSFwBYcJ1kSfICADAMyQsAYM4LAMBQJte83L90Un7sjp/N/UsnjV0KADwudJKl7rm2MUyuedn/wPPywQe+N1c98NyxSwEARrBh81JVz6yqT1fVgao6fu3xbhS52m/c//2P+RMA2L7lObcxbKb5uDfJBUk+sM7xoH7wr9+Yjz50zqPHe/NwkuT6h/5m9vyH/+PR1y944ufy+9/+jt0oCQAY0YbJS3cf6O6vrnc8tF962tV5Yh189PhQTnjMn0nyxDqYX37ah3arJAB4XOl0lubcxrDjc16q6rKqurWq7vrrOx/e9vXOP/mW7D/jXY9pYFZ7Yh3MVWf8bznv5P+47c8CAI59O968dPe+7j67u0/79tNP2PgHNuH8k2/J+0/flxPr0GNeP7EO5f2n79O4AMB2dLI05zaGydxtdN/SSTk+yzkuSzmpDuW4LOX4LOc+t0wDwELZzN1GJ1TVtUmel+SaqnrR2uOhi0ySX7//pXmo9+a5T7gzHzjj3XnuE+7MQ73XXUcAsE2dx9ndRt39cJIL17y89nhwTz3uG3n7t16ZX/wb1+a46rzirH+Wd957YT7xjbN3uxQAYESTebbRB874N4853lOdNz3tI3lTPjJSRQDweFFZSo1dxKZNZs4LAEAyoeQFABhGJ1mezkOlNS8AQAwbAQAMRfICAAuuI3kBABiM5AUAyHJLXgAABiF5AYAFZ84LAMCAJC8AsOA6laUJ5RnTqRQAIJIXACDuNgIAGMygycufP/C0fNfHXzfkR+yop+0/aewStuz4u+4au4StOfTw2BVsXS+PXQHAoNxtBAAwIHNeAGDhVZZ6OnnGdCoFAIjmBQAWXidZznFzbRupqouq6paquq2q3nqE93+mqu6qqs/Mtn+w0TUNGwEAg6iqPUnek+TvJLkjyY1Vtb+7P7fm1N/p7jds9rqaFwBgqLuNXpjktu7+QpJU1RVJLk2ytnnZEsNGAMB2nFpVN63aLlv13ulJbl91fMfstbX+XlV9tqqurKozN/pAyQsALLjubd1tdHd3n7uNj78qyfu7+2BV/XdJ3pvkFUf7AckLAJDl1FzbBu5MsjpJOWP22qO6+57uPjg7/LUk/+VGF9W8AABDuTHJ2VX1rKram+TVSfavPqGqvm3V4SVJPr/RRQ0bAcCCW3k8wM7nGd39SFW9Ick1SfYk+fXuvrmqLk9yU3fvT/LfV9UlSR5Jcm+Sn9noupoXAGAw3X11kqvXvPa2Vfu/lOSXtnJNzQsALDyPBwAAGIzkBQAW3OHHA0zFdCoFAIjkBQBIstSDPB5gEJIXAGBSJC8AsOA6Ncg6L0OZTqUAANlE81JVz6yqT1fVgao6fvbaO6rqE1X1zuFLBACGttzHzbWNYTOfem+SC5L8cZJU1fcmeVJ3vyzJ3qp6wYD1AQA8xoZzXrr7QJIDVY/OQv6+JB+Z7V+b5MVZefASADBBQz3baCjzVHpKkq/N9u+fHT9GVV1cVfuWHjywndoAAL7JPM3L/UmeMtt/SpL71p7Q3Vd192V7Tj5xO7UBALugU1nq+bYxzNO8/FFW5sAkyYWZzYUBANgNm7nb6ISqujbJ85Jck+SErMyB+USSpe7+1MA1AgADW85xc21j2MyE3YezkrCsdsMw5QAAHJ0VdgFgwXUnSyOt2TIPzQsALLzKcjyYEQBgEJIXAFhwnWkNG02nUgCASF4AgDz+Hw8AADAayQsALLhOZXmkpf7nIXkBACZF8gIAmPMCADAUyQsALLhOsmydFwCAYUheAGDhVZY82wgAYBiSFwBYcFOb8zJo87L3K8lZ755ODHX8XXeNXcLW3fPVsSvYkuWDB8cuYct6uccuYet6gjUDbJLkBQAw5wUAYCiSFwBYcN01qTkv06kUACCSFwAgydKEkhfNCwAsuE6ybMIuAMAwJC8AsPBqUsNG06kUACCSFwBYeCuPBzDnBQBgEJIXACBLE8ozplMpAEAkLwCw8DplzgsAwFAkLwBAlieUZ0ynUgCASF4AYOF1J0vmvAAADEPyAgA8/u82qqrjq+qKqvqDqnr7ThcFALCeeYeN/m6SP+vu85OcVFXP28GaAIBdtLLOy3FzbWOY91OfneSzs/3PJHnJzpQDAHB08zYvtyR5+Wz//CSnHH6jqi6rqlur6q4DB+/fbn0AwC5YSs21jWHe5uWqrAwXfTTJwSRfPvxGd+/r7rO7+7QTn/DUnagRAOBRc91t1N1LSX4hSapqX5JrdrIoAGD3dKZ1t9FczUtVnZ7kt5MsJ/nN7r5zR6sCAFjHvMnLnUnO29lSAIBx1Gh3Ds3DInUAQJZHmnw7j+m0WQAAkbwAwMLzYEYAgAFJXgCASU3YnU6lAACRvADAwlt5MKM5LwAAg5C8AADWeQEAGIrkBQAW3NQezCh5AQAmRfICAFjnBQBgKJIXAFh0bZ0XAIDBDJq81IFDOf6W24f8iJ116OGxK9iy5YMHxy5hS/rhR8YuYet6eewKAAbVsc4LAECSpKouqqpbquq2qnrrUc77e1XVVXXuRtc05wUAGGTOS1XtSfKeJH8nyR1Jbqyq/d39uTXnPTnJLya5YTPXlbwAAEN5YZLbuvsL3X0oyRVJLj3Cef80yb9McmAzF9W8AMCCO7zC7jzbBk5Psnry6x2z1x5VVd+b5Mzu/tBm6zVsBABsZ9jo1Kq6adXxvu7et5kfrKrjkvzrJD+zlQ/UvAAA23F3d683yfbOJGeuOj5j9tphT07ynCTXVVWSPCPJ/qq6pLtXN0SPoXkBgAXXGWyRuhuTnF1Vz8pK0/LqJD/16Od235/k1MPHVXVdkjcfrXFJzHkBAAbS3Y8keUOSa5J8PsnvdvfNVXV5VV0y73UlLwDAYIvUdffVSa5e89rb1jn3vM1cU/ICAEyK5AUAFl0Ps0jdUCQvAMCkSF4AYMEdXqRuKiQvAMCkSF4AAMkLAMBQJC8AsOAGXGF3EJIXAGBSJC8AQHpCyctczUtVPTHJv0tycpL7k/xEdx/cycIAAI5k3mGji5LcMHsGwadmxwDARC2n5trGMG/z8hdZSV2S5JQk9+xMOQAARzfvnJdbk7y4qm5O8pUkb1n9ZlVdnOTik457yjbLAwCG1gvybKPXJbmqu787yYeSvGb1m919VXdfdsJxe7dbHwDAY8ybvFSSe2f7dyd56s6UAwCM4XF/t1GS9yX5nap6bZKHk/zXO1cSALC7prVI3VzNS3ffl+SVO1wLAMCGLFIHAExq2MjjAQCASZG8AMCC6yzGrdIAAKOQvADAouuVheqmQvICAEyK5AUAGO0hi/OQvAAAkyJ5AYAF17HOCwDAYCQvALDwpvVsI8kLADApkhcAwDovAABDkbwAAO42AgAYyqDJSy8tZ/n+rw/5ETurl8euYMt6eUKDlMkkv+NJDQQDzKFb8gIAMBhzXgCASa3zonkBACY1Qm7YCACYFMkLAGDCLgDAUCQvALDgOiV5AQAYiuQFAMiEbjaSvAAA0yJ5AYBF5/EAAADDkbwAAJOa9CJ5AQAmRfICAJjzAgAwFMkLAOCp0gAAQ5G8AMCC65jzAgAwmLmal6q6qKqum21frKof3enCAIBd0km65ttGMNewUXd/OMmHk6Sqbkhy7U4WBQCwnm0NG1XVs5N8ubsfWPXaZVV1a1XddaAf3HaBAMDwuufbxrDdOS8/luQDq1/o7n3dfXZ3n3ZinbzNywMAu6Ln3Eaw3ebl4iT7d6IQAIDNmPtW6ap6RpJD3X3PDtYDAOy6WphbpS9N8sGdKgQAYDPmTl66+1d3shAAYEQeDwAAMAyPBwCARdceDwAAMBjJCwBgzgsAwFAkLwBAEnNeAAAGIXkBAMx5AQAYiuQFAJC8AAAMRfICAIuuk1hhFwBgGJIXACBtzgsAwDAkLwDApO42GrZ56U4/8vCgH7HwppTzAXDsGmjCblVdlOSdSfYk+bXu/hdr3v+HSX4+yVKSB5Jc1t2fO9o1DRsBAIOoqj1J3pPkh5Kck+Qnq+qcNae9r7u/p7ufn+TtSf71RtfVvAAAqZ5v28ALk9zW3V/o7kNJrkhy6eoTuvtrqw5PziYGsMx5AQCGcnqS21cd35HkRWtPqqqfT/KmJHuTvGKji0peAGDR9Ta25NSqumnVdtmWP777Pd39nUnekuR/3uh8yQsAsB13d/e567x3Z5IzVx2fMXttPVck+ZWNPlDyAgALr1buNppnO7obk5xdVc+qqr1JXp1k/2M+uersVYc/nOTWjS4qeQEABtHdj1TVG5Jck5VbpX+9u2+uqsuT3NTd+5O8oaouTPJwkq8med1G19W8AACDLVLX3VcnuXrNa29btf+LW72mYSMAYFIkLwDApB4PIHkBACZF8gIASF4AAIYieQGARdcZ7KnSQ5C8AACTInkBADbzhOhjhuQFAJgUyQsA4G4jAIChzN28VNVPV9VHq+q6qjp9J4sCAFjPXMNGs2bl5d19wQ7XAwCMYBEm7L4yyZ5Z8vKuqtqzk0UBAKxn3ubl6Un2zpKXh5JcuvrNqrq4qvY9kkPbrQ8A2A1d820jmLd5uT/Jx2f7H0vyt1e/2d1Xdfdlx2fvdmoDAPgm8zYv1yd57mz/+Un+086UAwDsut7GNoK5Jux292eq6htVdV2Su5O8Y0erAgBYx9yL1HX3m3eyEABgRAtwtxEAwCg8HgAAWIh1XgAARiF5AQDMeQEAGIrkBQCQvAAADEXyAgALrtrdRgAAg5G8AACjPSF6HpIXAGBSJC8AgLuNAACGInkBACZ1t5HmBQAwbAQAMJThk5eeUCsHAIvIInUAAMMx5wUAMOcFAGAokhcAQPICADAUyQsA4G4jAIChaF4AgEnRvAAAk2LOCwDgbiMAgKFIXgBg0Xm2EQDAcCQvAIA5LwAAQ5G8AACSFwCAoUheAGDBVdxtBAAwGMkLADCpOS9zNS9VdVaSG5J8Psmh7v7BHawJANhNE1ukbjvJy0e6+zU7VgkAwCZsZ87L+VX1iap64+oXq+qyqrq1qu46kAe3WR4AsCt6zm0E8zYvX0zyXUnOT3JhVT338Bvdva+7z+7u007MyTtRIwDAo+ZqXrr7YHc/2N2PJPm9JM/Z2bIAgF31eE9equrJqw6/P8lf7Ew5AABHN++w0cuq6k+q6vokd3b3DTtZFACwu6rn28Yw191G3X11kqt3uBYAgA1ZpA4AmNQidR4PAABMiuQFABbdiHcOzUPyAgBMiuQFAJjUs40kLwDApEheAABzXgAAhiJ5AQDMeQEAGIrkBQAw5wUAYCiSFwBYdBNbYVfzAgALrmbbVBg2AgAmRfMCAPznoaOtbhuoqouq6paquq2q3nqE999UVZ+rqs9W1Uer6js2uqbmBQAYRFXtSfKeJD+U5JwkP1lV56w57U+TnNvdz01yZZK3b3RdzQsAkOr5tg28MMlt3f2F7j6U5Iokl64+obv/oLsfmh3+cZIzNrqo5gUAGMrpSW5fdXzH7LX1vD7J/7vRRd1tBABs51bpU6vqplXH+7p731YvUlWvSXJukpdvdK7mBQDYjru7+9x13rszyZmrjs+YvfYYVXVhkn+U5OXdfXCjDzRsBAAMdbfRjUnOrqpnVdXeJK9Osn/1CVX1XyT51SSXdPdXNlOq5gUAGER3P5LkDUmuSfL5JL/b3TdX1eVVdcnstH+V5ElJ/l1Vfaaq9q9zuUcZNgKARbe5O4fmu3T31UmuXvPa21btX7jVa0peAIBJkbwAAJN6MKPkBQCYFMkLADDYnJchSF4AgEmRvAAA5rwAAAxF8gIAmPMCADAUyQsALLrNPafomKF5AQAm1bwYNgIAJmVbzUtVvbGqPrlTxQAAu6+yMmF3nm0MczcvVfWEJM/fwVoAADa0neTl9Uneu1OFAAAj6jm3EczVvFTVCUnO6+6PrfP+xVW175Ec2lZxAABrzZu8vDbJ+9Z7s7uv6u7Ljs/eOS8PAOym6p5rG8O8zcvfSvKzVfXhJN9dVb+wgzUBAKxrrnVeuvsth/er6pPd/a6dKwkA2FUTW6Ru2+u8dPdLd6IQAIDNsMIuAODBjAAAQ5G8AACLNecFAGA3SV4AAHNeAACGInkBAMx5AQAYiuQFABZdm/MCADAYyQsAMKk5L5oXAFhwFcNGAACDkbwAAElPJ3qRvAAAkyJ5AQDMeQEAGIrkBQAWXWdSt0pLXgCASZG8AACp5bEr2DzJCwAwKZIXAMCcFwCAoUheAADrvAAADEXyAgCLruPZRgAAQ5G8AADmvAAADEXyAgBY5wUAYCiSFwBYcJVpzXnRvADAout2qzQAwFAkLwDApIaNJC8AwKRIXgCAx/+t0lX1nKq6vqo+UVW/UVW104UBABzJvMNGt3T3S7r7ZbPjc3eqIABg91XPt41hrualux9edXgwye2HD6rqsqq6taruOpAHt1sfAMBjzD1ht6ouqao/T/L0JPccfr2793X32d192ok5eSdqBACG1EmWe75tBHM3L929v7ufk+SOJD+ycyUBAKxvrruNquoJ3X1wdvi1JN/YuZIAgF03obuN5r1V+qKqetNs/9Ykv79D9QAAHNVczUt3fzDJB3e4FgBgJFbYBQAYiBV2AQBPlQYAGIrkBQAw5wUAYCiSFwBYdJ1JrfMieQEAJkXyAgALrpLUhO420rwAAMny2AVsnmEjAGBSJC8AwKSGjSQvAMCkSF4AYNG5VRoAYDiaFwBYeL3yYMZ5tg1U1UVVdUtV3VZVbz3C+z9QVZ+uqkeq6lWbqVbzAgAMoqr2JHlPkh9Kck6Sn6yqc9ac9tdJfibJ+zZ7XXNeAIChHsz4wiS3dfcXkqSqrkhyaZLPHT6hu/9y9t6mV5qRvAAA23FqVd20arts1XunJ7l91fEds9e2RfICAGxq/so67u7uc3eylI1IXgCAodyZ5MxVx2fMXtsWyQsALLpONj/jZEtuTHJ2VT0rK03Lq5P81HYvKnkBAAbR3Y8keUOSa5J8PsnvdvfNVXV5VV2SJFX1gqq6I8mPJ/nVqrp5o+tKXgCA7cx52eCyfXWSq9e89rZV+zdmZThp0yQvAMCkSF4AAM82AgAYiuQFAEgNNOdlCJIXAGBSJC8AwGB3Gw1B8wIAi66TDLNI3SAMGwEAkyJ5AYAFV2kTdgEAhiJ5AQAmNWFX8gIATIrkBQB4/CcvVfWiqrq+qj5ZVe/Y6aIAANYz77DRXyV5RXe/NMm3VtX37GBNAMBuOrzOyzzbCOYaNuruL606fDjJ0s6UAwBwdNua81JVz01yWnd/bs3rFye5+KScvJ3LAwC7ZCHWeamqb0ny7iSvX/ted1/V3Zcdn73bqQ0A4JvMlbxU1fFJfivJm9cMIQEAU7QAycuPJ3lBkrdX1XVV9eIdrAkAYF3zTth9f5L373AtAMAoeiGSFwCAUVhhFwAWXUfyAgAwFMkLADDaarnzkLwAAJMieQEAFmOFXQCAMUheAIBJ3W2keQGARddJlqfTvBg2AgAmRfICAAvP4wEAAAYjeQEAJC8AAEORvAAAkhcAgKFIXgBg0VnnBQBgOJIXAFh4nfTy2EVsmuQFAJgUyQsA4G4jAIChSF4AYNG52wgAYDiSFwDAnBcAgKFIXgAAyQsAwFAkLwCw8HpSyYvmBQAWXSdZ9ngAAIBBSF4AgEkNG0leAIBJkbwAAJIXAIChSF4AYOH14//BjFX1zKr6dFUdqCoNEACwa+ZtPO5NckGSD+xgLQDAGDrpns46L3M1L919IMmBqtrhcgAAjm7HJ+xW1WVVdWtV3XUgD+705QGAISz3fNsIdrx56e593X12d592Yk7e6csDAAvOZFsA4PG/zktVnVBV1yZ5XpJrqupFO1sWAMCRzTth9+EkF+5wLQDAGLo9VRoAYCjmvAAAj/85LwAAY5G8AABpc14AAIYheQGAhdeTmvOieQGARdcZban/eRg2AgAmRfICACRtwi4AwCAkLwCw4DpJm/MCADAMyQsALLpuc14AAIaieQEA0ss917aRqrqoqm6pqtuq6q1HeP8JVfU7s/dvqKqzNrqm5gUAGERV7UnyniQ/lOScJD9ZVeesOe31Sb7a3X8zyTuS/MuNrqt5AQBW5rzMsx3dC5Pc1t1f6O5DSa5Icumacy5N8t7Z/pVJLqiqOtpFB52w+/V89e5r+8q/GuDS357krwe47lCemuT+sYvYIt/x8HzHw/MdD2tq32/iO17tOw7vfD1fvebavvLUOa9zYlXdtOp4X3fvm+2fnuT2Ve/dkeRFa37+0XO6+5Gquj/J05Lcvd4HDtq8dPdpQ1y3qu7q7nOHuPYQqmpfd182dh1b4Tsenu94eL7jYU3t+018x+vp7ouG/oydNNVho/vGLmCLrhq7gDn4jofnOx6e73hYU/t+E9/xbrszyZmrjs+YvXbEc6rq+KykY/cc7aJTbV6mFPmlu6f2H0viO94NvuPh+Y6HNanvN/Edj+DGJGdX1bOqam+SVyfZv+ac/UleN9t/VZKPdfdRb2Oa6iJ1+zY+hW3yHQ/Pdzw83/GwfL/Dm/R3PJvD8oYk1yTZk+TXu/vmqro8yU3dvT/J/5nk31bVbUnuzUqDc1S1QXMDAHBMmeqwEQCwoCbVvFTVM6vq01V1YDap55hWVS+qquur6pNV9Y6x69mMqnrOrOZPVNVvbHSv/bGiqt5YVZ8cu47NqKqzqurLVXVdVf3+2PVsRlX9dFV9dFbz6WPXs5HZip7XzbYvVtWPjl3T0VTVE6vqQ7N6P1hVTxi7pqOpquOr6oqq+oOqevvY9RzNkX5vVNU7Zn/HvXPs+tZaW+/Ufu/tlkk1L1kZC7sgyR+PXcgm/VWSV3T3S5N8a1V9z9gFbcIt3f2S7n7Z7PiYvw1y9hf988euY4s+0t3ndfcPjl3IRmbNysu7+4JZzWvvFDjmdPeHZ7Wel5U1Mq4duaSNXJTkhlm9n5odH8v+bpI/6+7zk5xUVc8bu6CjeMzvjar63iRPmv0dt7eqXjBmcUew9vfc1H7v7YpJNS/dfaC7vzp2HZvV3V/q7gOzw4eTLI1Zz2Z098PDX3xOAAACmklEQVSrDg/msYsLHaten/+8OuNUnD/7P783jl3IJrwyyZ5Z8vKu2XLfk1BVz07y5e5+YOxaNvAXSU6e7Z+SDW4TPQY8O8lnZ/ufSfKSEWs5qiP83vi+JB+Z7V+b5MW7X9X61tY7td97u2VSzctUVdVzk5zW3Z8bu5bNqKpLqurPkzw9x/hfolV1QpLzuvtjY9eyBV9M8l1Jzk9y4ezfj2PZ05Ps7e4LkjyUb17a+1j2Y0k+MHYRm3BrkhdX1c1ZSTuvH7mejdyS5OWz/fOz0nBNxSlJvjbbvz/Tqp0ZzcvAqupbkrw7K+nAJHT3/u5+TlaWcf6RsevZwGuTvG/sIraiuw9294Pd/UiS30vynLFr2sD9ST4+2/9Ykr89Yi1bdXG+eU2JY9HrklzV3d+d5ENJXjNyPRu5KivDRR/NSkL75ZHr2Yr7kzxltv+UTH8RuIWkeRnQbHLVbyV5c3d/aex6NmPNRMGvJfnGWLVs0t9K8rNV9eEk311VvzB2QRupqievOvz+rAwZHMuuT3I4HXp+kv80Yi2bVlXPSHKou4/p9HCmsjK3IVl5nstTR6xlQ9291N2/MEvjlrKyhsdU/FFW5pAkyYUxl2SSJtW8VNUJVXVtkucluaaq1j7c6Vjz40lekOTts7sIjqmx1XVcVFUfr6qPZ2W44Ji+G6a739Ldr5w9l+Pm7n7X2DVtwsuq6k+q6vokd3b3DWMXdDTd/Zkk36iq67Ly7/OV41a0aZcm+eDYRWzS+5L8xOw7/vtJfnvcco6uqk6f/Z32sSTXH8uTuNf+3khyQpIDVfWJJEvd/alRC1zjSL/nJvZ7b1dYpA4AmJRJJS8AAJoXAGBSNC8AwKRoXgCASdG8AACTonkBACZF8wIATMr/D9vYLTM5Bj3pAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 1440x1440 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"V_dx = np.vstack((np.diff(V, axis=0), np.zeros(nvmdp.width)))\n",
"V_dy = np.hstack((np.diff(V, axis=1), np.zeros((nvmdp.height,1))))\n",
"\n",
"plt.figure(figsize=(20,20))\n",
"nvmdp.visualize_grid(np.sqrt(V_dx**2+V_dy**2), trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value Gradient\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0.56, 0.33, 0.1 , 0.03, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0.33, 0.16, 0.06, 0.02, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0.1 , 0.06, 0.03, 0.01, 0.01, 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0.03, 0.02, 0.01, 0.01, 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0.01, 0.01, 0.01, 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ]])"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.sqrt(V_dx**2+V_dy**2).round(2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Low Temperature"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAIVCAYAAAAZNR+PAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X+QZWd5H/jvM6MZzVggMBG2sfidFS5A/FhHRoBNISIliMRCjtf2ghfjbFFWrROoLby4TLIO67D5J6QSkrWpJLMVJ8YYE0zKYWQUxgiQA5H5IWxBLLxEMsYgWYYBNCKDNOrp7mf/6CvRGjTTc2/3mTNH9/NRndI9994+95mj6e5H3/O+76nuDgDAVOwauwAAgHloXgCASdG8AACTonkBACZF8wIATIrmBQCYFM0LcEpV9eSq6qo6Z+xaABLNCyyFqnp/Vb35IZ6/uqr+QmMCTInmBZbDryV5VVXVCc//VJLf6O7VEWoCWIjmBZbDf0zyl5K86P4nquo7k/xwkrdX1d+sqj+sqm9U1Zeq6pdOdqCq+kJVXbFp/5eq6h2b9p9fVTdW1ZGq+nRVXTbEHwhYXpoXWALdfW+Sdyd59aanfyLJ/9fdn07yzdlrj07yN5P8bFX9yLyfU1UXJnlfkn+U5DFJ3pDkP1TVY7f3JwD4Fs0LLI9fS/JjVbVvtv/q2XPp7hu6+79293p3fybJbyZ58QKf8aok13X3dbNjfSDJTUn+xg7UD5BE8wJLo7s/muSrSX6kqv5ykucleWeSVNWlVfXhqjpcVXcn+d+SXLDAxzwpyY/PLhkdqaojSX4oyeN25k8BkJhhAMvl7dlIXL4vyaHu/vLs+Xcm+ZUkL+vuY1X1z3Py5uWbSb5j0/73bHr8pSS/3t0/s7NlA3yL5AWWy9uTXJHkZzK7ZDTzyCRfnzUuz0vyk6c4xs1JXlFVe6rqkiQ/tum1dyS5qqpeWlW7q2pfVV1WVY/f4T8HsMQ0L7BEuvsLSW5Mcl6Sg5te+jtJ3lxV/z3Jm7IxuPdk/kGSv5zkriT/MLNLT7PjfynJ1Un+fpLD2Uhifj5+1gA7qLp77BoAAE6b/xsCACZF8wIADKaqfrWqvlJVf3SS16uq/p+quq2qPlNV37/VMTUvAMCQ/l2SK0/x+suSXDTbrknyL7c6oOYFABhMd//nJF8/xVuuTvL23vCxJI+uqlOuDTXoOi9769zel/OG/Igd9bRn3zN2CXO7Y3X/2CXM5ch906o3SdZXdo9dwtx2HR+7gvnUBG8LuWtt7ArmU6vTm5xR69OqeWr13rtyJCur91SSvPQl5/XXvr7YX+pPfea+W5Ic2/TUge4+MMchLszGzMT73T577s6TfcGgzcu+nJdL6/IhP2JHHTp089glzO0Xv/KssUuYy8EvXDx2CXM7+sXzxy5hbvvvnFbDtf/wtH7oJ8m+I+tjlzCXvUem1yHuOTqtLnzX0ZWxS5jLx277Nw88/trX1/KJQ09c6Di7H3frse6+ZKfqOh0uGwEAY7ojyRM27T9+9txJaV4AYMl1kvUF/9kBB5O8ejbr6PlJ7u7uk14yStzbCAAYUFX9ZpLLklxQVbcn+b+S7EmS7v5XSa7Lxp3nb0tyT5L/datjal4AYOl11nqYcVzd/cotXu8kf3eeY7psBABMiuQFAJbcxpiX6cz6k7wAAJMieQEAdmrm0BmheQGAJdfprLXLRgAAg5C8AAAG7AIADEXyAgBLrpOsSV4AAIYheQEAjHkBABjK5JqX1fP25tP/8GVZPW/v2KUAwMNCJ1nrXmgbw+Sal8MvfHIOv+ipOfyCJ49dCgAwgi2bl6r63qr6g6o6VlXnnLh/Jorc7M+vfPrGv1/29DP90QDwsLW+4DaG02k+vp7k8iS/fZL9QX3qn7w8d/2VJzywXytrSZIjz3xcrv/Q333g+e/81JfyV37+4JkoCQAY0ZbJS3cf6+67TrY/tKf8xqey69jxb9Wzd/eD/p0ku44dz1PecdOZKgkAHlY6nbUFtzHs+JiXqrqmqm6tqsPH8s1tH+8xN9+R5/799z2ogdls17Hjee7f+5085tN/vu3PAgDOfjvevHT3ge6+qLsfuy/n7cgxH3PzHXnWmw9l132rD3p+132redabD2lcAGA7OllbcBvDZGYbrT7i3NTaerK2vpHCrK2n1taz+ohzxy4NADiDTme20Z6quj7Jc5IcqqpLT9wfushkY3bR2r49eeTnv5bn/IPr8sjPfy1r+/aYdQQA29R5mM026u7jSa444ekT9we3+5sruehf/Zc88T98OtXJY372t/LFH31O7nr24850KQDAiCZzb6Pnvuk/PWi/1jtPes/NedJ7bh6pIgB4uKispcYu4rRNZswLAEAyoeQFABhGJ1mfzk2lNS8AQFw2AgAYiuQFAJZcR/ICADAYyQsAkPWWvAAADELyAgBLzpgXAIABSV4AYMl1KmsTyjOmUykAQCQvAEDMNgIAGMygycvTnn1PDh26eciP2FG/+JVnjV3C3A5+4eKxS5jL0S+eP3YJc9t/5+6xS5jb/sMTusNakn1H1scuYW57j6yOXcJc9hw9PnYJc9t1z8rYJcyljt03dgnzWf/W953ZRgAAAzLmBQCWXmWtp5NnTKdSAIBIXgBg6XWS9QnlGdOpFAAgkhcAIGYbAQAMRvICAEuue1qzjTQvAEDWXTYCABiG5AUAltzG7QGmk2dMp1IAgEheAAC3BwAAGI7kBQCWnNsDAAAMSPICAGStrfMCADAIyQsALLlOWecFAGAoWzYvVfW9VfUHVXWsqs6ZPffWqvpIVf2L4UsEAIa23rsW2sZwOp/69SSXJ/lYklTV9yd5RHe/KMneqvqBAesDAHiQLce8dPexJMeqHhiF/PwkH5g9vj7JC5J8cpDqAIDBLcO9jR6d5Buzx3fP9h+kqq6qqgNHvrG+ndoAAL7NIs3L3UnOnz0+P8mRE9/Q3dd29zWPPn86XRwALKtOZa0X28awSHfx+9kYA5MkV2Q2FgYA4Ew4ndlGe6rq+iTPSXIoyZ5sjIH5SJK17v7EwDUCAANbz66FtjGczoDd49lIWDb7+DDlAACcmhV2AWDJdSdrI63ZsgjNCwAsvcp63JgRAGAQkhcAWHKdaV02mk6lAACRvAAAefjfHgAAYDSSFwBYcp3K+khL/S9C8gIATIrkBQAw5gUAYCiSFwBYcp1k3TovAADDkLwAwNKrrLm3EQDAMCQvALDkpjbmZdDm5Y7V/fnFrzxryI/YUQe/cPHYJczt6BfPH7uEuey/c/fYJcxt/+Eeu4S57TuyPnYJc9l7ZHXsEua25+jxsUuYy657VsYuYW5177RqrmPTqjfr0/vZdj/JCwBgzAsAwFAkLwCw5LprUmNeplMpAEAkLwBAkrUJJS+aFwBYcp1k3YBdAICkqq6sqs9V1W1V9caHeP2JVfXhqvrDqvpMVf2NrY4peQGApVeDXDaqqt1J3pbkryW5Pcknq+pgd39209t+Mcm7u/tfVtUzklyX5MmnOq7kBQAYyvOS3Nbdn+/ulSTvSnL1Ce/pJPevuPqoJH++1UElLwCw5DZuD7DwmJcLquqmTfsHuvvA7PGFSb606bXbk1x6wtf/UpLfrarXJTkvyRVbfaDmBQDYjq929yXb+PpXJvl33f1Pq+oFSX69qi7u7pPe50TzAgBkbZiRJHckecKm/cfPntvsNUmuTJLu/v2q2pfkgiRfOdlBjXkBAIbyySQXVdVTqmpvklckOXjCe76Y5PIkqaqnJ9mX5PCpDip5AYAl16ntjHk5+XG7V6vqtUkOJdmd5Fe7+5aqenOSm7r7YJL/I8n/W1Wvz8bwm7/d3ae85bXmBQAYTHdfl43pz5ufe9Omx59N8oPzHFPzAgBkfUIjSaZTKQBAJC8AsPS6k7UBxrwMRfICAEyK5AUAGGS20VAWSl6q6pyqetfsLpBv2emiAABOZtHLRn8ryae7+yVJ9lfVc3awJgDgDNpY52XXQtsYFv3Upyb5zOzxzUleuDPlAACc2qLNy+eSvHj2+CVJHn3/C1V1TVXdWlWHv3HnPdutDwA4A9ZSC21jWLR5uTYbl4s+mOS+JF++/4XuPtDdF3X3Y89/3HfsRI0AAA9YaLZRd68leV2SVNWBbNyzAACYoM60Zhst1LxU1YVJfiPJepK3d/eJt7cGABjEosnLHUku29lSAIBx1GgzhxZhkToAIOsjDb5dxHTaLACASF4AYOm5MSMAwIAkLwDApAbsTqdSAIBIXgBg6W3cmNGYFwCAQUheAADrvAAADEXyAgBLbmo3ZpS8AACTInkBAKzzAgAwFMkLACy7ts4LAMBgBk1ejty3Pwe/cPGQH7Gjjn7x/LFLmNv+O3ePXcJc9h/usUuY274j62OXMLe9R1bHLmEue44eH7uEue26Z2XsEuZS906r3iSpY9OquY9P7O9xf+vnccc6LwAAgzHmBQAw5gUAYCiSFwBYclNbYVfzAgBMqnlx2QgAmBTJCwAsuY5F6gAABiN5AQAsUgcAMBTJCwAsuzbbCABgMJIXAFhyU1ukTvICAEyK5AUAkLwAAAxF8gIAS84KuwAAA5K8AADpCSUvCzUvVfUdSX4ryXlJ7k7yE919304WBgDwUBa9bHRlko9392VJPjHbBwAmaj210DaGRZuXP8lG6pIkj07ytZ0pBwDg1BYd83JrkhdU1S1JvpLkFza/WFVXJblq7/d85zbLAwCG1ktyb6OfTnJtdz8zyfuSvGrzi919bXdfs+u8fdutDwDgQRZNXirJ12ePv5rkUTtTDgAwhof9bKMk70zy76vqp5IcT/I/71xJAMCZNa1F6hZqXrr7SJKX7nAtAABbskgdADCpy0ZuDwAATIrkBQCWXGc5pkoDAIxC8gIAy643FqqbCskLADApkhcAYLSbLC5C8gIATIrkBQCWXMc6LwAAg5G8AMDSm9a9jSQvAMCkSF4AAOu8AAAMRfICAJhtBAAwlEGTl/WV3Tn6xfOH/Igdtf/O3WOXMLf9hyd0kTLJviPrY5cwt71HVscuYW57jh4fu4S57LpnZewS5lb3TqvmOjatepOkVyZW88q0vu82D3LplrwAAAzGmBcAYFLrvGheAABTpQEAhiJ5AQAM2AUAGIrkBQCWXKckLwAAQ5G8AACZ0GQjyQsAMC2SFwBYdm4PAAAwHMkLADCpQS+SFwBgUiQvAIAxLwAASVJVV1bV56rqtqp640ne8xNV9dmquqWq3rnVMSUvAMAgd5Wuqt1J3pbkryW5Pcknq+pgd39203suSvL3kvxgd99VVd+11XElLwDAUJ6X5Lbu/nx3ryR5V5KrT3jPzyR5W3fflSTd/ZWtDqp5AYAl19kY87LIluSCqrpp03bNpkNfmORLm/Zvnz232dOSPK2q/ktVfayqrtyqXpeNAIDt+Gp3X7KNrz8nyUVJLkvy+CT/uaqe1d1HTvYFCyUvs8E3N8y2O6vqRxYqFwAYXyfpWmw7tTuSPGHT/uNnz212e5KD3X28u/80yX/LRjNzUgs1L939/u6+rLsvS/LFJNcvchwA4GHtk0kuqqqnVNXeJK9IcvCE9/zHbKQuqaoLsnEZ6fOnOui2xrxU1VOTfLm7j2567pqqurWqDq/eddd2Dg8AnCHdi22nPmavJnltkkNJ/jjJu7v7lqp6c1W9fPa2Q0m+VlWfTfLhJD/f3V871XG3O+blR5P89gmFHkhyIEnOfeITJrTYMAAssYF+Y3f3dUmuO+G5N2163El+bradlu3ONroq3x7/AAAMZuHkpaq+J8nKVtEOAHC2e2Da8yRsJ3m5Osl7d6oQAIDTsXDy0t3/eicLAQBGNKFRqlbYBQAmxQq7ALDsOksz5gUA4IyTvAAAxrwAAAxF8gIAJDHmBQBgEJIXAMCYFwCAoUheAADJCwDAUCQvALDsOokVdgEAhiF5AQDSxrwAAAxD8gIATGq20aDNy67jyf47dw/5ETtq/+EJ/Zeb2XdkfewS5rL3yOrYJcxtz9HjY5cwt133rIxdwlzq3mnVmyR1bFo198q06k2SrEzre69XJ/bz7cTrRAbsAgAMw2UjACA1oYsPkhcAYFIkLwCw7DqTGrAreQEAJkXyAgBLr8w2AgAYiuQFADDmBQBgKJIXAEDyAgAwFMkLACB5AQAYiuQFAJZdxzovAABDkbwAAO4qDQAwFMkLAGC2EQDAUBZuXqrq1VX1waq6oaou3MmiAABOZqHLRrNm5cXdffkO1wMAjGAZBuy+NMnuWfLyy1W1eyeLAgA4mUWbl+9OsneWvNyT5OrNL1bVVVV1YO2+Y9utDwA4E7oW20awaPNyd5Lfmz3+UJKnb36xu6/t7mt2n7tvO7UBAHybRZuXG5M8e/b4uUn+dGfKAQDOuN7GNoKFBux2981VdW9V3ZDkq0neuqNVAQCcxMKL1HX3G3ayEABgREsw2wgAYBRuDwAALMU6LwAAo5C8AADGvAAADEXyAgBIXgAAhiJ5AYAlV222EQDAYCQvAMBod4hehOQFAJgUyQsAYLYRAMBQJC8AwKRmG2leAACXjQAAhjJo8lKryf7D02nl9h1ZH7uEue09sjp2CXPZc/T42CXMbdc9K2OXMLe6d1o117Fp1ZskvTKxmlem973Xq9P6+ZbjEzvHven3s0XqAACGY8wLAGDMCwDAUCQvAIDkBQBgKJIXAMBsIwCAoWheAIBJ0bwAAJNizAsAYLYRAMBQJC8AsOzc2wgAYDiSFwDAmBcAgKFIXgAAyQsAwFAkLwCw5CpmGwEADEbyAgBMaszLQs1LVT05yceT/HGSle7+6ztYEwBwJk1skbrtJC8f6O5X7VglAACnYTtjXl5SVR+pqtdvfrKqrqmqW6vq8PGjd22zPADgjOgFtxEs2rzcmeRpSV6S5Iqqevb9L3T3ge6+qLsfu+cR37kTNQIAPGCh5qW77+vub3b3apLfSXLxzpYFAJxRAyUvVXVlVX2uqm6rqjee4n3/U1V1VV2y1TEXal6q6pGbdn8wyZ8schwA4OGrqnYneVuSlyV5RpJXVtUzHuJ9j0zyv2djMtCWFr1s9KKq+lRV3Zjkju4+rQ8DAM5O1YttW3hektu6+/PdvZLkXUmufoj3/d9J/nGSY6dT60Kzjbr7uiTXLfK1AMDDygVVddOm/QPdfWD2+MIkX9r02u1JLt38xVX1/Ume0N3vq6qfP50PtEgdALCdmUNf7e4tx6k8lKraleSfJfnb83yd2wMAAEO5I8kTNu0/fvbc/R6ZjUk/N1TVF5I8P8nBrQbtSl4AYNkNt2bLJ5NcVFVPyUbT8ookP/nAx3bfneSC+/er6oYkb+jum3IKkhcAYBCzJVVem+RQNm4p9O7uvqWq3lxVL1/0uJIXAGCwexs91CSf7n7TSd572ekcU/ICAEyK5AUAGO0+RYuQvAAAkyJ5AQAGG/MyBMkLADApkhcAwJgXAIChSF4AYNkNt8LuIDQvALDkarZNhctGAMCkSF4AAJeN7rdrLdl3ZH3Ij9hRe4+sjl3C3PYcPT52CXPZdc/K2CXMre6dYM3HplVzr0yr3iTJyrS+93p1ej/fcnxi53htOr/vkkn1Kt9G8gIAWKQOAGAokhcAYFLXkSQvAMCkSF4AAMkLAMBQJC8AsOzabCMAgMFIXgAAY14AAIYieQEAjHkBABiK5AUAMOYFAGAokhcAwJgXAIChSF4AYNl1JjXmRfMCAEyqeXHZCACYlG01L1X1+qr66E4VAwCceZWNAbuLbGNYuHmpqnOTPHcHawEA2NJ2kpfXJPm1nSoEABhRL7iNYKHmpar2JLmsuz90ktevqqoDqyv3bqs4AIATLZq8/FSSd57sxe6+truvOWfv/gUPDwCcSdW90DaGRZuX70vys1X1/iTPrKrX7WBNAAAntdA6L939C/c/rqqPdvcv71xJAMAZNbFF6ra9zkt3/9BOFAIAcDqssAsAuDEjAMBQJC8AwHKNeQEAOJMkLwCAMS8AAEORvAAAxrwAAAxF8gIAy66NeQEAGIzkBQCY1JgXzQsALLmKy0YAAIORvAAASU8nepG8AACTInkBACY15mXQ5qVWO3uPrA75ETtqz9HjY5cwt133rIxdwlzq3mnVmyR1bHo198rEal6Z3vder07nZ1uS5PgEz/Ha+tglzKcnVu+UphedQPICAMuuM6lexpgXAGBSJC8AQGpCV70kLwDApEheAABjXgAAhiJ5AQAmtc6L5AUAmBTJCwAsu457GwEADEXyAgAY8wIAMBTJCwBgnRcAgKFIXgBgyVWmNeZF8wIAy67bVGkAgKFIXgCASV02krwAAJMieQEAHv5Tpavq4qq6sao+UlX/tqpqpwsDAHgoi142+lx3v7C7XzTbv2SnCgIAzrzqxbYxLNS8dPfxTbv3JfnS/TtVdU1V3VpVh+87dmS79QEAPMjCA3ar6uVV9UdJvjvJ1+5/vrsPdPdF3f3Yc/c9eidqBACG1EnWe7FtBAs3L919sLsvTnJ7kh/euZIAAE5uodlGVXVud9832/1Gknt3riQA4Iyb0GyjRadKX1lVPzd7fGuS392hegAATmmh5qW735vkvTtcCwAwEivsAgAMxAq7AIC7SgMADEXyAgAY8wIAkCRVdWVVfa6qbquqNz7E6z9XVZ+tqs9U1Qer6klbHVPzAgDLrrexnUJV7U7ytiQvS/KMJK+sqmec8LY/THJJdz87yXuSvGWrcjUvAMBQnpfktu7+fHevJHlXkqs3v6G7P9zd98x2P5bk8Vsd1JgXAFhylaSGmW10YTbdvDkbtxS69BTvf02S/7TVQTUvAECyvvBXXlBVN23aP9DdB+Y9SFW9KsklSV681Xs1LwDAdny1uy85yWt3JHnCpv3Hz557kKq6Isn/meTFm+6deFKaFwBgqMtGn0xyUVU9JRtNyyuS/OSDPrfqf0zyr5Nc2d1fOZ2DGrALAAyiu1eTvDbJoSR/nOTd3X1LVb25ql4+e9s/SfKIJL9VVTdX1cGtjit5AYBldxrTnhc+dPd1Sa474bk3bXp8xbzHlLwAAJMieQGApdduzAgAMJRBk5da7+w5enzIj9hRu+5ZGbuEudW906q5jk2r3iTplenVnJXpfN8lSa+ujl3C/I5P7ByvLb6Ix2h6WjX3+nSSiyTfNsbFjRkBAAZizAsAYMwLAMBQJC8AsOw6qQkNMZK8AACTInkBAIx5AQAYiuQFABjs3kZDkLwAAJMieQEAUsa8AAAMQ/ICAExqtpHmBQCWXSexSB0AwDAkLwCw5CptwC4AwFAkLwDApAbsSl4AgEmRvAAAD//kpaouraobq+qjVfXWnS4KAOBkFr1s9GdJ/mp3/1CS76qqZ+1gTQDAmXT/Oi+LbCNY6LJRd//Fpt3jSdZ2phwAgFPb1piXqnp2ksd292dPeP6qJFftP/cx2zk8AHCGLMU6L1X1mCS/kuQ1J77W3dd29zXnnLNvO7UBAHybhZKXqjonyTuSvOGES0gAwBQtQfLy40l+IMlbquqGqnrBDtYEAHBSiw7Y/c0kv7nDtQAAo+ilSF4AAEZhhV0AWHYdyQsAwFAkLwDAaKvlLkLyAgBMiuQFAFiOFXYBAMYgeQEAJjXbSPMCAMuuk6xPp3lx2QgAmBTJCwAsPbcHAAAYjOQFAJC8AAAMRfICAEheAACGMmjyUuudXUdXhvyIHVXH7hu7hLnVsemc3yTp48fHLmF+K9OruVdXxy5hPhP8e9FrE7qLXZL0xOpN0hNadyTJJM/xA6zzAgAwHGNeAGDp9aSSI8kLADApkhcAwGwjAIChSF4AYNmZbQQAMBzJCwBgzAsAwFAkLwCA5AUAYCiSFwBYej2p5EXzAgDLrpOsuz0AAMAgJC8AwKQuG0leAIBJkbwAAJIXAIChSF4AYOn1w//GjFX1vVX1B1V1rKo0QADAGbNo4/H1JJcn+e0drAUAGEMn3dNZ52Wh5qW7jyU5VlU7XA4AwKnt+IDdqrqmqm6tqsP3rty904cHAIaw3ottI9jx5qW7D3T3Rd392P17H7XThwcAlpzBtgDAw3+dl6raU1XXJ3lOkkNVdenOlgUA8NAWHbB7PMkVO1wLADCGbneVBgAYijEvAMDDf8wLAMBYJC8AQNqYFwCAYUheAGDp9aTGvGheAGDZdUZb6n8RLhsBAJMieQEAkjZgFwBgEJIXAFhynaSNeQEAGIbkBQCWXbcxLwAAQ9G8AADp9V5o20pVXVlVn6uq26rqjQ/x+rlV9e9nr3+8qp681TE1LwDAIKpqd5K3JXlZkmckeWVVPeOEt70myV3d/T8keWuSf7zVcTUvAMDGmJdFtlN7XpLbuvvz3b2S5F1Jrj7hPVcn+bXZ4/ckubyq6lQHHXTA7jfuvfOrv/tf/9GfDXDoJyb54gDHHcqjktw9dhFzco6H5xwPzzke1tTOb+Icb/ak+x/899x16Pp+zwULHmdfVd20af9Adx+YPb4wyZc2vXZ7kktP+PoH3tPdq1V1d5K/lOSrJ/vAQZuX7n7sEMetqsPdfckQxx5CVR3o7mvGrmMezvHwnOPhOcfDmtr5TZzjk+nuK4f+jJ001ctGR8YuYE7Xjl3AApzj4TnHw3OOhzW185s4x2faHUmesGn/8bPnHvI9VXVONtKxr53qoFNtXqYU+aW7p/bNkjjHZ4JzPDzneFiTOr+JczyCTya5qKqeUlV7k7wiycET3nMwyU/PHv9Ykg919ymnMU11kboDW7+FbXKOh+ccD885HpbzO7xJn+PZGJbXJjmUZHeSX+3uW6rqzUlu6u6DSf5Nkl+vqtuSfD0bDc4p1RbNDQDAWWWql40AgCU1qealqr63qv6gqo7NBvWc1arq0qq6sao+WlVvHbue01FVF89q/khV/dut5tqfLarq9VX10bHrOB1V9eSq+nJV3VBVvzt2Paejql5dVR+c1Xzh2PVsZbai5w2z7c6q+pGxazqVqvqOqnrfrN73VtW5Y9d0KlV1TlW9q6o+XFVvGbueU3mo3xtV9dbZz7h/MXZ9Jzqx3qn93jtTJtW8ZONa2OVJPjZ2Iafpz5L81e7+oSTfVVXPGrug0/C57n5hd79otn/WT4Oc/aB/7th1zOkD3X1Zd//1sQvZyqxchHgWAAADDklEQVRZeXF3Xz6r+cSZAmed7n7/rNbLsrFGxvUjl7SVK5N8fFbvJ2b7Z7O/leTT3f2SJPur6jljF3QKD/q9UVXfn+QRs59xe6vqB8Ys7iGc+Htuar/3zohJNS/dfay77xq7jtPV3X/R3cdmu8eTrI1Zz+no7uObdu/LgxcXOlu9Jt9anXEqXjL7P7/Xj13IaXhpkt2z5OWXZ8t9T0JVPTXJl7v76Ni1bOFPkpw3e/zobDFN9Czw1CSfmT2+OckLR6zllB7i98bzk3xg9vj6JC8481Wd3In1Tu333pkyqeZlqqrq2Uke292fHbuW01FVL6+qP0ry3TnLf4hW1Z4kl3X3h8auZQ53JnlakpckuWL29+Ns9t1J9nb35Unuybcv7X02+9Ekvz12Eafh1iQvqKpbspF23jhyPVv5XJIXzx6/JBsN11Q8Osk3Zo/vzrRqZ0bzMrCqekySX8lGOjAJ3X2wuy/OxjLOPzx2PVv4qSTvHLuIeXT3fd39ze5eTfI7SS4eu6Yt3J3k92aPP5Tk6SPWMq+r8u1rSpyNfjrJtd39zCTvS/KqkevZyrXZuFz0wWwktF8euZ553J3k/Nnj8zP9ReCWkuZlQLPBVe9I8obu/oux6zkdJwwU/EaSe8eq5TR9X5Kfrar3J3lmVb1u7IK2UlWP3LT7g9m4ZHA2uzHJ/enQc5P86Yi1nLaq+p4kK919VqeHM5WNsQ3Jxv1cHjViLVvq7rXuft0sjVvLxhoeU/H72RhDkiRXxFiSSZpU81JVe6rq+iTPSXKoqk68udPZ5seT/ECSt8xmEZxV11ZP4sqq+r2q+r1sXC44q2fDdPcvdPdLZ/fluKW7f3nsmk7Di6rqU1V1Y5I7uvvjYxd0Kt19c5J7q+qGbPx9fs+4FZ22q5O8d+wiTtM7k/zE7Bz/L0l+Y9xyTq2qLpz9TPtQkhvP5kHcJ/7eSLInybGq+kiSte7+xKgFnuChfs9N7PfeGWGROgBgUiaVvAAAaF4AgEnRvAAAk6J5AQAmRfMCAEyK5gUAmBTNCwAwKf8/shqwlW09U5MAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 1440x1440 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"boltz_temp = 0.01\n",
"v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, \n",
" lambda Q, t=boltz_temp: np.exp((1/t)*Q) / (np.exp((1/t)*Q).sum()), 30)\n",
"V = get_value_grid(v, nvmdp, s_to_idx)\n",
"\n",
"plt.figure(figsize=(20,20))\n",
"nvmdp.visualize_grid(V, trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value\")"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0. , 1. , 0.8 , 0.64, 0.51, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13],\n",
" [1. , 0.8 , 0.64, 0.51, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13, 0.1 ],\n",
" [0.8 , 0.64, 0.51, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13, 0.11, 0.08],\n",
" [0.64, 0.51, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13, 0.11, 0.08, 0.06],\n",
" [0.51, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13, 0.11, 0.08, 0.07, 0.05],\n",
" [0.41, 0.33, 0.26, 0.21, 0.17, 0.13, 0.11, 0.08, 0.07, 0.05, 0.04],\n",
" [0.33, 0.26, 0.21, 0.17, 0.13, 0.11, 0.08, 0.07, 0.05, 0.04, 0.03],\n",
" [0.26, 0.21, 0.17, 0.13, 0.11, 0.08, 0.07, 0.05, 0.04, 0.03, 0.02],\n",
" [0.21, 0.17, 0.13, 0.11, 0.08, 0.07, 0.05, 0.04, 0.03, 0.02, 0.01],\n",
" [0.17, 0.13, 0.11, 0.08, 0.07, 0.05, 0.04, 0.03, 0.02, 0.01, 0.01],\n",
" [0.13, 0.1 , 0.08, 0.06, 0.05, 0.04, 0.03, 0.02, 0.01, 0.01, 0.01]],\n",
" dtype=float32)"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V.round(2)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAIVCAYAAADPkMXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3X20ZXdZJ/jvkzcSwkuURJQkCHTHXkYEmhXeoUlMRoJNjDrogALaCztrsGH1gqYH2p5GB7tnjfSaYWzEl+oeRFSgbXqQRNIEAkTAQCAq0IbumAhqkuElLySEJJWquveZP+6peFNU1T333Lvvrp3z+WTtVXufs8/vPNnrpu6TZz+/367uDgDAlBw1dgAAAJslgQEAJkcCAwBMjgQGAJgcCQwAMDkSGABgciQwMEFV9Ziq6qo6ZuxYFlVVf1VV5832f76q/sPYMQHTIYGBEVTVB6rqjQd5/cKq+srYiUlVvaiqrqqqu6rqa7P9n6uqGuL7uvt/7+6f3eo4D4TEDpiPBAbG8dtJXnKQhOClSX6vu/eNEFOSpKr+WZJfSfJvk3xnkkcm+Z+TPCvJcYf4zNE7FiBAJDAwlj9I8ogkz9n/QlV9W5IXJHnH7PgfVtWfVdU3quqGqvrFQw22/nbM7PgXq+p31x0/vaqurKrbq+pzVXX2IcZ5eJI3Jvm57n5Pd9/Za/6su3+qu++dnff2qvr1qrq0qu5Kcs5G8VbVS6vqr6vq1qr6lwe8N3e8VXVFVf1SVf1xVd1ZVR+sqpNnb39s9uftVfXNqnrGoa4ZMG0SGBhBd9+T5PeTvGzdyz+R5L939+dmx3fN3j8pyT9M8oqq+pHNfldVnZrk/Un+dZJvT/LaJP+5qk45yOnPSPKgJO+bY+ifTPJvkjw0yScOF29VnZnk17NWYXpU1pK307YQ708m+UdJviNrVaHXzl7/B7M/T+ruh3T3J+f49wAmSAID4/ntJC+squNnxy+bvZYk6e4ruvu/dvdqd38+ybuSPHeB73lJkku7+9LZWB9KcnWSHzrIuScnuWX9Lax1lZB7quofrDv3fd39x7Mxd28Q7wuT/GF3f2xWxflXSVa3EO9vdfdfrEsEn7TpqwJMmgQGRtLdn0hyS5Ifqaq/k+SpSd65//2qelpVfbSqbq6qO7LWh3LywUc7rO9O8uOzJOT2qro9ybOTfNdBzr01ycnrm2C7+5ndfdLsvfV/Z9yw/oMbxPuo9ed3912z8RaN9yvr9u9O8pBD/csDD0wSGBjXO7JWeXlJksu6+6vr3ntnkouTnN7dD0/yG0kONQvoriQPXnf8nev2b0jyO9190rrtxO7+Pw4yzieT3JvkwjliP/BR9oeL98tJTt9/YlU9OGu3kQ5mM/FuFBPwACWBgXG9I8l5Sf5x1t0+mnloktu6e3dVPTVrfR+H8tkkL6qqY6vqrKzdstnvd5NcUFXPq6qjq+r4qjq7qr6lB6W7b0/yvyX5tap6YVU9tKqOqqonJTlxg3+Xw8X7niQvqKpnV9VxWWsUPtTfP3PHexA3Z+3W1OPmOBeYMAkMjKi7/yrJlVlLDi4+4O2fS/LGqrozyRuy1utxKP8qyd9J8vWsJSD33Yrq7huyVlH5+az9gr8hyT/PIf777+43JXlNkv8lyVdn228med0s1kM5ZLzdfU2SfzKL68uzOG88xPdvKt4DPnt31hqL/3h2++npG30GmKbqVnEFAKZFBQYAmBwJDAAwORIYAGByJDAAwOQM+sTWk7/96H7M6ccO+RXb6tovLbJG2LhqZVpN2LV6qMVXj2BTbHRfnVjMU7zGU1tyZorXeGIhT21SzO6+K3t6dyXJ8845sW+9bWWhcf7k8/de1t3nb2twcxg0gXnM6cfm05edvvGJR4hzX/LysUPYtGPv3DN2CJty1N3TijdJcu/0Yq7d04q59+4dO4TN2zfaA8MXs3di8SbplcV+oY6l90zr5/hTez9w3/6tt63k05c9eqFxjv6u60b5v3+3kACAwVTV26rqa1X15xuc95Sq2ldVLzzceftJYABgyXWS1QX/mcPbkxz2FlNVHZ3kl5N8cN6YJTAAwGC6+2NJbtvgtFcl+c9JvjbvuIP2wAAAU9BZ6YUnWZxcVVevO97V3bvm/XBVnZrkR5Ock+Qp835OAgMAbMUt3X3WFj7/fyd5XXevVtWGJ+8ngQGAJbfWAzPaNPCzkrx7lrycnOSHqmpfd//B4T4kgQEARtPdj92/X1VvT/KHGyUviQQGAEjmnVG0aVX1riRnZ61X5sYkv5Dk2CTp7t9YdFwJDAAsuU5nZaCVhLv7xZs492fmPdc0agBgclRgAIAxm3gXogIDAEyOCgwALLlOsqICAwAwLBUYAEAPDADA0CaXwNyxckJ+7MZX5I6VE8YOBQAeEDrJSvdC21gml8Bc/M0n5n3ffHIu+eYTxg4FABjJhglMVT2qqv60qnZX1TEHHu9EkOv91h3Put+fAMDWrS64jWWeBOS2JOcmee8hjgf1g3/z6nz47jPvOz4ue5MkV979d3P0f//3971+7oO/kA8++s07ERIAMLINKzDdvbu7v36o46H9i0dcmgfXvfcd71l7/tN9fybJg+ve/Pwj3r9TIQHAA0qns7LgNpZt74Gpqouq6rqquvlvbtq75fHOOfHaXHzaW+6XxKz34Lo3l5z273L2iX+x5e8CAKZh2xOY7t7V3Wd09ymPPvXYjT8wh3NOvDbvOnVXjq8993v9+NqTd526S/ICAFvRycqC21gmMwvp9pUTckxWc1RWckLtyVFZyTFZze2mUwPA0plnFtKxVXV5kicmuayqnnbg8dBBJsnb7nh27u7j8oQH3ZT3nvarecKDbsrdfZzZSACwRZ0H4Cyk7t6b5LwDXj7weHAPP+qevOk73pN/+m2X56jq/MBj/nV+5bbz8vF7ztjpUACAkU3mWUjvPe3X7nd8dHVe84gP5TX50EgRAcADRWUlNXYQmzKZHhgAgP0mU4EBAIbRSVan9TBqCQwAELeQAACGpgIDAEuuowIDADA4FRgAIKutAgMAMCgVGABYcnpgAAB2gAoMACy5TmVlYjWNaUULABAVGAAgZiEBAAxu0ArMtV86Oee+5OVDfsW2OvbOPWOHsGlH3T2xmO+dWLxJavf0Yu69e8cOYXP27Rs7gs3bO62Ye2Vl7BA2ryf2dMFeHTuCTer77ZmFBAAwMD0wALD0Kis9rZrGtKIFAIgKDAAsvU6yOrGaxrSiBQCICgwAELOQAAAGpwIDAEuue3qzkCQwAEBW3UICABiWCgwALLm1RwlMq6YxrWgBAKICAwB4lAAAwPBUYABgyXmUAADADlCBAQCy0taBAQAYlAoMACy5TlkHBgBgaBsmMFX1qKr606raXVXHzF57c1V9vKp+ZfgQAYChrfZRC21jmeebb0tybpJPJUlVPTnJQ7r7OUmOq6qnDBgfAMC32LAHprt3J9lddV938tOTfGi2f3mSZyT5zCDRAQCDW5ZnIZ2U5Buz/Ttmx/dTVRdU1a59+3ZvJTYAgINaJIG5I8nDZvsPS3L7gSd09yXdfdExxxy/ldgAgB3Qqaz0YttYFklgPpm1npgkOS+z3hgAgJ0yzyykY6vq8iRPTHJZkmOz1hPz8SQr3f3pgWMEAAa2mqMW2sYyTxPv3qxVWta7aphwAIAHkqp6W5IXJPladz/+IO//VJLXJakkdyZ5RXd/bqNxp9VyDABsu+5kpY9aaJvD25Ocf5j3v5Tkud39/Ul+KcmueQb1KAEAWHqV1QzTkNvdH6uqxxzm/SvXHX4qyWnzjCuBAQC24uSqunrd8a7unquKchAvT/Jf5jlRAgMAS66TeW8HHcwt3X3WVmOoqnOylsA8e57zJTAAwKiq6glJ/kOS53f3rfN8RgIDAIz2KIGqenSS/zfJS7v7L+b9nAQGABhMVb0rydlZ65W5MckvZG1NuXT3byR5Q5JHJPm12XMX981zS0oCAwBLrlNZHeixAN394g3e/9kkP7vZca0DAwBMjgoMADBaD8yiphUtAEBUYABg6XWS1cXXgRnFtKIFAIgKDACQyspAz0IaigoMADA5KjAAsOSm2AMzaAJTK51j79wz5Fdsq6Punk6s97l3WjHX7mnFmyS9d+/YIWzevn1jR7A5eycWb5JeWRk7hM3pHjuCTZvcNWZHqcAAAHpgAACGpgIDAEuuuybXAzOtaAEAogIDACRZmVgFRgIDAEuuk6xq4gUAGJYKDAAsvZrcLaRpRQsAEBUYAFh6a48S0AMDADAoFRgAICsTq2lMK1oAgKjAAMDS65QeGACAoanAAABZnVhNY1rRAgBEBQYAll53sqIHBgBgWCowAMByzEKqqmOq6t1V9dGqetN2BwUAcDiL3kL60SSf6+5zkpxQVU/cxpgAgB20tg7MUQttY1n0mx+X5POz/c8meeb2hAMAsLFFE5hrkzx3tn9OkpP2v1FVF1XVdVV18+49t281PgBgB6ykFtrGsmgCc0nWbh19OMm9Sb66/43u3tXdZ3T3Kccfd9IhBwAAWNRCs5C6eyXJq5KkqnYluWw7gwIAdk5nerOQFkpgqurUJL+XZDXJO7r7pm2NCgDgMBatwNyU5OztDQUAGEeNOqNoERayAwCyOmJD7iKmlW4BAEQFBgCWnoc5AgDsABUYAGByTbzTihYAICowALD01h7mqAcGAGBQKjAAgHVgAACGpgIDAEtuig9zVIEBACZHBQYAsA4MAMDQVGAAYNm1dWAAAAY3aAWmVldz1N17hvyK7XXvhGKdqd3Tirn37h07hM3bt2/sCDZv77Ri7pWVsUPYvO6xI9iUSV7j1Yld44nFm77/rnVgAAAGpgcGANADAwAwNBUYAFhyU1yJVwIDAEwugXELCQCYHAkMACy5ztpCdotsG6mqt1XV16rqzw/xflXVv6uq66vq81X15HlilsAAAEN6e5LzD/P+85OcMdsuSvLr8wwqgQEAsppaaNtId38syW2HOeXCJO/oNZ9KclJVfddG40pgAIAxnZrkhnXHN85eOyyzkABg2fWWZiGdXFVXrzve1d27tiGqw5LAAABbcUt3n7WFz9+U5PR1x6fNXjsst5AAYMntX8huiFlIc7g4yctms5GenuSO7v7yRh9SgQEABlNV70pydtZuNd2Y5BeSHJsk3f0bSS5N8kNJrk9yd5J/NM+4EhgAYLCVeLv7xRu830n+yWbHdQsJAJgcFRgAWHL7V+KdEhUYAGByVGAAgPTEKjALJTBV9eAk/ynJiUnuSPIT3X3vdgYGAHAoi95COj/JVd19dpJP5/APaQIAjnBDPQtpKIsmMH+ZtepLkpyU5NbtCQcAYGOL9sBcl+QZVXVNkq8led36N6vqgiQXnHDct20xPABgaL21ZyGNYtEKzE8nuaS7vy/J+5O8ZP2b3X1Jd1907FEP2mp8AADfYtEKTCW5bbZ/S5KHb084AMAYlmIWUpJ3JvmPVfXSJHuT/E/bFxIAsLOmt5DdQglMd9+e5HnbHAsAwFwsZAcATO4WkkcJAACTowIDAEuuszzTqAEARqMCAwDLrtcWs5sSFRgAYHJUYACAUR/MuAgVGABgclRgAGDJdawDAwAwOBUYAFh603sWkgoMADA5KjAAgHVgAACGpgIDAJiFBAAwtGErMN3JvXsG/YrtVLunE+t+vXfv2CFszr59Y0eweXunF3OvrIwdwuZM7eZ7JniNV6d3jdk53SowAACD0wMDAExuHRgJDAAwuTu5biEBAJOjAgMAaOIFABiaCgwALLlOqcAAAAxNBQYAyMQmIanAAADTowIDAMvOowQAAIanAgMATK4JRgUGAJgcFRgAQA8MAMDQVGAAAE+jBgAYmgoMACy5jh4YAIDBLZTAVNX5VXXFbPtyVf3IdgcGAOyQTtK12DaShW4hdfcHknwgSarqqiSXb2dQAACHs6VbSFX1uCRf7e5vrnvtoqq6rqpuvmffN7YcIAAwvO7FtrFstQfmx5K8d/0L3b2ru8/o7lNOOOZhWxweANgRveA2kq0mMBckuXg7AgEAmNfC06ir6juT7OnuW7cxHgBgx9VSTaO+MMn7tisQAIB5LVyB6e7f3M5AAIAReZQAAMCwPEoAAJZde5QAAMDgVGAAAD0wAAD7zZ6feG1VXV9Vrz/I+4+uqo9W1Z9V1eer6ofmGVcCAwAkqQW3w4xYdXSStyZ5fpIzk7y4qs484LT/Ncnvd/ffT/KiJL82T7QSGABgKE9Ncn13f7G79yR5d9bWkVuvk+x/9tDDk/x/8wysBwYA2EoPzMlVdfW6413dvWu2f2qSG9a9d2OSpx3w+V9M8sGqelWSE5OcN8+XSmAAgK24pbvP2sLnX5zk7d39f1bVM5L8TlU9vrtXD/chCQwAMNQspJuSnL7u+LTZa+u9PMn5SdLdn6yq45OcnORrhxtYDwwAMJTPJDmjqh5bVcdlrUn34gPO+Zsk5yZJVX1vkuOT3LzRwCowALDsOskAK/F2976qemWSy5IcneRt3X1NVb0xydXdfXGSf5bk31fVq2eR/Ex3b1gPksAAAIPp7kuTXHrAa29Yt/+FJM/a7LgSGAAgG9c8jix6YACAyVGBAQAm9yykYROY1U7t3jPoV2yn3rt37BA2b9++sSPYnL0TizdJr6yMHcLmTawWPMlrvDqta5zDL6nBdpj6NR6giXdIbiEBAJPjFhIAkJpYUVEFBgCYHBUYAFh2nck18arAAACTowIDAEuvzEICABiaCgwAoAcGAGBoKjAAgAoMAMDQVGAAABUYAIChqcAAwLLrWAcGAGBoKjAAgKdRAwAMTQUGADALCQBgaAsnMFX1sqr6cFVdUVWnbmdQAACHs9AtpFnC8tzuPneb4wEARrAsTbzPS3L0rALzlqo6ejuDAgA4nEUTmEcmOW5Wgbk7yYXr36yqC6pq176+d6vxAQA7oWuxbSSLJjB3JPmj2f5Hknzv+je7+5LuvuiYetBWYgMAOKhFE5grkzxhtv+kJF/annAAgB3XW9hGslATb3d/tqruqaorktyS5M3bGhUAwGEsvJBdd792OwMBAEa0JLOQAABG41ECAMDSrAMDADAaFRgAQA8MAMDQVGAAABUYAIChqcAAwJKrNgsJAGBwKjAAwKhPll6ECgwAMDkqMACAWUgAAENTgQEAJjcLSQIDALiFBAAwtGErMN3pvXsH/YpttW/f2BFs3t5pxdwrK2OHsHk9sf8tyQSv8+r0rnF6dewIHvB6ij8XU2UhOwCA4emBAQD0wAAADE0FBgBQgQEAGJoKDABgFhIAwNAkMADA5EhgAIDJ0QMDAJiFBAAwNBUYAFh2noUEAPC3qur8qrq2qq6vqtcf4pyfqKovVNU1VfXOecZVgQEABumBqaqjk7w1yf+Q5MYkn6mqi7v7C+vOOSPJv0jyrO7+elV9xzxjq8AAAEN5apLru/uL3b0nybuTXHjAOf84yVu7++tJ0t1fm2dgCQwAsFaBWWRLTq6qq9dtF60b9dQkN6w7vnH22nrfk+R7quqPq+pTVXX+POG6hQQAbMUt3X3WFj5/TJIzkpyd5LQkH6uq7+/u2w/3IRUYAFhylbVZSItsG7gpyenrjk+bvbbejUku7u693f2lJH+RtYTmsCQwAMBQPpPkjKp6bFUdl+RFSS4+4Jw/yFr1JVV1ctZuKX1xo4ElMADAVnpgDj1k974kr0xyWZL/luT3u/uaqnpjVf3w7LTLktxaVV9I8tEk/7y7b90o3IV6YKrqMUmumgWzp7t/cJFxAIAjwIAL2XX3pUkuPeC1N6zb7ySvmW1z20oT74e6+yVb+DwAwEK2cgvpnKr6eFW9ev2LVXVRVV1XVTffs3LnFsMDAHbEALeQhrRoAvPlrDXZnJPkvKp6wv43untXd5/R3aeccPRDtyNGAID7WSiB6e57u/uuWXPOHyZ5/PaGBQDsqGWowFTV+tLKs5L85faEAwCwsUVvIT2nqv6kqq5MclN3X7WdQQEAO2ughewGs9AspINNiQIA2CmehQQAjNrPsggr8QIAk6MCAwDLbuQZRYtQgQEAJkcFBgAYdUbRIlRgAIDJUYEBAPTAAAAMTQUGANADAwAwNBUYAEAPDADA0FRgAGDZTXAlXgkMACy5mm1T4hYSADA5KjAAgFtI99fJvn3DfsV22juhWGd6ZWXsEDanJ/ZfSCZ4jZNkdWLXuVfHjuABr6f2MwEbUIEBACxkBwAwNBUYAGByPTAqMADA5KjAAAAqMAAAQ1OBAYBl12YhAQAMTgUGANADAwAwNBUYAEAPDADA0FRgAAA9MAAAQ1OBAQD0wAAADE0FBgCWXWdyPTASGABgcgmMW0gAwORsKYGpqldX1Se2KxgAYOdV1pp4F9nGsnACU1UPSvKkbYwFAGAuW6nAvDzJb29XIADAiHrBbSQLJTBVdWySs7v7I4d4/4Kq2rV3dc+WggMAOJhFKzAvTfLOQ73Z3Zd090XHHnXcgsMDADupuhfaxrJoAvP3kryiqj6Q5Puq6lXbGBMAwGEttA5Md79u/35VfaK737J9IQEAO2qCC9lteR2Y7n72dgQCADAvK/ECAB7mCAAwNBUYAGD5emAAAHaaCgwAoAcGAGBoKjAAgB4YAIChqcAAwLJrPTAAAPepqvOr6tqqur6qXn+Y8/7HquqqOmuecSUwAMDfPg9ps9thVNXRSd6a5PlJzkzy4qo68yDnPTTJP01y1bzhSmAAYMlV1m4hLbJt4KlJru/uL3b3niTvTnLhQc77pSS/nGT3vDFLYACArTi5qq5et1207r1Tk9yw7vjG2Wv3qaonJzm9u9+/mS/VxAsAJL1wF+8t3T1X38qBquqoJP9Xkp/Z7GdVYACAodyU5PR1x6fNXtvvoUken+SKqvqrJE9PcvE8jbwqMADAUNOoP5PkjKp6bNYSlxcl+cn9b3b3HUlOvi+GqiuSvLa7r95o4GETmO5k775Bv2I79crK2CFs3uIlv1FM8hqvTusaJ0l6dewIHvB6ij8XUzO1n+OJ/X28E7p7X1W9MsllSY5O8rbuvqaq3pjk6u6+eNGxVWAAYNnNMSV64aG7L01y6QGvveEQ554977h6YACAyVGBAQBSE7tjpwIDAEyOCgwAMFgPzFBUYACAyVGBAQCGWgdmMCowAMDkqMAAwLLrTG4hPhUYAGByVGAAAD0wAABDU4EBAKwDAwAwNBUYAFhylen1wEhgAGDZdZtGDQAwNBUYAGByt5BUYACAyVGBAQCWYxp1VT2+qq6sqo9X1W9VVW13YAAAh7LoLaRru/uZ3f2c2fFZ2xUQALDzqhfbxrJQAtPde9cd3pvkhv0HVXVRVV1XVTffs3rXVuMDAPgWCzfxVtUPV9WfJ3lkklv3v97du7r7jO4+5YSjTtyOGAGAIXWS1V5sG8nCCUx3X9zdj09yY5IXbF9IAACHt9AspKp6UHffOzv8RpJ7ti8kAGDHTWwW0qLTqM+vqtfM9q9L8sFtigcAYEMLJTDd/b4k79vmWACAkViJFwBgYFbiBQA8jRoAYGgqMACAHhgAgKGpwADAsutMbh0YFRgAYHJUYABgyVWSmtgsJAkMAJCsjh3A5riFBABMjgoMADC5W0gqMADA5KjAAMCyM40aAGB4KjAAsPTawxwBAIY2bAWmk15ZGfQrttXEss9kYtc3SVand40ZXvu5GF5PbJEPdpyHOQIADEwPDAAwubsQKjAAwOSowADAsuukJtYmpQIDAEyOCgwAoAcGAGBoKjAAgGchAQAMTQUGAEjpgQEAGJYKDAAwuVlIEhgAWHadxEJ2AADDUoEBgCVXaU28AABDk8AAAGtNvItsG6iq86vq2qq6vqpef5D3X1NVX6iqz1fVh6vqu+cJVwIDAAyiqo5O8tYkz09yZpIXV9WZB5z2Z0nO6u4nJHlPkjfNM7YEBgAYqgLz1CTXd/cXu3tPkncnufD+X9sf7e67Z4efSnLaPOEulMBU1dOq6sqq+kRVvXmRMQCAB4STq+rqddtF6947NckN645vnL12KC9P8l/m+dJFZyH9dZIf6O7dVfV7VfX93f1fFxwLABjT1taBuaW7z9pqCFX1kiRnJXnuPOcvlMB091fWHe5NsrLIOADAA9pNSU5fd3za7LX7qarzkvzLJM/t7nvnGXhL68BU1ROSnNLdXzjg9QuSXHBCPXQrwwMAO2SgdWA+k+SMqnps1hKXFyX5yft9b9XfT/KbSc7v7q/NO/DCTbxV9e1JfjVr96vup7sv6e6Ljq3jFh0eAJi47t6X5JVJLkvy35L8fndfU1VvrKofnp32b5M8JMl/qqrPVtXF84y9UAWmqo5J8rtJXnvA7SQAYIoGWom3uy9NcukBr71h3f55i4y7aAXmx5M8JcmbquqKqnrGguMAAGzaok2870ryrm2OBQAYxXyr6h5JLGQHAEyOp1EDwLLrqMAAAAxNBQYA2MpKvKNQgQEAJkcFBgAYaiXewajAAACTowIDAExuFpIEBgCWXSdZnVYC4xYSADA5KjAAsPQ8SgAAYHAqMACACgwAwNBUYAAAFRgAgKENWoHp7vSevUN+xfbqiT3JaoJ6YusMTJafZQ40sf+7ZodZBwYAYHh6YABg6fXkKrcqMADA5KjAAACT65NSgQEAJkcFBgCWnVlIAADDU4EBAPTAAAAMTQUGAFCBAQAYmgoMACy9nlwFRgIDAMuuk6x6lAAAwKBUYACAyd1CUoEBACZHBQYAUIEBABiaCgwALL1ejoc5VtWjqupPq2p3VUmCAIAdtWjycVuSc5O8dxtjAQDG0En3tNaBWSiB6e7dSXZX1TaHAwCwsW1v4q2qi6rquqq6eXfftd3DAwBDWO3FtpFsewLT3bu6+4zuPuX4OnG7hwcAMAsJAMhyrANTVcdW1eVJnpjksqp62vaGBQBwaIs28e5Nct42xwIAjKHb06gBAIamBwYAWI4eGACAManAAABpPTAAAMNSgQGApdeT64GRwADAsuuM+liARbiFBABMjgoMAJC0Jl4AgEGpwADAkuskrQcGAGBYKjAAsOy69cAAAAxNAgMApFd7oW0jVXV+VV1bVddX1esP8v6Dquo/zt6/qqoeM0+8EhgAYBBVdXSStyZ5fpLlxMfHAAAFeElEQVQzk7y4qs484LSXJ/l6d//dJG9O8svzjC2BAQDWemAW2Q7vqUmu7+4vdveeJO9OcuEB51yY5Ldn++9Jcm5V1UYDD9rEe2ffdsuH9rzzrwcY+tFJ/maAcYfy8CR3jB3EJrnGw3ONh+caD2tq1zdxjdf77v07d+brl13e7zl5wXGOr6qr1x3v6u5ds/1Tk9yw7r0bkzztgM/fd05376uqO5I8Iskth/vSQROY7j5liHGr6ubuPmuIsYdQVbu6+6Kx49gM13h4rvHwXONhTe36Jq7xoXT3+UN/x3ab6i2k28cOYJMuGTuABbjGw3ONh+caD2tq1zdxjXfaTUlOX3d82uy1g55TVcdkrUp260YDTzWBmVL5L909tf9gEtd4J7jGw3ONhzWp65u4xiP4TJIzquqxVXVckhclufiAcy5O8tOz/Rcm+Uh3bzi9aaoL2e3a+BS2yDUenms8PNd4WK7v8CZ9jWc9La9MclmSo5O8rbuvqao3Jrm6uy9O8v8k+Z2quj7JbVlLcjZUcyQ5AABHlKneQgIAltikEpiqelRV/WlV7Z41+hzRquppVXVlVX2iqt48djzzqKrHz2L+eFX91jxz8Y8EVfXqqvrE2HHMo6oeU1VfraorquqDY8czj6p6WVV9eBbzqWPHs5HZyp9XzLYvV9WPjB3T4VTVg6vq/bN431dVDxo7psOpqmOq6t1V9dGqetPY8RzOwX5vVNWbZ3/H/crY8R3owHin9ntvJ00qgcnavbFzk3xq7EDm9NdJfqC7n53kO6rq+8cOaA7Xdvczu/s5s+Mjfork7C/7J40dxyZ9qLvP7u4fHDuQjcwSlud297mzmA+cQXDE6e4PzGI9O2traFw+ckgbOT/JVbN4Pz07PpL9aJLPdfc5SU6oqieOHdBh3O/3RlU9OclDZn/HHVdVTxkzuIM48Pfc1H7v7ZhJJTDdvbu7vz52HPPq7q909+7Z4d4kK2PGM4/u3rvu8N7cfwGiI9XL87erOE7FObP/A3z12IHM4XlJjp5VYN4yWxp8EqrqcUm+2t3fHDuWDfxlkhNn+ydljimkI3tcks/P9j+b5JkjxnJYB/m98fQkH5rtX57kGTsf1aEdGO/Ufu/tpEklMFNVVU9Ickp3f2HsWOZRVT9cVX+e5JE5wv8irapjk5zd3R8ZO5ZN+HKS70lyTpLzZj8fR7JHJjmuu89Ncne+dRnwI9mPJXnv2EHM4bokz6iqa7JW9bxy5Hg2cm2S5872z8la0jUVJyX5xmz/jkwrdtaRwAysqr49ya9mrUowCd19cXc/PmtLPr9g7Hg28NIk7xw7iM3o7nu7+67u3pfkD5M8fuyYNnBHkj+a7X8kyfeOGMtmXZBvXXPiSPTTSS7p7u9L8v4kLxk5no1ckrVbRx/OWqX2qyPHsxl3JHnYbP9hmf5CcUtLAjOgWcPV7yZ5bXd/Zex45nFA8+A3ktwzVixz+ntJXlFVH0jyfVX1qrED2khVPXTd4bOydvvgSHZlkv1Voicl+dKIscytqr4zyZ7uPqKriDOVtV6HZO35Lw8fMZYNdfdKd79qVpVbydoaH1Pxyaz1lCTJedFbMlmTSmCq6tiqujzJE5NcVlUHPhDqSPPjSZ6S5E2z2QVH1L3WQzi/qv6oqv4oa7cOjuhZMt39uu5+3uw5Htd091vGjmkOz6mqP6mqK5Pc1N1XjR3Q4XT3Z5PcU1VXZO3n+T3jRjS3C5O8b+wg5vTOJD8xu8Y/leT3xg3n8Krq1NnfaR9JcuWR3Nh94O+NJMcm2V1VH0+y0t2fHjXAAxzs99zEfu/tGAvZAQCTM6kKDABAIoEBACZIAgMATI4EBgCYHAkMADA5EhgAYHIkMADA5Pz/E6+uulYNRiUAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 1440x1440 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"V_dx = np.vstack((np.diff(V, axis=0), np.zeros(nvmdp.width)))\n",
"V_dy = np.hstack((np.diff(V, axis=1), np.zeros((nvmdp.height,1))))\n",
"\n",
"plt.figure(figsize=(20,20))\n",
"nvmdp.visualize_grid(np.sqrt(V_dx**2+V_dy**2), trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value Gradient\")"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[1.41, 0.28, 0.23, 0.18, 0.14, 0.12, 0.09, 0.07, 0.06, 0.05, 0.03],\n",
" [0.28, 0.23, 0.18, 0.14, 0.12, 0.09, 0.07, 0.06, 0.05, 0.04, 0.02],\n",
" [0.23, 0.18, 0.14, 0.12, 0.09, 0.07, 0.06, 0.05, 0.04, 0.03, 0.02],\n",
" [0.18, 0.14, 0.12, 0.09, 0.07, 0.06, 0.05, 0.04, 0.03, 0.03, 0.01],\n",
" [0.14, 0.12, 0.09, 0.07, 0.06, 0.05, 0.04, 0.03, 0.03, 0.02, 0.01],\n",
" [0.12, 0.09, 0.07, 0.06, 0.05, 0.04, 0.03, 0.03, 0.02, 0.02, 0.01],\n",
" [0.09, 0.07, 0.06, 0.05, 0.04, 0.03, 0.03, 0.02, 0.02, 0.01, 0.01],\n",
" [0.07, 0.06, 0.05, 0.04, 0.03, 0.03, 0.02, 0.02, 0.01, 0.01, 0.01],\n",
" [0.06, 0.05, 0.04, 0.03, 0.03, 0.02, 0.02, 0.01, 0.01, 0.01, 0. ],\n",
" [0.05, 0.04, 0.03, 0.03, 0.02, 0.02, 0.01, 0.01, 0.01, 0.01, 0. ],\n",
" [0.03, 0.02, 0.02, 0.01, 0.01, 0.01, 0.01, 0.01, 0. , 0. , 0. ]])"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"np.sqrt(V_dx**2+V_dy**2).round(2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "irl_python3",
"language": "python",
"name": "irl_python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: OpenAI gym not installed.\n"
]
}
],
"source": [
"# Other Imports.\n",
"from simple_rl.tasks import NavigationWorldMDP\n",
"from simple_rl.agents import QLearningAgent\n",
"from simple_rl.planning import ValueIteration\n",
"from simple_rl.tasks.grid_world.GridWorldStateClass import GridWorldState\n",
"from simple_rl.tasks.navigation.NavigationStateClass import NavigationWorldState\n",
"\n",
"# Python Imports.\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"%matplotlib inline\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def value_iter(S, A, R, T, gamma, policy, n_iters):\n",
" \n",
" nS, nA = len(S), len(A)\n",
" V = np.zeros((nS))\n",
" Q = np.zeros((nS, nA))\n",
" Pi = np.ones((nS, nA)) / nA\n",
" s_to_idx = {s: idx for idx, s in enumerate(S)}\n",
" \n",
" for _iter in range(n_iters):\n",
" for si, s in enumerate(S):\n",
" \n",
" if s.is_terminal():\n",
" continue\n",
" for ai, a in enumerate(A):\n",
" s_prime_idx = s_to_idx[T(s,a)]\n",
" Q[si,ai] = R(s,a) + gamma * V[s_prime_idx]\n",
" V[si] = policy(Q[si,:]).dot(Q[si,:])\n",
" return V"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAFyCAYAAAApoxulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFOtJREFUeJzt3X2QZXV95/H3B2aAyMMMhBEDSTtFiljG4SEGiww6AYQSzAbUJJoHn1JlMlVWlk25S6LZyiYmf8QqUrusq7VbO7VVKRUIRqOBEYoncSioUVi0MCvZYkcq2ohAZkAaH+hxGL77xz0jl85gD6dP3/4x9/2qmurzcO/5fk9P96d//Tvn3k5VIUlqyyEr3YAk6V8znCWpQYazJDXIcJakBhnOktQgw1mSGmQ4qwlJ3p7k5mU69kyS7yU5dDmOLy0Hw1kHLMk3kvxLkiPHtv1ekm1LPXZVXVVVb1jqceBHfV4wduzZqjqqqvYOcfwFtar7nKwa27a621Zj27YlmU/y3SRPJvlykg8kOXzsMR9Msqf7QfJEku1JNg7ds14cDGe9UIcCf7jSTTTmO8Abx9bf2G1b6N9W1dHATwH/Afgt4IYkGXvMJ6vqKGAdcCfwmQX7NSUMZ71Qfw1clmTt/nYm+XCSB8dGh5u67ScmeSrJcWOP/YUku7qR5u8muXNs3xuS3J9kLsl/T3J7kt/r9v1sktuSPNY9/6p9/ST5BDADbO1GoH+cZH03wl011st1SR5P8vUkvz9W94NJ/i7Jx7tR7n1Jzlzkc/IJ4F1j6+8CPv58D66q71fVNuASYCPwb/bzmD3Ax4CXAT+5SH0dhAxnvVD3ANuAy55n//8GzgCOA64GPpXkiKr6NvBF4NfHHvs7wKe7IPqRJMcDnwb+hFEw3Q+cPf4Q4EPAicArgZ8BPghQVe8EZoGLu6mMy/fT4zXAt7rn/wbwV0leP7b/ku4xa4HrgI8+z7nu8w/ALydZm+RYYBNw7SLPoapmGX0+Ny3c1013/C7wYFXtWuxYOvgYzurjz4BLk6xbuKOqrqyqx6rq6ar6z8DhwCu63VcDvw3Q/ar+W922hX4FuK+qPlNVTwP/DXhkrMbXq+qWqtpdVTuB/wKccyCNJ/kZ4LXA+6tqvqruBf4Xzx353llVN3Rz1J8ATl/ksPPAVuA3u3/XddsOxLcZ/SDb521JngAeBH4ReMsBHkcHGcNZL1hVfQ34HPCBhfuSXJbk/3bTEU8Aa4Dju91/D2xM8lPALwPPAHfsp8SJjMJpX71iNNLdV+OEJNckeSjJk8CVYzUWcyLweFV9d2zbN4GTxtYfGVv+AXDE+AW/5/FxRgH/Y6c09uMk4PGx9b+rqrVV9dKqen1VffkFHEsHEcNZff058PuMhVo3v/zHwNuAY6tqLTDHaBqCqvoOcDOj0eXvANfU/t8W8WHgp8eOm/F14K+AAk6tqmOAd+yr0flxb7X4beC4JEePbZsBHvpxJ3sA7mB0oe8ERhfyFtWN4n+R/f+A0pQznNVLVX0d+CTw78Y2Hw08DewEViX5M+CYBU+9mtHo8jfY/5QGwPXAqUne3I1Y/4DRhbHxOt8D5pKcBPzRguc/Cpz8PH0/CGwHPpTkiCSnAe9hNPrurfshczFwyfP8wPmRJC9Jcg6jeem7gRuWUlsHJ8NZS/GXwJFj6zcBNwL/j9FUwTxj0xOd64BTgEeq6qv7O2h3AeytwOXAY8DPM7pwtrt7yF8Ar2Y0Kr8e+MyCQ3wI+NPuXuH9Xbj8bWA9o1H0Z4E/r6pbFznXRVXVfVV13495yEeTfJfRD4//ymia56KqemaptXXwiW+2r9YlOYTRnPPbq+oLK92PNAmOnNWkJBd2t6YdDvxHRnPKX1rhtqSJMZzVqo3AA8AuRnO5b66qp1a2JWlynNaQpAY5cpakBi12Y/2SHH/88bV+/frlLCFJLyrf+MY32LVr16JvZrWs4bx+/Xruueee5SwhSS8qZ5652PtojTitIUkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNWjRcE5yYpKvJJlPsmrh+iSalKRpcyAj58eB83n2r1AsXJckDWzRcK6q+e5P2u93faEkm5PsSLJzdnZ2qD4laaoMPudcVVuq6pSqWjczMzP04SVpKnhBUJIaZDhLUoMO5G6N1UluBU4Hbkpy1sL15W5SkqbNorfCVdUe4IIFmxeuS5IG5LSGJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMWDeckJyb5SpL5JKu6bVckuSPJh5e/RUmaPgcycn4cOB/4EkCSVwNHVdUm4LAkr1nG/iRpKi0azlU1X1XfGdv0S8At3fKtwMblaEySplmfOee1wJPd8ly3/hxJLk6yZW5ubim9SdLU6hPOc8Ax3fIxwBMLH1BVW6tq85o1a5bSmyRNrT7h/EVGc9AAF9DNRUuShnMgd2usTnIrcDpwE7AamE9yB7C3qu5e5h4laeqsWuwBVbWH0Qh53F3L044kCXwRiiQ1yXCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJalCvcE6yKsk1Sb6Q5PKhm5Kkadd35PwW4KtVdR7wE0lOH7AnSZp6fcP5ZOAfu+V7gbP37UiyOcmOJDtnZ2eX2p8kTaW+4Xw/cE63fB6wdt+OqtpSVadU1bqZmZml9idJU6lvOG9lNJ3xeWA38OhwLUmSeoVzVe2tqkur6nxgL3DTsG1J0nRb1edJSU4CrgKeAT5eVQ8N2pUkTble4dyF8bnDtiJJ2scXoUhSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1qNdf307yEuBTwJHAHPC2qto9ZGOSNM36jpwvAu6qqnOBu7t1SdJA+obzA4xGzQBrgceGaUeSBP3DeQewMcl9wJnA9vGdSS5OsmVubm6p/UnSVOobzu8GtlbVq4DrgXeM76yqrVW1ec2aNUvtT5KmUt9wDvB4t7wLMIUlaUC97tYArgY+meSdwB7gN4drSZLUK5yr6gngwoF7kSR1fBGKJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqUK9wTnJRkm3dv4eTvHnoxiRpmq3q86SquhG4ESDJXcCtQzYlSdNuSdMaSU4GHq2q7w3UjySJpc85/xrw2fENSTYn2ZFk5+zs7BIPL0nTaanhfDFw3fiGqtpSVadU1bqZmZklHl6SplPvcE7yMuCHVfXYgP1IkljayPlNwLVDNSJJelavuzUAqup/DtmIJOlZvghFkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIa1PsVgtJKSCZfs2ryNSVHzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWQe/I5+Gv/w/o4/Si4ThrIPf2btg02OwcddKdyIdMMNZB783PvLcj9KLQO83PkryLuDdwKHA26vqocG6kpbir++FM594dv2H3bslbZiDL2x7dvs9a+GPzphoa9KB6hXOSU4Czqmq8wfuR1q6q14OG56EI54ZrR9Wz/0IMH8IXPnyyfcmHaC+0xoXAocm+XySjyQ5dHxnkouTbJmbm1t6h9ILde+x8CenjgJ4f+YPgQ+cCl89drJ9SS9A33A+ATisGzn/AHjT+M6q2lpVm9esWbPU/qR+7j0W/uLnYfeCL/Hdh4y2G8xqXN9wngNu75ZvA145TDvSgI56GvYy+jd/yLPLR3lLndrXN5y3A6d1y2cA/zxMO9KAfuWR0bzzA0fBn24YfTziGe/a0ItCrwuCVXVvkqeSbAN2AVcM2pU0hO8fCv/jZ+Hvfxoq8N5j4dcfhNO8FqL29b6VrqouG7IRaXD/6dTnrj8T+NQMfGpl2pFeCF+EIkkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWpQ7xehSCuhavHHSAcDR86S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJalCvcE6yPsmjSbYluXnopiRp2i3lXeluqap3DNaJJOlHljKtcV6SO5K8b7BuJElA/3B+GPg54DzggiSn7duRZHOSHUl2zs7ODtGjJE2dXuFcVbur6vtV9TTwOWDD2L4tVXVKVa2bmZkZqk9Jmip9LwgePbb6WuCBYdqRJEH/aY1NSb6cZDvwUFXdNWRTkjTtet2tUVU3ADcM3IskqeOLUCSpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoOWFM5J3pfkzqGakSSN9A7nJIcDZwzYiySps5SR83uAjw3ViCTpWb3COclq4Nyquu159l+cZMvc3NySmpOkadV35PxO4Orn21lVW6tq85o1a3oeXpKmW99wfgXw3iQ3Aq9KcumAPUnS1FvV50lV9f59y0nurKqPDNeSJGnJ9zlX1euGaESS9CxfhCJJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhrUK5yTbEiyPckdSf4mSYZuTJKmWd+R8/1VdXZVberWzxyqIUlSz3Cuqj1jq7uBB/etJNmcZEeSnbOzs0vtT5KmUu855ySXJPkacALw2L7tVbWlqk6pqnUzMzND9ChJU6d3OFfVdVW1AfgW8KvDtSRJ6ntB8PCx1SeBp4ZpR5IEsKrn8y5K8u+75R3AzQP1I0miZzhX1bXAtQP3Iknq+CIUSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAb1CuckZyXZnuTOJFcM3ZQkTbu+I+dvAq+vqtcBL01y6oA9SdLUW9XnSVX1yNjqHmDvMO1IkmCJc85JTgPWVdU/Ldh+cZItc3NzS2pOkqZV73BOchzwUeA9C/dV1daq2rxmzZql9CZJU6vvBcFVwJXAZQumOCRJA+g7cn4r8Brg8iTbkmwcsCdJmnp9Lwj+LfC3A/ciSer4IhRJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktSgXuGc5MQkX0kyn2TV0E1J0rTrO3J+HDgf+NKAvUiSOr1GvVU1D8wnGbgdSRIsw5xzks1JdiTZOTs7O/ThJWkqDB7OVbWlqk6pqnUzMzNDH16SpoJ3a0hSg/rerbE6ya3A6cBNSc4ati1Jmm59LwjuAS4YuBdJUsdpDUlqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJalCqavkOnuwEvtnjqTPASryl3RpgbgXqrsT5TtO5wsqc7zSdK/h1fKBeXlXrFnvQsoZzX0l2Hkjzy1B3S1VtXoG6Ez/faTrXru7Ez3eazrWr69fxgFqd1nhihepuXaG6K3G+03SusDLnO03nCn4dD6rVcF6JX1OoqpX6j574+U7TucKKne80nSv4dTyoVsN5y0o3MGHTdL6e68Frms532c+1yTlnSZp2rY6cJWmqNRfOSU5M8pUk80l6/TGAHjXPSrI9yZ1JrphEza7uhq7uHUn+JhP8c+ZJ3pfkzgnWW5/k0STbktw8qbpd7Xcl+XxX+6QJ1Luoq7UtycNJ3rzcNbu6L0lyfVf32iSHT6juqiTXJPlCksuXuda/yockV3TfQx+eVN1J5FRz4Qw8DpwPfGmCNb8JvL6qXge8NMmpE6p7f1WdXVWbuvUzJ1G0+6Y9YxK1Frilqs6tqjdMqmAXxudU1fld7YeWu2ZV3djVOpfRvbC3LnfNzkXAXV3du7v1SXgL8NWqOg/4iSSnL2Ot5+RDklcDR3XfQ4clec0k6u5nfXDNhXNVzVfVdyZc85Gqmu9W9wB7J1R3z9jqbuDBSdQF3gN8bEK1xp3XjXDeN8GaFwKHdiPnjyQ5dFKFk5wMPFpV35tQyQeAI7vltcBjE6p7MvCP3fK9wNnLVWg/+fBLwC3d8q3AxknUnURONRfOKynJacC6qvqnCda8JMnXgBOYwDdTktXAuVV123LXWuBh4OeA84ALus/1JJwAHFZV5wM/AN40oboAvwZ8doL1dgAbk9zH6Lew7ROqez9wTrd8HqMfDJOyFniyW56bcO1lZTh3khwHfJTRqHJiquq6qtoAfAv41QmUfCdw9QTqPEdV7a6q71fV08DngA0TKj0H3N4t3wa8ckJ1AS4GrptgvXcDW6vqVcD1wDsmVHcro+mMzzP6DfDRCdWF0f/vMd3yMazcC38GZzgzuqABXAlcVlWPTLDu+AWbJ4GnJlD2FcB7k9wIvCrJpROoSZKjx1Zfy+hX8EnYDuwbpZ8B/PMkiiZ5GfDDqprU1AJAGM2FAuxi9L4Ty66q9lbVpd1vJ3uBmyZRt/NFRnO/ABcw2WtVy6q5cE6yOsmtwOnATUnOmkDZtwKvAS7vrnQvy7zVflyU5PYktzP69XvZ72KoqvdX1YVVdRFwX1V9ZLlrdjYl+XKS7cBDVXXXJIpW1b3AU0m2Mfo//vQk6jKaPrl2QrX2uRp4W3eubweumkTRJCd13ze3AduX86LrwnwAVgPzSe4A9lbV3ZOo293htaw55YtQJKlBzY2cJUmGsyQ1yXCWpAYZzpLUIMNZkhpkOEtSgwxnSWrQ/wcH+j8cfc2HfAAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 432x432 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"np.random.seed(0)\n",
"nvmdp = NavigationWorldMDP(width=11, height=11, \n",
" nav_cell_types=['white'],\n",
" nav_cell_rewards=[0],\n",
" nav_cell_p_or_locs=[1.],\n",
" goal_cell_locs=[[(6,6)]],\n",
" goal_cell_rewards=[1.],\n",
" goal_cell_types=[\"blue\"],\n",
" slip_prob=0.00, step_cost=0.01, gamma=.99)\n",
"nvmdp.visualize_grid()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"S = nvmdp.get_reachable_states()\n",
"A = nvmdp.get_actions()\n",
"s_to_idx = {s: idx for idx, s in enumerate(S)}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# High Temperature"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAGCCAYAAADQaKXvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuwZWV55/HvrxuaVtQIQoxyJ+mkBBVMELwRIRBpxgAmpQZnVFJlhdIRa0bLTEwywYRMqhJTNU4qIcauCimNGuKlEpvYscNFLB0idquggQzVLRqgvQRoQIP07Zxn/tirzebQ3WeffXavtdc530/VKtZlr/W8q885++G9rHelqpAkqS0rui6AJGl5MfFIklpl4pEktcrEI0lqlYlHktQqE48kqVUmHi1bSU5MUkkO6bos0nJi4lGvJfl0kqv2sf+SJN8xqUjTx8SjvvsA8PokmbP/DcCHq2pPB2WSdAAmHvXd3wHPAM7euyPJEcAvAB9M8sokX0nyvST3Jvmd/V0oyTeTnD+0/TtJPjS0/aIktyR5OMntSc45GDckLXUmHvVaVT0GfBR449Du1wL/r6puBx5tjj0deCXwliSvWmicJMcAnwL+F3Ak8E7gE0mOXtwdSMuPiUdLwQeAVydZ3Wy/sdlHVd1cVV+rqtmq+irw18DLx4jxemBDVW1ornU9sBn4TxMov7SsmHjUe1X1eeAB4FVJfhw4E/gIQJKzknwmyf1JHgHeDBw1RpgTgNc0zWwPJ3kYeBnwrMnchbR8OOJHS8UHGdR0fgrYWFXfbfZ/BPhT4MKq2pHk/7D/xPMo8OSh7R8bWr8X+Kuq+tXJFltafqzxaKn4IHA+8Ks0zWyNpwLbm6RzJvCfD3CN24BLkxya5Azg1UPHPgRclOSCJCuTrE5yTpJjJ3wf0pJn4tGSUFXfBG4BDgfWDx36r8BVSb4PXMlgIML+/Dbw48BDwO/SNNc1178XuAT4TeB+BjWgX8O/IWnB4ovgJElt8v/WJEmtMvFIklpl4pEktcrEI0lqlYlHktSqg/oA6aocVqs5/GCG0DSZOz+02uHA1GVjB4+yq3YG4IJzD68Ht8+MdN6XvrpzY1WtPaiFW4CDmnhWczhn5byDGWI6PWGG/jZjd1eJzQozTxdqtsPMU7Mdxl5+GffWuvGH6w9un+GLG48f6byVz9oyzjRRB41T5khSDxUwS4eJfxFMPJLUS8VMlzXORXBwgSSpVdZ4JKmHBk1t/eznMvFIUk/ZxyNJak1RzPR0ZJ99PJKkVlnjkaSeso9HktSaAmZMPJKkNvW1xjNvH0+SZyf5cpIdSQ6Zu91GISVJS8cogwu2A+cBX9jPtiSpZQXMVI20TJt5ayxVtQPYkWbiy7nbkqRu9PMpHvt4JKmXiurt4IKJP8eT5PIkW5Lcv4NHJ315SVLPTTzxVNW6qlpTVUf7EjhJOkgKZkZcps0oo9oOTXIDcBqwMclZc7cPdiElSY83mCR0tGXajDK4YDdw/pzdc7clSa0KMz1937xztUmSWuWoNknqoQJmp7D/ZhQmHknqqb42tZl4JKmHBpOE9jPx2McjSWqVNR5J6qnZ6meNx8QjST3U56Y2E48k9VARZnraW9LPUkuSessajyT1lH0806bL9wWlu4pkVq7sMHaHFegO7xuAmZkOY3c3G1d1eNudzkI2BS9X63Mfj01tktRLYaZWjLTMe6VkbZK7kmxN8q59HH9zkq8luS3J55OcMnTsN5rz7kpywSglN/FI0jKWZCVwNXAhcArwuuHE0vhIVT2vqk4H3gP87+bcU4BLgVOBtcCfNdc7IBOPJPXQ4LUIK0Za5nEmsLWq7q6qXcC1wCWPi1X1vaHNw5vwNJ+7tqp2VtU3gK3N9Q5o6fbxSNISN6E+nmOAe4e27wOe8J61JG8F3gGsAn5u6NwvzDn3mPkCWuORpB6qWlAfz1FJNg8tly88Xl1dVT8O/DrwPxdTdms8krT0PVBVZ+zn2DbguKHtY5t9+3Mt8L4xzwWs8UhSb82SkZZ5bALWJDkpySoGgwXWD38gyZqhzVcCW5r19cClSQ5LchKwBvjifAGt8UhSDw2e41l83aGq9iS5AtgIrASuqao7klwFbK6q9cAVSc4HdgMPAZc1596R5KPAncAe4K1V8z/dZeKRpF7KSM/ojKKqNgAb5uy7cmj9vx3g3N8Hfn8h8WxqkyS1yhqPJPXQ3ud4+sjEI0k9NdPTSULnTZdJnp3ky0l2JDmk2ffeJJ9L8scHv4iSpLn2vo9nlGXajFKi7cB5NE+nJvlp4ClVdTawKskLD2L5JElLzLxNbVW1A9iR/3jNwIuA65v1G4AXMxgHLklq0eyERrW1bZxSPx3YO2HcI822JKlFe5/j6WNT2ziDCx4BntasPw14eO4HklwEXPQkDl9E0SRJ+1Nk6Q4u2Id/YtDnA3A+j5+ZFICquq6qLj+EVYspmyRpCRplVNuhSW4ATmMwpcKhDPp8PgfMVNW88/JIkiZvQu/jad0ogwt2M6jZDLv14BRHkjSKKiY2ZU7bfIBUknpppJmnp1I/06Ukqbes8UhSDxU2tUmSWjaNz+iMwsQjST1UhNll9ByPJEljs8YjST1lU5skqTVFfycJNfFIUi+FGZ/jkSRpfge/xpOOMnK6y6lZubK72KsP6yz2isOf3FlsOrxvAHbs7Cz07KM/6Cx2l/ddM52FBma7CVuPX7WpTZLUqr42tZl4JKmHqtLbGk8/Sy1J6i1rPJLUU87VJklqTUFvX4tg4pGkXkpvazz9LLUkqbes8UhSDw2e47GpTZLUIicJlSS1xvfxSJI0Ims8ktRTsz2tO4yVeJIcAnwIeCawqar+x0RLJUk6oCqYWWZNbb8I3F5V5wJPSnLaBMskSRrBbGWkZdqMm3hOBr7arN8GvGQyxZEkLXXjJp67gJc36+cCT59McSRJoxiMalsx0jJtxi3RdQya2G4EdgLf3XsgyeVJtiS5fwcdvqBKkpa4meb11/Mt02asxFNVM1X1tqo6D5gBNg4dW1dVa6rq6NV0+EZKSVrC9s5c0Mc+nnFHtR0DfJjB+18/WFXbJloqSdKSNVbiaRLNOZMtiiRpdL6BVJLUslky0jKfJGuT3JVka5J37eP4O5LcmeSrSW5McsLQsZkktzXL+lHK7cwFktRDk3qANMlK4Grg54H7gE1J1lfVnUMf+wpwRlX9IMlbgPcAv9wce6yqTl9ITGs8krS8nQlsraq7q2oXcC1wyfAHquozVbV3mPIXgGMXE9DEI0k9NaHneI4B7h3avq/Ztz9vAv5haHt1ks1JvpDkVaOU26Y2SeqhBb4W4agkm4e211XVuoXGTPJ64Az+YwIBgBOqaluSk4Gbknytqr5+oOuYeCSpp0YZONB4oKrO2M+xbcBxQ9vHNvseJ8n5wG8BL6+qnXv3732cpqruTnIz8ALggInHpjZJWt42AWuSnJRkFXAp8LjRaUleALwfuLiq/m1o/xFJDmvWjwJeCgwPStgnazyS1EN7Zy5Y9HWq9iS5gsEMNCuBa6rqjiRXAZuraj3wR8BTgI8lAbinqi4GngO8P8ksg4rMH8wZDbdPJh5J6qlJPUBaVRuADXP2XTm0fv5+zrsFeN5C45l4JKmPpnQetlEc/MSTbrqRsnJlJ3EBsvqwzmKvOLK7N1TsOv6ozmJ///ju/s0BnnrPzvk/dJCsuueBzmLPbn+4s9js6O7fvGY6C70kWOORpB4qFjSqbaqYeCSpp2xqkyS1ZlKj2rrgczySpFZZ45GknuprjcfEI0k9tMC52qaKiUeSeqqvo9rs45EktcoajyT1UdnHI0lqUZ+HU5t4JKmn+pp47OORJLVqrBpPkicDHwMOBx4BXjv8RjpJ0sHV5+HU49Z41gK3VtU5wBebbUlSi6oy0jJtxk08X2dQ2wF4OvDgZIojSRrVLBlpmTbjJp4twIuT3AGcAdwyuSJJkpaycRPPZcB1VXUq8Cng9cMHk1yUZN0edi+2fJKkfajmOZ5RlmkzbuIJsL1ZfwD4keGDVXVdVV1+CIcupmySpAPoax/PuM/xfAT4myRvAHYDvzy5IkmS5jedtZlRjJV4quph4IIJl0WStAw4c4Ek9dQ0NqONwsQjST3kXG2SpHbVYGRbHzlXmySpVdZ4JKmnpnFWglGYeCSphwoHF0iSWtXf53js45EktcoajyT1VF9HtZl4JKmn7OPZl0BWdPMPk5XdtSKuOPzJncXedfxRncX+5itXdxZ7yxvf11lsgDUffEtnsU/8VHc/81U7d3UWe3Z3h7Pf12w3cYfCVvU38djHI0lqlU1tktRTfR3VZuKRpJ5ycIEkqVX28UiSNAJrPJLUQ8V0vtZ6FCYeSeqpnnbxmHgkqZd8jkeSpNGYeCSpr2rEZR5J1ia5K8nWJO/ax/F3JLkzyVeT3JjkhKFjlyXZ0iyXjVJsE48k9VRVRloOJMlK4GrgQuAU4HVJTpnzsa8AZ1TV84GPA+9pzj0SeDdwFnAm8O4kR8xXbhOPJPXUYL62+Zd5nAlsraq7q2oXcC1wyePj1Geq6gfN5heAY5v1C4Drq2p7VT0EXA+snS/gWImnqZbd3CzfTvKqca4jSercMcC9Q9v3Nfv2503AP4x5LjDmqLaq+jTwaYAktwI3jHMdSdJ4Fvjq66OSbB7aXldV6xYaM8nrgTOAly/03GGLGk6d5GTgu1X174u5jiRpgQoYPfE8UFVn7OfYNuC4oe1jm32Pk+R84LeAl1fVzqFzz5lz7s3zFWaxfTy/BPztIq8hSRrDhPp4NgFrkpyUZBVwKbB++ANJXgC8H7i4qv5t6NBG4BVJjmgGFbyi2XdAi008F+2jgJc3w+ru3/HDvihJ0jSqqj3AFQwSxr8AH62qO5JcleTi5mN/BDwF+FiS25Ksb87dDvweg+S1Cbiq2XdAYze1JfkxYFdVPTjnJtYB6wCetuLIvs7oIEnTb0LfsFW1AdgwZ9+VQ+vnH+Dca4BrFhJvMX08lwCfXMT5kqSxLcNJQqvq/ZMsiCRpgXrapuQDpJKkVjk7tST1UY9npzbxSFJf9bSpzcQjSb3VzxqPfTySpFZZ45GkvrKpTZLUKhOPJKk1C5skdKrYxyNJapU1HknqqRFmnp5KSzfxrFzZXezVh3UW+vvHdxd7yxvf11nsC559emexAbZ8q7t7f9HX3txZ7Gfc293vW6d/43v2dBd7mIlHktQq+3gkSZqfNR5J6qnY1CZJak1hH48kqU2xj0eSpFFY45GkvrKpTZLUKhOPJKlVPU089vFIklpljUeS+sjZqaV+2nP4Km7/3QvZc/iqrosiLVhqtGXajJ14krwxyY1Jbk5yzCQLJbXl/pecyP1nn8z9Lz6x66JIC1cjLlNmrMTTJJqXV9V5VXVOVW2bcLmkVnxr7XMG/73wOR2XRFo+xu3juQBYmeRG4E7gv1fVzOSKJR0cX/qji3noZ4774XZ2DX5tHz71Wdxw01t/uP+IL93Lz/za+tbLJy0H4za1PRNYVVXnAT8ALplckaSD56QPf4kVO3b/cLtWrXzcfwFW7NjNSR/a3HrZpIVabn08jwCfbdZvAh7XTpHkoiTr9tTuJ5wodenI27Zx+m9+6nHJZ9iKHbs5/Tf+niNv/1bLJZPGUBltmTLjJp5bgOc366cD3xg+WFXXVdXlh+TQxZRNOiiOvG0bz7tqIyt2Pv4tkit27uF5V2006UgH2Vh9PFV1W5LHktwMPAC8d6Klkg6yPU85jMzMwswsK3bPMHvoSjIzy56ndPgqZ2khpnTE2ijGHk5dVe9sRrS9uqp2TbJQ0sH2rQufw8zqQ3nq3Q9y2m9v4Kl3P8jM6kMd3aZ+6elwamcu0LK08tFdrPnz/8vxn7idFBz5lo9xzy+dxkPPf1bXRZNGNo0DB0Zh4tGydPqV//C47cwWJ3z8Nk74+G0dlUhaPkw8ktRX1ngkSa0y8UiS2jKtD4eOwtmpJUmtssYjSX01hbMSjMLEI0l91dOmNhOPJPWUfTySpF5KsjbJXUm2JnnXPo7/bJIvJ9mT5NVzjs0kua1ZRnqXyNKt8cx0+HqgHTs7C/3Ue7qLveaDb+ks9pZvva+z2NDtvZ94z47OYnf5u97p3/i0mECNJ8lK4Grg54H7gE1J1lfVnUMfuwf4FeCd+7jEY1V1+kJiLt3EI0lL2eSGU58JbK2quwGSXMvgHWs/TDxV9c3m2OwkAtrUJkl9NZlJQo8B7h3avq/ZN6rVSTYn+UKSV41ygjUeSVr6jkoy/FrddVW1bkLXPqGqtiU5Gbgpydeq6usHOsHEI0l9NXpT2wNVdcZ+jm0DjhvaPrbZN1oRqrY1/727eUfbC4ADJh6b2iSpp/ZOmzPfMo9NwJokJyVZBVwKjDQ6LckRSQ5r1o8CXspQ39D+mHgkaRmrqj3AFcBG4F+Aj1bVHUmuSnIxQJIXJrkPeA3w/iR3NKc/B9ic5HbgM8AfzBkNt082tUnSMldVG4ANc/ZdObS+iUET3NzzbgGet9B4Jh5J6quezlxg4pGkPurxaxFMPJLUVz1NPA4ukCS1yhqPJPVVT2s8YyWeJCcCtzIYererql4xwTJJkuYRlmcfz/VV9fqJlUSStDA9TTyL6eM5N8nnkrx9YqWRJC154yaebwM/CZwLnJ/k+ZMrkiRpXiNOlzONzXFjJZ6q2llVjzZTLfw98Ny9x5JcnmRLkvt31A8mVU5J0lyTeS1C68ZKPEmeOrT5UoZmIq2qdVW1pqqOXp0nL7Z8kqT9WU6JBzg7yZeS3AJsq6pbJ1koSdLSNdaotn1NKCdJatc09t+MwgdIJamvTDySpNZMaf/NKJyrTZLUKms8ktRT9vFIktpl4pEktamvNR77eCRJrbLGI0l91dMaj4lHkvqox8OpTTyS1ENplj6yj0eS1KqDW+MpqNmO6oIzs93EBWYf7e51EKvueaCz2Cd+6qjOYr/oa2/uLDbAiffs6Cx2lz/zLn/Xq8O/8c6+1+aGtalNktSmvg6nNvFIUl/1NPHYxyNJapU1Hknqq57WeEw8ktRHZR+PJKltPU089vFIklpljUeSesqmNklSu0w8kqQ29bXGYx+PJKlVi0o8Sd6e5POTKowkaUS1gGXKjN3UluQw4PQJlkWStBBTmFRGsZgaz5uAD0yqIJKk0YVBH88oy7QZK/EkORQ4p6pumnB5JElL3LhNbW8APrK/g0kuAi56Ek8Z8/KSpHlNYW1mFOM2tf0U8JYknwZOTfK24YNVdV1VXX4Ihy66gJKkfUvVSMu0GSvxVNWvV9UFVbUWuKOq/mTC5ZIkHcgER7UlWZvkriRbk7xrH8d/NsmXk+xJ8uo5xy5LsqVZLhul6It+jqeqXrbYa0iSupFkJXA1cCFwCvC6JKfM+dg9wK8wp4slyZHAu4GzgDOBdyc5Yr6YPkAqST01oVFtZwJbq+ruqtoFXAtcMvyBqvpmVX0VmJ1z7gXA9VW1vaoeAq4H1s4X0MQjSX01maa2Y4B7h7bva/aNYqxznatNknpqAc/oHJVk89D2uqpaN/kSjcbEI0lL3wNVdcZ+jm0DjhvaPrbZN4ptwDlzzr15vpNsapOkvppMU9smYE2Sk5KsAi4F1o9Ygo3AK5Ic0QwqeEWz74BMPJLURyMOLJivOa6q9gBXMEgY/wJ8tKruSHJVkosBkrwwyX3Aa4D3J7mjOXc78HsMktcm4Kpm3wHZ1CZJfTWhZ0OragOwYc6+K4fWNzFoRtvXudcA1ywknjUeSVKrrPFIUg/tnZ26j0w8ktRXUzgP2yhMPJLUU9Z49qfmzrDQjprpJOzAjp2dhZ7d/nBnsVft3NVZ7Gfce1hnsYFuf+aP/qCz2NXhfddMh3/kHX2vLRXWeCSpj0aceXoamXgkqafS04qXiUeS+qqnNR6f45EktcoajyT1lKPaJEntKXyOR5LUrr7WeOzjkSS1yhqPJPVVT2s8Jh5J6iEnCZUktauqt4MLxurjSfLcJLck+VySv0ySSRdMkrQ0jTu44K6qeklVnd1snzGpAkmSRjOJV193YaymtqraPbS5E7h3MsWRJI1sCpPKKMYeTp3k4iT/DDwTeHByRZIkjaKvNZ6xE09Vra+q5wL3Ab+wd3+Sy5NsSXL/Drp7T4gkaTqNO7hg+K1b3wMe27tRVeuqak1VHb2aJy+2fJKkfSlgtkZbpsy4w6nXJnlHs74F+McJlUeSNKrpyykjGXdwwSeBT064LJKkBZjG/ptROFebJKlVzlwgSX3V05kLTDyS1FN9bWoz8UhSHxW9HVxgH48kqVXWeCSphwavRehnlcfEI0l9Ndt1AcZj4pGknuprjcc+HklSq6zxSFIf9XhUm4lHknqpv6++PviJp7N/mO563Wqms9CwY2dnoWd3757/QwfLypXdxQaY6e6HXjNd/q53+MteHfasT8kXfl8fILWPR5LUKpvaJKmvpqTmtVAmHknqo4L4HI8kqVU9rfHYxyNJapU1Hknqq35WeKzxSFJfpWqkZd7rJGuT3JVka5J37eP4YUn+pjl+a5ITm/0nJnksyW3N8uejlNsajyT11QT6eJKsBK4Gfh64D9iUZH1V3Tn0sTcBD1XVTyS5FPhD4JebY1+vqtMXEtMajyQtb2cCW6vq7qraBVwLXDLnM5cAH2jWPw6clyTjBjTxSFIfFYMJWkZZDuwY4N6h7fuaffv8TFXtAR4BntEcOynJV5J8NsnZoxTdpjZJ6qEwWv9N46gkm4e211XVugkU49vA8VX1YJKfAf4uyalV9b0DnTRW4klyFvBeBrl0U1W9fZzrSJIWYfTE80BVnbGfY9uA44a2j2327esz9yU5BPgR4MGqKmDnoCj1pSRfB34S2MwBjNvU9q/Az1XVy4AfTfK8Ma8jSerWJmBNkpOSrAIuBdbP+cx64LJm/dXATVVVSY5uBieQ5GRgDXD3fAHHqvFU1XeGNncDXc7HLEnL0wRGtVXVniRXABuBlcA1VXVHkquAzVW1HvgL4K+SbAW2M0hOAD8LXJVkN4MWsDdX1fb5Yi6qjyfJ84Gj5wy7kyQdbHsHF0ziUlUbgA1z9l05tL4DeM0+zvsE8ImFxhs78SQ5EvhT4LX7OHYRcNGTOHzcy0uS5rGAwQVTZaw+nqZz6UPAO+c0uwFQVddV1eWHsGqx5ZMkLTHjDi54DfBC4D1Jbk7y4gmWSZI0iqrRlikz7uCCvwb+esJlkSSNbDqTyih8gFSS+qjobeJxyhxJUqus8UhSX/nqa0lSm/o6nNrEI0l91dPEYx+PJKlV1ngkqY8KmO1njcfEI0m95HM8kqS29TTx2McjSWqVNR5J6que1niWbuLp9AfS3VNd1eUr+arDp9n27Okudseqyw7mLn/mPf3SnRgHF0iS2lXdJv5FsI9HktQqazyS1Fc9bW408UhSH9nHI0lqXU9rPPbxSJJaZY1HkvqqpzUeE48k9ZJztUmS2lTArM/xSJI0L2s8ktRXPW1qG6vGk+TZSb6cZEcSk5ckdaFqtGXKjJs0tgPnAX87wbJIkkZWy+sB0qraAexIMuHiSJKWOpvJJKmPCsrZqQeSXJ5kS5L7d/DopC8vSdprtkZbpszEE09VrauqNVV19GoOn/TlJUl79XRwwbij2g5NcgNwGrAxyVmTLZYkaakad3DBbuD8CZdFkjSqqt7OXODgAknqqylsRhuFiUeSeqp6WuNxrjZJUqus8UhSL03niLVRmHgkqY+KqXxGZxQmHknqK2cukCRpfiYeSeqhAmq2Rlrmk2RtkruSbE3yrn0cPyzJ3zTHb01y4tCx32j235XkglHKbuKRpD6qGjS1jbIcQJKVwNXAhcApwOuSnDLnY28CHqqqnwDeC/xhc+4pwKXAqcBa4M+a6x2QiUeSempCNZ4zga1VdXdV7QKuBS6Z85lLgA806x8HzsvgvTiXANdW1c6q+gawtbneAZl4JGl5Owa4d2j7vmbfPj9TVXuAR4BnjHjuExzUUW3f56EHbqiP/+uYpx8P3DPJ8izAjzD4hx3P4kY49ve+ZxYVu7/3vTjed/v6fN8n7F35Pg9tvGH2o0eNeN7qJJuHttdV1bpFlGNRDmriqaqjxz03yf1VdcYky7OA2Ouq6vKOYnvf7cf2vtuP7X0vUlWtncR1gG3AcUPbxzb79vWZ+5IcwiB5PzjiuU8wzU1tD3cY+7oOY3vf7fO+2+d9T49NwJokJyVZxWCwwPo5n1kPXNasvxq4qaqq2X9pM+rtJGAN8MX5Ak7zA6RdVYWpqi5/Mb3vlnnfnfC+p0RV7UlyBbARWAlcU1V3JLkK2FxV64G/AP4qyVZgO4PkRPO5jwJ3AnuAt1bVvI3u05x4Omt/7Jj3vbx438vLVN53VW0ANszZd+XQ+g7gNfs59/eB319IvFRPJ5mTJPXT1PXxJHl2ki8n2dF0YrUZ+6wktyT5fJL3thz7uU3szyX5y2aMfKuSvD3J51uOeWKS7ya5Ock/thm7if/GJDc28ecdBjrBuGubmDcn+XaSV7UY+8lJPtXE/mSSw1qMfUiSa5N8Jsl7Wor5hO+UJO9t/tb+uM3YXX6/TZOpSzwM2g/PA77QQex/BX6uql4G/GiS57UY+66qeklVnd1stzrip/nyOb3NmEOur6pzquoVbQZtEs3Lq+q8Jv68o3Empao+3cQ8h8Hw2hvais3gCfNbm9hfbLbb8ovA7VV1LvCkJKe1EPNx3ylJfhp4SvO3tirJC9uKvY/tZWnqEk9V7aiqhzqK/Z2mLRNgN4t9MmVhsXcPbe7k8Q9lteFN/MeTyW07t/m/z7e3HPcCYGVT4/mTUab6mLQkJwPfrap/bzHs14HDm/WnMxgW25aTga8267cBLznYAffxnfIi4Ppm/QbgxW3F7vL7bZpMXeKZBkmeDxxdVXe2HPfiJP8MPJMWvwySHAqcU1U3tRVzyLeBnwTOBc5v/u3b8kxgVVWdB/yAJ04T0oZfAv625ZhbgBcnuYNBzfqWFmPfBby8WT+XQeJr29OB7zXrj3RUhmXNxDNHkiOBP2VQA2hVVa2vqucymHbiF1oM/QbgIy3G+6FmjqdHm2k4/h54bovhHwE+26zfBDynxdh7XcQTn5k42C4DrquqU4EqYaBVAAABTklEQVRPAa9vMfZ1DJrYbmRQs/9ui7H3egR4WrP+NKbz2ZolzcQzpOns+xDwzqr6Tsuxhzt4vwc81mL4nwLekuTTwKlJ3tZW4CRPHdp8KYNmoLbcAuytYZ0OfKPF2CT5MWBXVbXZ1AUQBn0NAA8weAq9FVU1U1Vva2qZMwyeHWnbPzHoZwE4n2Xe39KFqUs8SQ5NcgNwGrAxyVkthn8N8ELgPc2In4PW9rsPa5N8NslnGTQBtTbCq6p+vaouaKbguKOq/qSt2MDZSb6U5BZgW1Xd2lbgqroNeCzJzQx+7h9vK3bjEuCTLceEQe32tc19/xfgw20FTnJM87d1E3BLGwM65n6nAIcCO5J8DpipqnmftJ9U7GbkbFffb1PD53gkSa2auhqPJGlpM/FIklpl4pEktcrEI0lqlYlHktQqE48kqVUmHklSq0w8kqRW/X9IISVkGTqRuAAAAABJRU5ErkJggg==\n",
"text/plain": [
"<Figure size 1008x576 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"def get_value_grid(V_array, nvmdp, s_to_idx):\n",
" \"\"\"Returns value over states space grid.\n",
" \"\"\"\n",
" V = np.zeros((nvmdp.height, nvmdp.width), dtype=np.float32)\n",
" for row in range(nvmdp.height):\n",
" for col in range(nvmdp.width):\n",
" x, y = nvmdp._rowcol_to_xy(row, col)\n",
" V[row, col] = V_array[s_to_idx[NavigationWorldState(x, y)]]\n",
" return V\n",
" \n",
"## Greedy Policy\n",
"# v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, lambda Q: np.max(Q), 100)\n",
"## Boltzmann Policy\n",
"boltz_temp = 100\n",
"v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, \n",
" lambda Q, t=boltz_temp: np.exp((1/t)*Q) / (np.exp((1/t)*Q).sum()), 100)\n",
"V = get_value_grid(v, nvmdp, s_to_idx)\n",
"\n",
"plt.figure(figsize=(14,8))\n",
"nvmdp.visualize_grid(V, trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0.01, 0.01, 0.01, 0.01, 0.01, 0. , 0. , 0. ],\n",
" [0. , 0. , 0.01, 0.01, 0.02, 0.03, 0.02, 0.01, 0.01, 0. , 0. ],\n",
" [0. , 0.01, 0.01, 0.03, 0.06, 0.1 , 0.06, 0.03, 0.01, 0.01, 0. ],\n",
" [0. , 0.01, 0.02, 0.06, 0.16, 0.33, 0.16, 0.06, 0.02, 0.01, 0. ],\n",
" [0. , 0.01, 0.03, 0.1 , 0.33, 0. , 0.33, 0.1 , 0.03, 0.01, 0. ],\n",
" [0. , 0.01, 0.02, 0.06, 0.16, 0.33, 0.16, 0.06, 0.02, 0.01, 0. ],\n",
" [0. , 0.01, 0.01, 0.03, 0.06, 0.1 , 0.06, 0.03, 0.01, 0.01, 0. ],\n",
" [0. , 0. , 0.01, 0.01, 0.02, 0.03, 0.02, 0.01, 0.01, 0. , 0. ],\n",
" [0. , 0. , 0. , 0.01, 0.01, 0.01, 0.01, 0.01, 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ]],\n",
" dtype=float32)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V.round(2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Low Temperature"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAGCCAYAAAAsfY4lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHsFJREFUeJzt3X20XXV95/H3J08kEjGlREUeBFvsElEYi+ATSyi0hLaA7aijHbWd5SprnOqasWOnttMyLeM/tWvV6bSutpnVB61PtXa1hpoaRcRqKQhaQKHDkFIrIGp4CDSaS5J7v/PHOcHLNck9Odnn7L3vfb9ce7n3Pvvu7++Q3PPN9/f77d9JVSFJUpNWtN0ASdLSY3KRJDXO5CJJapzJRZLUOJOLJKlxJhdJUuNMLlq2kpySpJKsarst0lJjclGvJfl4kqsOcP7yJF83cUjtMLmo794DvC5JFpx/PfD+qtrXQpukZc/kor77K+B7gfP2n0jyPcCPA+9N8mNJ/iHJo0nuSfJrB7tRkq8kuWje8a8led+84xcluT7JziS3Jjl/Em9IWgpMLuq1qtoNfBh4w7zTrwb+b1XdCnxr+NoG4MeANyV5xeHGSXIC8DHgHcCxwNuAv0iy8cjegbQ0mVy0FLwHeGWStcPjNwzPUVXXVdWXqmquqm4DPgi8fIwYrwO2VtXW4b0+CdwM/GgD7ZeWHJOLeq+qPgc8ALwiyfcB5wAfAEhybpJPJ9mR5BHgPwLHjRHmmcCrhl1iO5PsBF4GHN/Mu5CWFmfSaKl4L4OK5QeAbVX1jeH5DwC/C1xSVTNJ/hcHTy7fAp407/jp8/bvAf60qn622WZLS5OVi5aK9wIXAT/LsEts6MnAQ8PEcg7wU4e4xy3Aa5KsTnI28Mp5r70PuDTJxUlWJlmb5PwkJzb8PqQlweSiJaGqvgJcDxwNbJn30n8Crkryr8CVDAb/D+ZXge8DHgZ+nWHX2vD+9wCXA78M7GBQyfwC/g5JBxS/LEyS1DT/1SVJapzJRZKWuSR/lOSbSb58kNeT5H8n2Z7ktiQvWOyeJhdJ0p8Amw7x+iXAacPtCuD3FruhyUWSlrmq+lvgoUNccjnw3hq4AdiQ5JDPeJlcJEmLOYHBDMn97h2eO6iJPkS5ZtWTat2aDZMMcVC1YuEiucsk9qr2Ys+tbC001fLjwGlx7eUVs+3Fzr72ZptmbvnF3r1nJ3v2fTsAF19wdD340Gh/+F+47bHbgZl5pzZX1eYJNPFxE/2VXLdmAy/6/jdOMsRBza1f00pcgL3rV7cWe8+G9j5lZza0Vwjv3theUgVYt6O9D7q1O+dai71mZ3tZdfWuva3FXrFrTytxb9j+h4/vP/jQLJ/fdvJIP7fy+LtmqursIwh9H3DSvOMTh+cOym4xSeqhAuZG/F8DtgBvGM4aexHwSFXdf6gfcG0xSeqlYraaqVqTfBA4Hzguyb3A/wBWA1TV7wNbGawAvh34NvAfFrunyUWSlrmqeu0irxfwc4dzT5OLJPXQoFusu8t3mVwkqacaGk+ZCJOLJPVQUcx2eOFhZ4tJkhpn5SJJPeWYiySpUQXMmlwkSU3rcuWy6JhLkmck+WKSmSSrFh5Po5GSpH4ZZUD/IeBC4IaDHEuSpqyA2aqRtjYsWnlU1Qwwk+SAx5KkdnT3KRfHXCSpl4rq9IB+48+5JLkiyV1Jduze80jTt5ck9UDjyaWqNlfVaVW1cd2apzR9e0kSQMHsiFsbRpkttjrJNcCZwLYk5y48nnQjJUlPNFi4crStDaMM6O8FLlpweuGxJGmqwizdnVjl2mKSpMY5W0ySeqiAue5OFjO5SFJfdblbzOQiST00WLiyu8nFMRdJUuOsXCSpp+aqu5WLyUWSeqjr3WImF0nqoSLMdnhko7stkyT1lpWLJPXUsh1zqRVhbv2aSYY4qL3rV7cSF2DPhvZy9syG9orR3Rvb+4u++/jZ1mIPrGwxdpsdEMvz36dtfbrUiu/8jjnmIkmagDBb3R3Z6G7LJEm9ZeUiST00WHK/u/WByUWSesoxF0lSo6occ5EkLTNWLpLUU3N2i0mSmjR4zqW7nU8mF0nqJcdcJEnLjJWLJPWQz7lIkiZitsMLVy6a9pI8I8kXk8wkWTU8964kn03y25NvoiRpof3f5zLK1oZRoj4EXAjcAJDkBcD6qjoPWJPkhRNsnySphxbtFquqGWAmebz8ehHwyeH+NcCLgZsm0jpJ0kHNLbHZYhuAR4f7jwyPJUlTtP85l652i40zoP8IcMxw/xhg58ILklwKXLruqGOPoGmSpIMp0u8B/QP4ewZjMAAXMRyLma+qrq6qK1atWnskbZMk9dQos8VWJ7kGOBPYxuAbPmeSfBaYrarPT7iNkqQDmGPFSFsbRhnQ38ugQpnvxsk0R5I0iio6vfyLD1FKUi+l06sidzftSZJ6y8pFknqosFtMkjQBfp+LJKlRRZhbYs+5SJJ0SFYuktRTdotJkhpVdHvhSpOLJPVSmPU5F0nScjLRyqVWhL3rV08yxEHt2dBeUTazob2cvXtje/+S2X38bGux15/86OIXTdCuxxcKb8PKFmO3+e/T5dfxUiu+8/ttt5gkaSK63C1mcpGkHqpKpyuX7rZMkjQVSTYluTPJ9iRvP8DrJyf5dJJ/SHJbkh9d7J5WLpLUU02sLZZkJfBu4IeBe4GbkmypqjvmXfYrwIer6veSnA5sBU451H2tXCSphwqYGy67v9i2iHOA7VV1d1XtAT4EXH6AcPtnrTwF+NpiN7VykaReSlOrIp8A3DPv+F7g3AXX/BrwiSRvAY7mu79A8rtYuUjS0ndckpvnbVcc5s+/FviTqjoR+FHgT5McMn9YuUhSDw2ecxl5KvIDVXX2QV67Dzhp3vGJw3PzvRHYBFBVf59kLXAc8M2DBbRykaSemmXFSNsibgJOS3JqkjXAa4AtC675KnAhQJLnAGuBHYe6qZWLJPVQU9/nUlX7krwZ2MZguYc/qqrbk1wF3FxVW4D/CvyfJG9lUDT9TFXVoe5rcpGkZa6qtjKYXjz/3JXz9u8AXno49zS5SFJPzXV4ZGOs5JJkFfA+4GnATVX13xptlSTpkKpgdgl+zfFPALdW1QXAuiRnNtgmSdII5iojbW0YN7k8C7htuH8L8JJmmiNJWgrGTS53Ai8f7l8AbGimOZKkUQxmi60YaWvDuFGvZtAd9ingMeAb+19IckWSu5LseGxmZxNtlCQdwOzwq44X29owVnKpqtmqektVXQjMMpgfvf+1zVV1WlVtPGqtBY0kTcL+J/S7OuYy7myxE4D3A3PAe6tq4VIBkqRlbKzkMkwm5zfbFEnS6Lr9TZQ+RClJPTXCd7W0xuQiST20VB+ilCTpoKxcJKmnHHORJDWqqSX3J8XkIkk91eUB/e7WVJKk3rJykaQe2v+EfleZXCSppxzQlyQ1q8V1w0Yx0eRSq8KeDe3kr5kN7WX03Rvb+wPfffxsa7HXn/xoa7EvO+XLrcUG2MIZrcXexTGtxYaVLcZu81/t7Xyu1aruJpOFrFwkqYeKbs8WM7lIUk8t224xSdJkdH22WHenGkiSesvKRZJ6qsuVi8lFknrItcUkSRPR5dlijrlIkhpn5SJJfVSOuUiSGtb1qcgmF0nqqS4nF8dcJEmNG6tySfIk4M+Bo4FHgFdX1WNNNkySdHBdn4o8buWyCbixqs4HPj88liRNUVVG2towbnL5JwZVC8AG4MFmmiNJGtUcGWlrw7jJ5S7gxUluB84Grm+uSZKkvhs3ufw0cHVVPRf4GPC6+S8muTTJ5n17dh9p+yRJB1DD51xG2dowbnIJ8NBw/wHgKfNfrKqrq+qKVWvWHUnbJEmH0OUxl3Gfc/kA8GdJXg/sBf5dc02SJC2u27PFxkouVbUTuLjhtkiSlgif0Jeknmqry2sUJhdJ6iHXFpMkNa8GM8a6yrXFJEmNs3KRpJ7q8jdRmlwkqYcKB/QlSY3r9nMujrlIkhpn5SJJPdXl2WImF0nqqWU75jK3EmY2tNPztntje//Rdx8/21rs9Sc/2lrsy075cmux3/HUL7UWu21bOKO12Ls4prXYsLLF2O18rs3Ne8tV3U4ujrlIkhpnt5gk9VSXZ4uZXCSppxzQlyQ1zjEXSdKyYuUiST1UtPcVxqMwuUhST3V4yMXkIkm95HMukqTlxuQiSX1VI26LSLIpyZ1Jtid5+0GueXWSO5LcnuQDi93TbjFJ6qkmusWSrATeDfwwcC9wU5ItVXXHvGtOA34JeGlVPZzkqYvd18pFknpqsL7Y4tsizgG2V9XdVbUH+BBw+YJrfhZ4d1U9PIhb31zspmMll2EJdd1wuz/JK8a5jySpdScA98w7vnd4br5nA89O8ndJbkiyabGbjtUtVlUfBz4OkORG4Jpx7iNJGs9hfs3xcUlunne8uao2H0a4VcBpwPnAicDfJnleVe081A+MLcmzgG9U1a4juY8k6TAVMHpyeaCqzj7Ia/cBJ807PnF4br57gRurai/wz0n+H4Nkc9PBAh7pmMtPAn95hPeQJI2hoTGXm4DTkpyaZA3wGmDLgmv+ikHVQpLjGHST3X2omx5pcrl0YSOSXJHkriQ79u56+AhvL0mapKraB7wZ2Ab8I/Dhqro9yVVJLhtetg14MMkdwKeBX6iqBw9137G7xZI8HdizMMCwH28zwJM2ntTl1Qkkqd8a+oStqq3A1gXnrpy3X8DPD7eRHMmYy+XAR4/g5yVJY1uiC1dW1R802RBJ0mHqcN+QD1FKkhrn8i+S1EcdXxXZ5CJJfdXhbjGTiyT1VncrF8dcJEmNs3KRpL6yW0yS1DiTiySpUYe3cOXUOeYiSWqclYsk9dQIKx63ZqLJpVbB7o3tlG27j59tJS7A+pMfbS32Zad8ubXY73jql1qLffEzzmotNsC2r93Savy2bOGM1mLv4pjWYsPKVqLWwk/s5ZpcJEkT5JiLJGk5sXKRpJ6K3WKSpEYVjrlIkpoWx1wkScuLlYsk9ZXdYpKkxplcJEmN63ByccxFktQ4KxdJ6iNXRZa6a9/Ra7j11y9h39Fr2m6KdNhSo21tGDu5JHlDkk8luS7JCU02SpqWHS85hR3nPYsdLz6l7aZIh69G3FowVnIZJpOXV9WFVXV+Vd3XcLukqfjapucM/v+S57TcEmlpGXfM5WJgZZJPAXcA/6Wq2lvjXhrRF37zMh7+wZMeP86ewV/bnc89nmuu/bnHz3/PF+7hB39hy9TbJy0V43aLPQ1YU1UXAt8GLm+uSdLknPr+L7BiZu/jx7Vm5RP+H2DFzF5Ofd/NU2+bdLiW4pjLI8BnhvvXAk/oU0hyaZLNs4/NHEnbpMYde8t9nPXLH3tCgplvxcxezvqlv+bYW7825ZZJY6iMtrVg3ORyPfD84f5ZwD/Pf7Gqrq6qK1YetfZI2iZNxLG33MfzrtrGisf2PeH8isf28byrtplYpAaMNeZSVbck2Z3kOuAB4F2NtkqasH3rjyKzczA7x4q9s8ytXklm59i3/qi2myaNpuNL7o89Fbmq3jacKfbKqtrTZKOkSfvaJc9hdu1qnnz3g5z5q1t58t0PMrt2tbPG1C8dnorsE/pallZ+aw+n/f7fcfJf3EoKjn3Tn/PVnzyTh59/fNtNk0bmN1FKHXPWlX/zhOPMFc/8yC088yO3tNQiaWkxuUhSX1m5SJIaZ3KRJDWpzQckR+GqyJKkxlm5SFJfdfj7XEwuktRXHe4WM7lIUk855iJJWlYmWrlkH6zb0VZqXbn4JROyi2Nai72FM1qL3aZtX2v34cdf+ebzWou95Svt/Znv+mp7f9fX3d/e73hbn2vZt+BEhysXu8UkqY86PhXZ5CJJfdXh5OKYiySpcVYuktRXHa5cTC6S1FNdHnOxW0yS1DiTiySpcXaLSVJfdbhbzOQiSX3kcy6SpInocHJxzEWS1DgrF0nqqw5XLmMllySnADcC/wjsqaofabBNkqRFhKU75vLJqnpdYy2RJB2eDieXIxlzuSDJZ5O8tbHWSJKWhHGTy/3As4ELgIuSPL+5JkmSFjWcijzKtpgkm5LcmWR7krcf4rp/m6SSnL3YPcdKLlX1WFV9q6r2AX8N3/mGqiRXJLkryY69ux4e5/aSpFHUiNshJFkJvBu4BDgdeG2S0w9w3ZOB/8xgvH1RYyWXYZD9Xgr80/6DqtpcVadV1cbV679nnNtLkkbRQHIBzgG2V9XdVbUH+BBw+QGu+5/AbwAzozRt3G6x85J8Icn1wH1VNVImkyR1zgnAPfOO7x2ee1ySFwAnVdXHRr3pWLPFqmorsHWcn5UkNeMwpiIfl+Tmecebq2rzSDGSFcBvAT9zOG3zIUpJ6qvRk8sDVXWwQfj7gJPmHZ84PLffkxmMq1+XBODpwJYkl1XV/IT1BCYXSeqj0cZTRnETcFqSUxkkldcAP/V4mKpHgOP2Hye5DnjboRILuLaYJC1rw1m/bwa2MVh15cNVdXuSq5JcNu59rVwkqaeaWv7lQOPoVXXlQa49f5R7mlwkqa86vPyLyUWSeqrLC1c65iJJapyViyT1VYcrF5OLJPVRc1ORJ8LkIkk9lOHWVY65SJIaN9HKZcUsrN05N8kQh4reUlyAla1F3sUxrcXe8p1vXlh2tnylvfe+66vt/Zmvu7+9v+vrdrTXJ9TW59qK2QUn7BaTJDWty1ORTS6S1FcdTi6OuUiSGmflIkl91eHKxeQiSX1UjrlIkiahw8nFMRdJUuOsXCSpp+wWkyQ1z+QiSWpalysXx1wkSY07ouSS5K1JPtdUYyRJI6rD2FowdrdYkqOAsxpsiyTpcCzRbrE3Au9pqiGSpNGFwZjLKFsbxkouSVYD51fVtQ23R5K0BIzbLfZ64AMHezHJpcClRx39vWPeXpK0qCXYLfYDwJuSfBx4bpK3zH+xqq6uqitWrVl3xA2UJB1Yqkba2jBW5VJVv7h/P8nnqup3mmuSJGlRLc4EG8URP+dSVS9roiGSpKXDJ/Qlqae6/IS+yUWS+srkIklqWpcrF9cWkyQ1zspFkvqqw5WLyUWS+qjFpV1GYXKRpL7qcHJxzEWS1DgrF0nqof2rIneVyUWS+qqldcNGYXKRpJ5atpVL9hVrdu6bZIhDaDNvtjmUtbK1yLs4prXYWzijtdgAu77a3ntfd397f+brdrT36bZ251xrsdv6XMu+DmeTBaxcJKmPOr4qsslFknoq7RVvizK5SFJfdbhy8TkXSVLjrFwkqaeW7WwxSdKEFD7nIklqXpcrF8dcJEmNs3KRpL7qcOVicpGkHnLhSklS86o6PaA/1phLkjOSXJ/ks0n+OEmabpgkqb/GHdC/s6peUlXnDY/PbqpBkqTRpEbb2jBWt1hV7Z13+BhwTzPNkSSNrLu9YuNPRU5yWZIvA08DHmyuSZKkUXS5chk7uVTVlqo6A7gX+PH955NckeSuJDsem9nZRBslST0z7oD+UfMOHwV27z+oqs1VdVpVbTxq7YYjbZ8k6UAKmKvRthaMOxV5U5KfH+7fBXyiofZIkkbV4TGXcQf0Pwp8tOG2SJIOQ5cfonRtMUlS40wuktRX+5/SX2xbRJJNSe5Msj3J2w/w+s8nuSPJbUk+leSZi93T5CJJPdXEVOQkK4F3A5cApwOvTXL6gsv+ATi7qp4PfAR452JtM7lIUh/VYWyHdg6wvarurqo9wIeAy58QqurTVfXt4eENwImL3dTkIknL2wk8cZWVe4fnDuaNwN8sdlNXRZakHhosuT/ydLHjktw873hzVW0+7JjJ6xisJfnyxa41uUhSX82NfOUDVXWwBYbvA06ad3zi8NwTJLkI+O/Ay6vqscUCmlwkqacOo3I5lJuA05KcyiCpvAb4qSfESf4N8AfApqr65ig3dcxFkpaxqtoHvBnYBvwj8OGquj3JVUkuG172m8B64M+T3JJky2L3tXKRpD4abSbYaLeq2gpsXXDuynn7Fx3uPU0uktRL3f6a44kml8wVq3ftXfzCJafNnN1mT+fK1iLv4pjWYgOsu7+9975uR3sfMGt3jj6i3LQ1O/e1Frutz7UsWOHYtcUkScuK3WKS1FfLtVtMkjQhBWmvV3JRJhdJ6qsOVy6OuUiSGmflIkl91d3CxeQiSX3V0PIvE2FykaS+6nByccxFktQ4KxdJ6qPicJbcnzqTiyT1UKhOj7mM1S2W5Nwk1yf5XJJ3Nd0oSdIIqkbbWjDumMu/AD9UVS8DnprkeQ22SZLUc2N1i1XV1+cd7gVmm2mOJGlkHe4WO6IxlyTPBzZW1R0NtUeSNIqlOqCf5Fjgd4FXH+C1S4FL1x117BE0TZJ0KEtxQH8V8D7gbQu6yACoqqur6opVq9YeafskST007oD+q4AXAu9Mcl2SFzfYJknSKDo8W2zcAf0PAh9suC2SpJG1lzhG4UOUktRHRaeTi2uLSZIaZ+UiSX21FKciS5La1eWpyCYXSeqrDicXx1wkSY2zcpGkPipgrruVi8lFknrJ51wkSZPQ4eTimIskqXFWLpLUVx2uXCaaXDJXrNi1Z5IhDmp1K1G7oM1/L7RZCK9sMTas29HeL/nane09Sbdm577WYq/etbe12G19rmX+AL4D+pKk5hVUdx/Rd8xFktQ4KxdJ6qvlOuYiSZoQx1wkSRPR4crFMRdJUuOsXCSprzpcuZhcJKmXXFtMktS0AuZ8zkWStIxYuUhSX3W4W2ysyiXJM5J8MclMEhOUJLWharStBeMmhoeAC4G/bLAtkqSR1dJ7iLKqZoCZJA03R5K0FNilJUl9VFDLaVXkJFckuSvJjt17Hmn69pKk/eZqtK0FjSeXqtpcVadV1cZ1a57S9O0lSft1eEB/3Nliq5NcA5wJbEtybrPNkiT12bgD+nuBixpuiyRpVFWdfkLfAX1J6qsOP0RpcpGknqoOVy6uLSZJapyViyT1kkvuS5KaViy95V8kSR2wnJ7QlyTJ5CJJPVRAzdVI22KSbEpyZ5LtSd5+gNePSvJnw9dvTHLKYvc0uUhSH1UNusVG2Q4hyUrg3cAlwOnAa5OcvuCyNwIPV9X3A+8CfmOx5plcJKmnGqpczgG2V9XdVbUH+BBw+YJrLgfeM9z/CHBhFvnOFZOLJC1vJwD3zDu+d3jugNdU1T7gEeB7D3XTic4We3T3/Q984kvv+Jcxf/xk4KtNtucwPIXBf7w2+L6nz/c9fb7v8Txz/86/8vC2a+Y+fNyIP7c2yc3zjjdX1eYjaMeiJppcqmrjuD+bZEdVnd1kew4j9uaquqKl2L7v6cf2fU8/tu/7CFXVpibuA9wHnDTv+MThuQNdc2+SVQwS9IOHummXu8V2thj76hZj+76nz/c9fb7v7rgJOC3JqUnWAK8Btiy4Zgvw08P9VwLXVh16eYAuP0TZ2tdYVlWbf/l831Pm+26F77sjqmpfkjcD24CVwB9V1e1JrgJurqotwB8Cf5pkO/AQgwR0SF1OLhPtD+ww3/fy4vteXjr5vqtqK7B1wbkr5+3PAK86nHtmkcpGkqTD1rkxlyTPSPLFJDPDgaNpxj43yfVJPpfkXVOOfcYw9meT/PFic8gn1Ia3JvnclGOekuQbSa5L8olpxh7Gf0OSTw3jL5x+Ocm4m4Yxr0tyf5JXTDH2k5J8bBj7o0mOmmLsVUk+lOTTSd45pZjf9ZmS5F3D37XfnmbsNj/fpq1zyYVBf96FwA0txP4X4Ieq6mXAU5M8b4qx76yql1TVecPjqc6kGX7AnDXNmPN8sqrOr6ofmWbQYTJ5eVVdOIy/cIbMxFTVx4cxz2cwNfWaacUGNgE3DmN/fng8LT8B3FpVFwDrkpw5hZhP+ExJ8gJg/fB3bU2SF04r9gGOl6zOJZeqmqmqh1uK/fVh3yLAXmB2irH3zjt8jCc+1DQNb+Q7T+BO2wXDf0W+dcpxLwZWDiuX3xkugzFVSZ4FfKOqdk0x7D8BRw/3N7DIlNKGPQu4bbh/C/CSSQc8wGfKi4BPDvevAV48rdhtfr5NW+eSSxckeT6wsarumHLcy5J8GXgaU/yFT7IaOL+qrp1WzHnuB54NXABcNPxvPy1PA9ZU1YXAt/nuJS+m4SeBv5xyzLuAFye5nUGFfP0UY98JvHy4fwGD5DZtG4BHh/uPtNSGJc/kskCSY4HfZfAv+amqqi1VdQaD5Rd+fIqhXw98YIrxHldVj1XVt4ZLSvw1cMYUwz8CfGa4fy3wnCnG3u9SvvuZgkn7aeDqqnou8DHgdVOMfTWD7rBPMajQvzHF2Ps9Ahwz3D+Gbj570nsml3mGA2zvA95WVV+fcuz5g6qPArunGP4HgDcl+Tjw3CRvmVbgJE+ed/hSBl0203I9sL9SOgv45ynGJsnTgT1VNc1uKYAw6PsHeIDB09ZTUVWzVfWWYbU4y+DZimn7ewbjHgAXsQzGP9rQueSSZHWSa4AzgW1Jzp1i+FcBLwTeOZxJM7G+2APYlOQzST7DoLtmajOnquoXq+ri4XISt1fV70wrNnBeki8kuR64r6punFbgqroF2J3kOgZ/7h+ZVuyhy4GPTjkmDKrUVw/f978H3j+twElOGP5uXQtcP41JFAs/U4DVwEySzwKzVfX5acUezkht6/NtqnzORZLUuM5VLpKk/jO5SJIaZ3KRJDXO5CJJapzJRZLUOJOLJKlxJhdJUuNMLpKkxv1/OAhfE3QHtDAAAAAASUVORK5CYII=\n",
"text/plain": [
"<Figure size 1008x576 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"boltz_temp = 0.01\n",
"v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, lambda Q, t=boltz_temp: np.exp((1/t)*Q) / (np.exp((1/t)*Q).sum()), 100)\n",
"V = get_value_grid(v, nvmdp, s_to_idx)\n",
"\n",
"plt.figure(figsize=(14,8))\n",
"nvmdp.visualize_grid(V, trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0.13, 0.17, 0.21, 0.26, 0.33, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13],\n",
" [0.17, 0.21, 0.26, 0.33, 0.41, 0.51, 0.41, 0.33, 0.26, 0.21, 0.17],\n",
" [0.21, 0.26, 0.33, 0.41, 0.51, 0.64, 0.51, 0.41, 0.33, 0.26, 0.21],\n",
" [0.26, 0.33, 0.41, 0.51, 0.64, 0.8 , 0.64, 0.51, 0.41, 0.33, 0.26],\n",
" [0.33, 0.41, 0.51, 0.64, 0.8 , 1. , 0.8 , 0.64, 0.51, 0.41, 0.33],\n",
" [0.41, 0.51, 0.64, 0.8 , 1. , 0. , 1. , 0.8 , 0.64, 0.51, 0.41],\n",
" [0.33, 0.41, 0.51, 0.64, 0.8 , 1. , 0.8 , 0.64, 0.51, 0.41, 0.33],\n",
" [0.26, 0.33, 0.41, 0.51, 0.64, 0.8 , 0.64, 0.51, 0.41, 0.33, 0.26],\n",
" [0.21, 0.26, 0.33, 0.41, 0.51, 0.64, 0.51, 0.41, 0.33, 0.26, 0.21],\n",
" [0.17, 0.21, 0.26, 0.33, 0.41, 0.51, 0.41, 0.33, 0.26, 0.21, 0.17],\n",
" [0.13, 0.17, 0.21, 0.26, 0.33, 0.41, 0.33, 0.26, 0.21, 0.17, 0.13]],\n",
" dtype=float32)"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V.round(2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "irl_python3",
"language": "python",
"name": "irl_python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Warning: OpenAI gym not installed.\n"
]
}
],
"source": [
"# Other Imports.\n",
"from simple_rl.tasks import NavigationWorldMDP\n",
"from simple_rl.agents import QLearningAgent\n",
"from simple_rl.planning import ValueIteration\n",
"from simple_rl.tasks.grid_world.GridWorldStateClass import GridWorldState\n",
"from simple_rl.tasks.navigation.NavigationStateClass import NavigationWorldState\n",
"\n",
"# Python Imports.\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"\n",
"%matplotlib inline\n",
"%load_ext autoreload\n",
"%autoreload 2"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"def value_iter(S, A, R, T, gamma, policy, n_iters):\n",
" \n",
" nS, nA = len(S), len(A)\n",
" V = np.zeros((nS))\n",
" Q = np.zeros((nS, nA))\n",
" Pi = np.ones((nS, nA)) / nA\n",
" s_to_idx = {s: idx for idx, s in enumerate(S)}\n",
" \n",
" for _iter in range(n_iters):\n",
" for si, s in enumerate(S):\n",
" \n",
" if s.is_terminal():\n",
" continue\n",
" for ai, a in enumerate(A):\n",
" s_prime_idx = s_to_idx[T(s,a)]\n",
" Q[si,ai] = R(s,a) + gamma * V[s_prime_idx]\n",
" V[si] = policy(Q[si,:]).dot(Q[si,:])\n",
" return V"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAFyCAYAAAApoxulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAFGBJREFUeJzt3XuMnPV97/H3B0yg4bKG4pBCu7GoaJSG20mJUpO4QECBtDVJ2oZecmmltJaiU3qUc2iTHvWS9o9Gourh5CQ6Va1KVRKgpEmTggPiFmIEcgKFiLShRxyC2iwhmGNzWXLBjjHf88c8i4etzZpnZ2d+9bxfEvLMPLPz+/rB+95nn5nZTVUhSWrLIZMeQJL07xlnSWqQcZakBhlnSWqQcZakBhlnSWqQcVYTkrwryc0r9NizSb6b5NCVeHxpJRhnHbAk/5bk/yU5cui230yyZbmPXVVXVdVblvs48PycFww99lxVHVVVe0bx+IvWqm6frBq67bDuthq6bUuSnUm+k+TpJPcm+VCSw4fu8+Eku7svJE8l2Zpk3ahn1n8Mxlkv1aHAf5n0EI15Enjr0PW3drct9ttVdTTwI8B/A34FuCFJhu7z6ao6ClgD3Al8btF2TQnjrJfqz4HLkqze18YkH03y8NDR4fru9hOTPJPkuKH7/qckO7ojzd9IcufQtrckeSDJfJL/neT2JL/ZbfvxJLclebz7+KsW5knyKWAW2Nwdgf5ekrXdEe6qoVmuS/JEkm8k+a2hdT+c5O+SfLI7yr0/yVlL7JNPAe8duv5e4JP7u3NVfa+qtgAXA+uAn9vHfXYDnwBeCfzwEuvrIGSc9VLdA2wBLtvP9n8EzgSOA64GPpPkiKr6NvBl4BeH7vtrwGe7ED0vyfHAZ4HfZxCmB4Czh+8CfAQ4EXgN8GPAhwGq6j3AHLChO5Vx+T5mvAb4VvfxvwT8WZI3D22/uLvPauA64OP7+bsu+AfgZ5KsTnIssB64domPoarmGOzP9Yu3dac7fgN4uKp2LPVYOvgYZ/XxR8ClSdYs3lBVV1bV41X1bFX9BXA48Opu89XArwJ036r/SnfbYj8L3F9Vn6uqZ4H/BWwbWuMbVXVLVe2qqu3A/wDOOZDBk/wY8Ebgg1W1s6ruA/6aFx753llVN3TnqD8FnLHEw+4ENgO/3P13XXfbgfg2gy9kCy5J8hTwMPBTwDsO8HF0kDHOesmq6uvAF4APLd6W5LIk/6c7HfEUMAMc323+e2Bdkh8BfgZ4DrhjH0ucyCBOC+sVgyPdhTVOSHJNkkeSPA1cObTGUk4Enqiq7wzd9k3gpKHr24Yufx84YvgJv/34JIPAv+gpjX04CXhi6PrfVdXqqnpFVb25qu59CY+lg4hxVl9/DPwWQ1Hrzi//HnAJcGxVrQbmGZyGoKqeBG5mcHT5a8A1te8fi/go8KNDj5vh68CfAQWcVlXHAO9eWKPzYj9q8dvAcUmOHrptFnjkxf6yB+AOBk/0ncDgibwldUfxP8W+v0Bpyhln9VJV3wA+DfzO0M1HA88C24FVSf4IOGbRh17N4Ojyl9j3KQ2A64HTkry9O2L9zwyeGBte57vAfJKTgN9d9PGPASfvZ+6Hga3AR5IckeR04H0Mjr57677IbAAu3s8XnOcleXmScxicl74buGE5a+vgZJy1HH8KHDl0/SbgRuD/MjhVsJOh0xOd64BTgG1V9bV9PWj3BNg7gcuBx4GfZPDE2a7uLn8CvI7BUfn1wOcWPcRHgD/oXiu8rycufxVYy+Ao+vPAH1fVrUv8XZdUVfdX1f0vcpePJ/kOgy8e/5PBaZ6Lquq55a6tg0/8YftqXZJDGJxzfldVfWnS80jj4JGzmpTkwu6laYcD/53BOeWvTHgsaWyMs1q1DngI2MHgXO7bq+qZyY4kjY+nNSSpQR45S1KDlnph/bIcf/zxtXbt2pVcQpJeknsn/raef6Nqx5I/zGpF47x27VruueeelVxCkl6Syf+Mv6V+jtaApzUkqUHGWZIaZJwlqUHGWZIaZJwlqUHGWZIaZJwlqUHGWZIaZJwlqUHGWZIaZJwl6UAc+Sz86T8P/hwD4yxJB+LsHbD+cVi3YyzLLRnnJCcm+WqSnQu/Hj7JFUnuSPLRlR9Rkhrw1m0v/HOFHchPpXsCOJ/BL8IkyeuAo6pqfZK/TPL6qvrHlRxSksbuz++Ds57ae/0H3Y+zO3UevrRl7+33rIbfPXPkyy955FxVO6vqyaGbfhq4pbt8K4NfJyRJB5erXgU7hxL5snrhnzDYfuWrVmT5PuecVwNPd5fnu+vPS7IxyYNJts/NzS13PkmajPuOhd8/7YWBHrbzEPjQafC1Y1dk+T5xngeO6S4fAzw1vLGqNlXVKVW1ZnZ2drnzSdLk3Hcs/MlPwq5Fqdx1yOD2FQoz9Ivzlxmcgwa4AH9dvaSD2VHPwh4G/+08ZO/lo1b2JXUH8mqNw5LcCpwB3AQcBuxMcgewp6ruXtEJJWmSfnYbHPEcPHQU/MGpgz+PeG7FX7Wx5Ks1qmo3gyPkYXetzDiS1JjvHQp/+ePw9z8KFXj/sfCLD8Pp8yu67Ir+gldJ+g/vD0974fXnAp+Zhc+s7LK+Q1CSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGuQ7BCVNlaql77OSzjrrwO7nkbMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDesU5yaok1yT5UpLLRz2UJE27vkfO7wC+VlXnAT+U5IwRziRJU69vnE8G/qm7fB9w9vDGJBuSbJqfn1/ObJI0tfrG+QHgnO7yecDq4Y1VtbmqNs7MzCxnNkmaWn3jvJnB6YwvAruAx0Y3kiSpV5yrak9VXVpV5wN7gJtGO5YkTbdev+A1yUnAVcBzwCer6pGRTiVJU65XnLsYnzvaUSRJC3wTiiQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoNW9fmgJC8HPgMcCcwDl1TVrlEOJknTrO+R80XAXVV1LnB3d12SNCJ94/wQg6NmgNXA46MZR5IE/eP8ILAuyf3AWcDWhQ1JNiZ5MMn2ubm5UcwoSVOnb5x/HdhcVa8FrgfevbChqjZV1SlVtWZ2dnYUM0rS1Okb5wBPdJd3ADOjGUeSBD1frQFcDXw6yXuA3cAvj24kSVKvOFfVU8CFI55FktTxTSiS1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1CDjLEkNMs6S1KBecU5yUZIt3X+PJnn7qAeTpGm2qs8HVdWNwI0ASe4Cbh3lUJI07ZZ1WiPJycBjVfXdRbdvSLJpfn5+WcNJ0rRa7jnnXwA+v/jGqtpcVRtnZmaW+fCSNJ2WG+cNwHWjGESStFfvOCd5JfCDqnp8hPNIkljekfPbgGtHNYgkaa9er9YAqKq/GuUgkqS9fBOKJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg4yzJDXIOEtSg3rHOcl7k3wxyZYkJ41yKEmadqv6fFAX43Oq6vwRzyNJov+R84XAod2R88eSHDrKoSRp2vWN8wnAy7oj5+8Db1vYkGRjkgeTbJ+bmxvFjJI0dfrGeR64vbt8G/CahQ1VtamqTqmqNbOzs8udT5KmUt84bwVO7y6fCfzraMaRJEHPJwSr6r4kzyTZAuwArhjpVJI05XrFGaCqLhvlIJKkvXwTiiQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1qFeck6xN8liSLUluHvVQkjTtVi3jY2+pqnePbBJJ0vOWc1rjvCR3JPnAyKaRJAH94/wo8BPAecAFSU4f3phkQ5JN8/Pzy51PkqZSrzhX1a6q+l5VPQt8ATh10fbNVbVxZmZmFDNK0tTp+4Tg0UNX3wg8NJpxJEnQ/7TG+iT3JtkKPFJVd41yKEmadr1erVFVNwA3jHgWSVLHN6FIUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOMsyQ1yDhLUoOWFeckH0hy56iGkSQN9I5zksOBM0c4iySps5wj5/cBn1h8Y5KNSR5Msn1ubm4ZDy9J06tXnJMcBpxbVbct3lZVm6rqlKpaMzs7u+wBJWka9T1yfg9w9SgHkSTt1TfOrwben+RG4LVJLh3hTJI09Vb1+aCq+uDC5SR3VtXHRjeSJGnZr3OuqjeNYhBJ0l6+CUWSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBxlmSGmScJalBveKc5NQkW5PckeRvkmTUg0nSNOt75PxAVZ1dVeu762eNaiBJUs84V9Xuoau7gIdHM44kCZZxzjnJxUm+DpwAPL5o24Ykm+bn55c7nyRNpd5xrqrrqupU4FvAzy/atrmqNs7MzCx3PkmaSn2fEDx86OrTwDOjGUeSBLCq58ddlOS/dpcfBG4e0TySJHrGuaquBa4d8SySpI5vQpGkBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBvWKc5I3JNma5M4kV4x6KEmadn2PnL8JvLmq3gS8IslpI5xJkqZerzhX1baq2tld3Q3sWdiWZGOSB5Nsn5ubG8WMkjR1lnXOOcnpwJqq+peF26pqU1WdUlVrZmdnlz2gJE2jVX0/MMlxwMeBS0Y3jiQJ+j8huAq4ErisqraNdiRJUt/TGu8EXg9cnmRLknUjnEmSpl6v0xpV9bfA3454FklSxzehSFKDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNcg4S1KDesU5yYlJvppkZ5JVox5KkqZd3yPnJ4Dzga+McBZJUqdXnKtqZ1U9ub/tSTYk2TQ/P99/MkmaYityzrmqNlfVxpmZmZV4eEk66PmEoCQ1yDhLUoP6vlrjsCS3AmcANyV5w2jHkqTp1utlcFW1G7hgxLNIkjqe1pCkBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWqQcZakBhlnSWpQqmrlHjzZDnxzGQ8xC8yNaJy+ZoBJ/mBq94H7YMGk94P7YDT74FVVtWapO61onJcryfYD+Uus8AybqmrjBNd3H7gPFmaY6H5wH4x3H7R+WuOpSQ8AbJ7w+u4D98GCSe8H98EY90HrcZ70t1BU1aT/QboP3AcLJrof3Afj3Qetx3nTpAdogPvAfbDA/TBF+6Dpc86SNK1aP3KWpKnUZJyTnJjkq0l2Jun121pGMMMbkmxNcmeSKyaw/qnd+nck+ZskGfcMQ7N8IMmdE1h3bZLHkmxJcvO41x+a471JvtjNcdIE1r+oW3tLkkeTvH3M6788yfXd+tcmOXyc63czrEpyTZIvJbl8jOv+uxYluaL7vPzoSq7dZJyBJ4Dzga9McIZvAm+uqjcBr0hy2pjXf6Cqzq6q9d31s8a8PgDdJ+KZk1i7c0tVnVtVb5nE4l2Mz6mq87s5Hhn3DFV1Y7f2uQxe43vrmEe4CLirW//u7vq4vQP4WlWdB/xQkjPGtO4LWpTkdcBR3efly5K8fqUWbjLOVbWzqp6c8Azbqmpnd3U3sGfM6+8euroLeHic6w95H/CJCa0NcF53lPKBCa1/IXBod+T8sSSHTmgOkpwMPFZV3x3z0g8BR3aXVwOPj3l9gJOBf+ou3wecPY5F99GinwZu6S7fCqxbqbWbjHNLkpwOrKmqf5nA2hcn+TpwAhP4hEhyGHBuVd027rU7jwI/AZwHXND9vxi3E4CXVdX5wPeBt01ghgW/AHx+Aus+CKxLcj+D7+C2TmCGB4BzusvnMfgiMQmrgae7y/MrOYdxfhFJjgM+zuDoceyq6rqqOhX4FvDzExjhPcDVE1gXgKraVVXfq6pngS8Ap05gjHng9u7ybcBrJjDDgg3AdRNY99eBzVX1WuB64N0TmGEzg9MZX2TwneRjE5gBBv8ejukuH8MKvinGOO9Hd/L/SuCyqto2gfWHn3R5Gnhm3DMArwben+RG4LVJLh3n4kmOHrr6RgbfXo/bVmDhiP1M4F8nMANJXgn8oKomcUohDM69Auxg8PMlxqqq9lTVpd13MHuAm8Y9Q+fLDM5BA1zACj4v1mSckxyW5FbgDOCmJG+YwBjvBF4PXN49S71i55b246Iktye5ncG31mN/tUJVfbCqLqyqi4D7q+pjYx5hfZJ7k2wFHqmqu8a8PlV1H/BMki0M/j18dtwzdN4GXDuhta8GLun2wbuAq8Y9QJKTus/D24Ct43pidnGLgMOAnUnuAPZU1d0rtrZvQpGk9jR55CxJ0844S1KDjLMkNcg4S1KDjLMkNcg4S1KDjLMkNej/A0dz8khgaaOcAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x432 with 1 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"np.random.seed(0)\n",
"nvmdp = NavigationWorldMDP(width=10, height=10, \n",
" nav_cell_types=['white'],\n",
" nav_cell_rewards=[0],\n",
" nav_cell_p_or_locs=[1.],\n",
" goal_cell_locs=[[(10,10)]],\n",
" goal_cell_rewards=[1.],\n",
" goal_cell_types=[\"blue\"],\n",
" slip_prob=0.00, step_cost=0.01, gamma=.99)\n",
"nvmdp.visualize_grid()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"S = nvmdp.get_reachable_states()\n",
"A = nvmdp.get_actions()\n",
"s_to_idx = {s: idx for idx, s in enumerate(S)}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# High Temperature"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZ4AAAGCCAYAAADQaKXvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XuwnXV97/H3h0AAUYsItgrIxcaO4AVbhHoHQYnHArajLe1R6YxTRo84HR07Yi/Ypqed1s7U07EcNTOl441SL9MaajQFEUcOFRMU8EDLIeCFxBs3wSIJyd7f88d6Qlc2Sfbaa6+stX7Z79fMM3ku63meb1ay88339/s9vydVhSRJ47LfpAOQJC0tJh5J0liZeCRJY2XikSSNlYlHkjRWJh5J0liZeLRkJTk2SSXZf9KxSEuJiUdNS/KFJKt2sf/cJD8wqUjTx8Sj1n0EeEOSzNn/RuATVbV9AjFJ2gMTj1r3z8CTgZfu2JHkScCvAB9N8pok30jyYJK7kvzx7i6U5NtJzuzb/uMkH+/b/uUk1yX5cZKbkpy2N35D0r7OxKOmVdXDwCeBN/Xt/nXgP6rqJuCh7tihwGuAtyZ57ULvk+RI4HPA/wQOA94FfCbJEYv7HUhLj4lH+4KPAK9LclC3/aZuH1V1TVV9s6pmq+pm4B+Alw9xjzcAa6tqbXetK4ENwH8bQfzSkmLiUfOq6lrgHuC1SZ4BnAJcBpDk1CRfSnJ3kgeAtwCHD3GbY4DXd81sP07yY+AlwFNH87uQlg5H/Ghf8VF6lc4vAOuq6ofd/suAvwVeXVVbkvwvdp94HgIe17f9c33rdwEfq6rfGW3Y0tJjxaN9xUeBM4HfoWtm6zwBuK9LOqcAv7WHa9wInJfkgCQnA6/rO/Zx4OwkZyVZluSgJKclOWrEvw9pn2fi0T6hqr4NXAccAqzpO/Q/gFVJfgJcTG8gwu78EfAM4H7gT+ia67rr3wWcC/w+cDe9Cuj38GdIWrD4IjhJ0jj5vzVJ0liZeCRJY2XikSSNlYlHkjRWJh5J0ljt1QdIl+fAOohD9uYt1Lq5c0pPSDIl/wc74IBJRwDAtidMz7PlJzzt7kmHAMD/u/lx839oL9vCQzxSWwNw1umH1L33zQx03g03b11XVSv3anALsFf/dh3EIZyaM/bmLTSsx7xFYDKybNmkQwAgBx886RAAyFOfMukQAPjRy6YjDoCvrfrgpEMA4KynnTTpELi+vvjo+r33zfC1dU8f6LxlT719mGmi9prp+W+NJGlgBcwyO+kwhmLikaQmFTPVZuKZkoZtSdJSYcUjSQ3qNbW1OeWZiUeSGmUfjyRpbIpiptFJnu3jkSSNlRWPJDXKPh5J0tgUMGPikSSNU6sVj308krSXbT9kOTf9yavZfsjySYcyFUw8krSX3f2iY7n7pcdz9wuPHdk1C5ipGmiZNvMmniRPS/L1JFuS7N/te3+SryT5m70foiS17Xsrn9X79dXPGul1Zwdcps0gfTz3AWcA/wSQ5BeBx1fVS5N8MMkLqmr93gxSklpyw1+dw/2/dPSj23mk9/qCH5/4VK66+m2P7n/SDXfxS7+3Zqh7FNXs4IJ5K56q2lJV9/ft+mXgym79KuCFeyMwSWrVcZ+4gf22bHt0u5Yv2+lXgP22bOO4j28Ye2zTYJg+nkOBB7v1B7rtRyW5IMntSe7ewkOLjU+SmnPYjZs56fc/t1Py6bfflm2c9J5/4bCbvjf8TQpmBlymzTCJ5wHgid36E4Ef9x+sqtVVtaKqjvDto5KWqsNu3MxzVq1jv63bd9q/39btPGfVusUlHXZMEtpmH88wieff6PX5AJwJfHV04UjSvmP74w8kM7MwM9urfmZmycws2x9/4AiuHmYGXKbNIKPaDkhyFfA8YB1wALAlyVeAmar62l6OUZKa9L1XP4uZgw7gCXfey/P+aC1PuPNeZg46YOSj21oz76i2qtpGr7Lpd/3eCUeS9h3LHnqEFR/6Pzz9MzeRgsPe+im++2vP4/7nPnXR1y5gdgr7bwbhlDmStJecdPHnd9rObHHMp2/kmE/fOJLrT2Mz2iBMPJLUoN4koW0mHqfMkSSNlRWPJDVqttqseEw8ktSglpvaTDyS1KAizDTaW9Jm1JKkZlnxSFKj7OORJI2NfTySpDELM9Vmb0mbUUuSmmXFs1RNyXvYa0omm8rMzKRDACAPPTzpEAA4dOPWSYfwqBUfe+ukQwDg9u99cNIhcMpZP310vfdahDZrhzajliSN7LUISVYmuS3JxiQX7eL4W5J8M8mNSa5NckK3/9gkD3f7b0zyoUHituKRpAZVjaaPJ8ky4BLglcAmYH2SNVV1a9/HLquqD3WfPwf4a2Bld+yOqjppIfe04pGkpe0UYGNV3VlVjwCXA+f2f6CqHuzbPIReS9/QrHgkqVGzoxlOfSRwV9/2JuDUuR9K8jbgncBy4BV9h45L8g3gQeAPq+or893QikeSGtR7jme/gRbg8CQb+pYLFny/qkuq6hnAu4E/7HZ/H3h6VT2fXlK6LMkT57uWFY8kNWlBfTz3VNXJuzm2GTi6b/uobt/uXA58EKCqtgJbu/UbktwBPBPYsKdgrHgkaWlbD6xIclyS5cB5wJr+DyRZ0bf5GuD2bv8R3eAEkhwPrADunO+GVjyS1KBRPcdTVduTXAisA5YBl1bVLUlWARuqag1wYZIzgW3A/cD53ekvA1Yl2QbMAm+pqvvmu6eJR5IaNTOiSUKrai2wds6+i/vWf3c3530G+MxC72fikaQG+T4eSZIGZMUjSY2abXR2ahOPJDVox3M8LTLxSFKDioxscMG4DZUuk+yf5PIkX0ryvlEHJUnadw1bp/0qcFNVnQ4cnOR5I4xJkjSAWfYbaJk2wza1HQ/c3K3fCLwIuGkkEUmS5lXFknv19W3Ay7v104FD+w8mOTvJ6u08spjYJEm7FWYHXKbNsInnCnpNbF+kN0HcD/sPVtUVVXXB/ixfbHySpH3MUE1tVTUDvB0gyWp6c/xIksakaLepbajEk+RI4BP0JoX7aFXtaQptSdJesKSe4+kSzWmjDUWSNKgizC6l53gkSRqWMxdIUqOWVFObJGmyCicJlSSNVZiZwmd0BtFmupQkNcuKR5IaZFObJGnsWm1qM/FIUoOq0mzF02bUkqRmWfFIUqOW1FxtkqTJKpjKVx4MwsSjyarZSUcAQG3bPukQAKif/OekQwDgwO/cO+kQHnXM55886RAAOPn2t046BP7je+/v20qzFU+bUUuSmmXFI0kN6j3HY1ObJGmMnCRUkjQ2vo9HkqQBWfFIUqNmG60dTDyS1KAqmGm0qc3EI0mNso9HkqQBmHgkqUG9UW37DbTMJ8nKJLcl2Zjkol0cf0uSbya5Mcm1SU7oO/ae7rzbkpw1SOw2tUlSo0bxPp4ky4BLgFcCm4D1SdZU1a19H7usqj7Uff4c4K+BlV0COg84EXgacFWSZ1bVzJ7uacUjSQ3aMXPBIMs8TgE2VtWdVfUIcDlw7k73qnqwb/OQ7vZ0n7u8qrZW1beAjd319siKR5KWtiOBu/q2NwGnzv1QkrcB7wSWA6/oO/erc849cr4bWvFIUpMW1MdzeJINfcsFC71bVV1SVc8A3g384WIiH6riSfI44FP0Sq4HgF+vqq2LCUSStDALeB/PPVV18m6ObQaO7ts+qtu3O5cDHxzyXGD4imclcH1VnQZ8rduWJI3JjgdIB1nmsR5YkeS4JMvpDRZY0/+BJCv6Nl8D3N6trwHOS3JgkuOAFfRywh4N28dzB//VBngoMD1vjZIkDayqtie5EFgHLAMurapbkqwCNlTVGuDCJGcC24D7gfO7c29J8kngVmA78Lb5RrTB8InnduCFSW4BfkSvzQ+Aru3w94BDD2D5kJeXJM1nkGd0BlFVa4G1c/Zd3Lf+u3s498+AP1vI/YaN+nzgiqo6Efgc8Ia+IFZX1YqqOuIgDhny8pKkPdnxWoQRDKceu2ErngD3dev3AD8zmnAkSYNawOCCqTJs4rkM+Mckb6TX5vcbowtJkrQvGyrxVNWPgYHm5JEkjd6OmQta5MwFktSoUQ0uGDcTjyS1aEoHDgyizXQpSWqWFY8kNahYeqPaJEkT1mpTm4lHkhrU8qg2+3gkSWNlxSNJjWq14jHxSFKDdszV1iITjyQ1qtVRbfbxSJLGyopHklpU9vFITauZeV+aOBazW7dOOgQA9rv3/kmH8KgDH9k26RAAeMr3Dp50CNzxk+2Prrc8nNrEI0mNajXx2McjSRorKx5JapDDqSVJY1cmHknSOPkcjyRJA7DikaQGlc/xSJLGzT4eSdIYtTuqzT4eSdJYWfFIUqNsapMkjY1ztUmSxqt6I9taNFQfT5KVSa7plu8nee2oA5Mk7ZuGqniq6gvAFwCSXA9cNcqgJEnza3XmgkU1tSU5HvhhVf3niOKRJA2gWLqDC34N+KdRBCJJWoh2n+NZbOI5m17y2UmSs4GzD+aQRV5ekrSvGfoB0iQ/BzxSVffOPVZVV1TVBfuzfFHBSZJ2r2qwZT7dgLHbkmxMctEujr8zya1Jbk7yxSTH9B2bSXJjt6wZJO7FVDznAp9dxPmSpEUYRR9PkmXAJcArgU3A+iRrqurWvo99Azi5qn6a5K3A+4Df6I49XFUnLeSeQyeeqvrwsOdKkhanV82MpI/nFGBjVd0JkORyeoXFo4mnqr7U9/mvAm9YzA2dq02S9n2HJ9nQt1zQd+xI4K6+7U3dvt15M/D5vu2Dumt+ddBnOp25QJIatYBRbfdU1cmLvV+SNwAnAy/v231MVW3uHq+5Osk3q+qOPV3HikeSGjWiwQWbgaP7to/q9u0kyZnAHwDnVNXW/4qhNne/3glcAzx/vhuaeCSpUVUZaJnHemBFkuOSLAfOA3YanZbk+cCH6SWdH/Xtf1KSA7v1w4EX09c3tDs2tUnSElZV25NcCKwDlgGXVtUtSVYBG6pqDfBXwOOBTyUB+G5VnQM8C/hwkll6hcxfzBkNt0smHklqUDFQNTPYtarWAmvn7Lu4b/3M3Zx3HfCchd7PxCNJjWr0rQgmHklq0uie4xk7BxdIksbKikeSWtVoW5uJR5Ia1WpTm4lHkho1yMzT08jEo8mamp+c2UkHAEBt2z7pEACYnZ2WPxfI9un4TnjwJ5OOALZtm3QEI2HikaQGLeVXX0uSJqEAE48kaZympqV6gXyOR5I0VlY8ktSqRiseE48kNWl0k4SOm4lHklrVaMVjH48kaayseCSpRQ3PTm3ikaRWNdrUZuKRpGa1WfHYxyNJGisrHklqVaNNbUNXPEnelOSLSa5JcuQog5IkDaAGXKbMUBVPl2heXlVnjDgeSdIgGp4kdNiK5yxgWVfxfCDJslEGJUnadw2beH4WWN5VPD8Fzh1dSJKkQVQNtkybYRPPA8CXu/WrgWftOJDkgiS3J7l7Cw8tNj5J0u402sczbOK5Dnhut34S8K0dB6pqdVWtqKojDuKQxcYnSdqdymDLlBlqcEFV3Zjk4STXAPcA7x9pVJKkfdbQz/FU1btGGYgkaWEyhc1og/ABUklq0ZT23wzCxCNJTZrO/ptBOFebJGmsrHgkqVU2tUmSxsrEI0kaq0YTj308krTEJVmZ5LYkG5NctIvj70xya5Kbuzk6j+k7dn43W83tSc4f5H4mHklq0Y7ZqRc5c0E3yfMlwKuBE4DfTHLCnI99Azi5qp4LfBp4X3fuYcB7gVOBU4D3JnnSfKGbeCSpUanBlnmcAmysqjur6hHgcuZM/FxVX6qqn3abXwWO6tbPAq6sqvuq6n7gSmDlfDc08UhSq0YzSeiRwF1925u6fbvzZuDzQ54LOLhAkpaCw5Ns6NteXVWrF3qRJG8ATgZevphgTDyStO+7p6pO3s2xzcDRfdtHdft2kuRM4A/ovX16a9+5p80595r5gjHxSDBFb8uanXQAANT26YgDoGZmJh0CANlv8tPTVO385zKiSULXAyuSHEcvkZwH/NZO90meD3wYWFlVP+o7tA74874BBa8C3jPfDU08ktSqEczVVlXbk1xIL4ksAy6tqluSrAI2VNUa4K+AxwOfSgLw3ao6p6ruS/Kn9JIXwKqqum++e5p4JGmJq6q1wNo5+y7uWz9zD+deCly6kPuZeCSpRb4WQZI0diYeSdI4tfoGUh8glSSNlRWPJLWq0YrHxCNJrTLxSJLGZcAJQKeSfTySpLGy4pGkVo1g5oJJMPFIUqsabWoz8UhSo5ZUH0+SY5P8MMk1Sf511EFJkvZdi6l4rqyqN4wsEknSwiyliqdzepKvJHnHyKKRJA2m/mtI9XzLtBk28XwfeCZwOnBmkueOLiRJ0kBqwGXKDJV4qmprVT1UVduBfwGe3X88ydlJVm/nkVHEKEnahww7uOAJfZsvBu7oP15VV1TVBfuzfDGxSZL2ZClVPMBLk9yQ5Dpgc1VdP8qgJEnza7WPZ6hRbbt6TaokSYNwrjZJ0lg5c4EktWoKm9EGYeKRpBZNaf/NIEw8ktSqRhOPfTySpLGy4pGkVjVa8Zh4JKlBwT4eSdK4NZp47OORJI2VFY8ktcjh1JKksTPxSJLGysQjadGq0X9J9qaamXQEAFRl0iE0m2jmMvFIUqPs45EkjVejicfh1JLUokHfPjpAckqyMsltSTYmuWgXx1+W5OtJtid53ZxjM0lu7JY1g4RuxSNJS1iSZcAlwCuBTcD6JGuq6ta+j30X+G3gXbu4xMNVddJC7mnikaRGjaiP5xRgY1XdCZDkcuBc4NHEU1Xf7o7NjuKGNrVJUqtG09R2JHBX3/ambt+gDkqyIclXk7x2kBOseCSpUQuoeA5PsqFve3VVrR5RGMdU1eYkxwNXJ/lmVd2xpxNMPJK077unqk7ezbHNwNF920d1+wZSVZu7X+9Mcg3wfGCPicemNklq1Wia2tYDK5Icl2Q5cB4w0Oi0JE9KcmC3fjjwYvr6hnbHxCNJLRrRcOqq2g5cCKwD/h34ZFXdkmRVknMAkrwgySbg9cCHk9zSnf4sYEOSm4AvAX8xZzTcLtnUJkkNSreMQlWtBdbO2Xdx3/p6ek1wc8+7DnjOQu9nxSNJGisrHklqVaNT5ph4JKlRrU4SuqimtiTvSHLtqIKRJC3AiOZqG7ehE083hG5B8/NIkrSYiufNwEdGFYgkaYGWUsWT5ADgtKq6ehfHLkhye5K7t/DQogOUJO1C9fp4BlmmzbAVzxuBy3Z1oKpWV9WKqjriIA4ZPjJJ0p4tpYoH+AXgrUm+AJyY5O0jjEmStA8bajh1Vb17x3qSa6vqA6MLSZI0iGlsRhvEop/jqaqXjCIQSdICLdXEI0majFYrHudqkySNlRWPJLVoSkesDcLEI0mtMvFIksYl2McjSdJArHgkqVWNVjwmHklqVKrNzGPikaQWNTyqzT4eSdJYWfFIUqNaHdVm4pGkQUxjf8oUhjQIE48kNarVisc+HknSWFnxSFKrGq14TDyS1KJqt6nNxCNJrWo08djHI0kaKyseSWpQy7NTm3gkqVXT+GzRAEw8ktSoVise+3gkSWNlxSNJLWp4dmoTjyQ1KrOTjmA4NrVJUqtqwGUeSVYmuS3JxiQX7eL4y5J8Pcn2JK+bc+z8JLd3y/mDhG3ikaQlLMky4BLg1cAJwG8mOWHOx74L/DZw2ZxzDwPeC5wKnAK8N8mT5rvnUIknybOTXJfkK0n+PkmGuY4kaXipwZZ5nAJsrKo7q+oR4HLg3P4PVNW3q+pmYG7j3lnAlVV1X1XdD1wJrJzvhsNWPLdV1Yuq6qXd9slDXkeSNIyi9xzPIMueHQnc1be9qds3iKHOHWpwQVVt69vcOufGkqQxWMBzPIcn2dC3vbqqVo8+osEMPaotyTnAnwO3A/fOOXY2cPbBHLK46CRJo3BPVe2uZWozcHTf9lHdvkFsBk6bc+4185009OCCqlpTVc+mV1r9ypxjV1TVBfuzfNjLS5LmM5pRbeuBFUmOS7IcOA9YM2AE64BXJXlSN6jgVd2+PRp2cMGBfZsPAg8Pcx1J0nB2TBK62MEFVbUduJBewvh34JNVdUuSVV3LFklekGQT8Hrgw0lu6c69D/hTeslrPbCq27dHwza1rUzyzm79duBfh7yOJGkYgw0cGPBStRZYO2ffxX3r6+k1o+3q3EuBSxdyv2EHF3wW+Oww50qSljanzJGkRrU6O7WJR5JaZeKRJI1TqxWPc7VJksbKikeSWlTAbJslj4lHklrVZt4x8UhSq+zjkSRpAFY8ktSqEc1cMG4mHklqVKtNbSYeSWrRYDNPTyX7eCRJY2XFI0kN6r0Woc2Sx8QjSa2anXQAwzHxSFKjWq147OORJI2VFY8ktajhUW0mHklq0uhefT1uJh5JalSrD5DaxyNJGisrHklqlU1tkqSxKYjP8UiSxqrRisc+HknSWFnxSFKr2ix4TDyS1KolNWVOklOTXJfk2iTvH3VQkqQBVA22TJlh+3i+A7yiql4CPCXJc0YYkyRpHzZUU1tV/aBvcxswM5pwJEkDKZp9LcKiRrUleS5wRFXd2rfvgiS3J7l7Cw8tOkBJ0mOFIjXYMm2GTjxJDgP+Fnhz//6qWl1VK6rqiIM4ZLHxSZJ2Zyn18STZH/g48K45zW6SJO3RsBXP64EXAO9Lck2SF44wJknSIJZSxVNV/1BVR1TVad3yb6MOTJK0BzsGFwyyzCPJyiS3JdmY5KJdHD8wyT92x69Pcmy3/9gkDye5sVs+NEjoPkAqSY0axcCBJMuAS4BXApuA9UnW9A8ao9eXf39V/XyS84C/BH6jO3ZHVZ20kHs6V5skLW2nABur6s6qegS4HDh3zmfOBT7SrX8aOCNJhr2hiUeSWjWaPp4jgbv6tjd1+3b5maraDjwAPLk7dlySbyT5cpKXDhK2TW2S1KQFDRw4PMmGvu3VVbV6BEF8H3h6Vd2b5JeAf05yYlU9uKeTTDyS1KJiIYnnnqo6eTfHNgNH920f1e3b1Wc2dY/T/Axwb1UVsBWgqm5IcgfwTGADe2BTmyQtbeuBFUmOS7IcOA9YM+cza4Dzu/XXAVdXVSU5ohucQJLjgRXAnfPd0IpHklo1grnaqmp7kguBdcAy4NKquiXJKmBDVa0B/g74WJKNwH30khPAy4BVSbZ10bylqu6b754mHklq1KjmYauqtcDaOfsu7lvfQm/igLnnfQb4zELvZ+KRpFZN4awEg7CPR5I0VlY8ktSiAmbbrHhMPJLUpOmcAHQQJh5JalWjicc+HknSWFnxSFKrGq14TDyS1CIHF0iSxqugRjB1wQTYxyNJGisrHklqlX08kqSxsY9HkjR2jVY89vFIksbKikeSWtVoxWPikaQmOVebJGmcCpj1OR5JkuY1VOJJ8rQkX0+yJYlVkyRNQtVgy5QZtuK5DzgD+OoIY5EkLUSjiWeoaqWqtgBbkow4HEnSYMoHSPslORs4+2AO2RuXlyQ1bK8MLqiqK6rqgv1ZvjcuL0kqqJodaJk2DgyQpFY12tQ27Ki2A5JcBTwPWJfk1NGGJUma1xIbXLANOHPEsUiSlgCb2iSpRVXNzlxg4pGkVk1hM9ogTDyS1KhqtOJxrjZJ0lhZ8UhSk6ZzxNogTDyS1KKi2ed4TDyS1KopnJVgEPbxSJLGysQjSQ0qoGZroGU+SVYmuS3JxiQX7eL4gUn+sTt+fZJj+469p9t/W5KzBondxCNJLarqNbUNsuxBkmXAJcCrgROA30xywpyPvRm4v6p+Hng/8JfduScA5wEnAiuB/91db49MPJLUqBFVPKcAG6vqzqp6BLgcOHfOZ84FPtKtfxo4I70Xsp0LXF5VW6vqW8DG7np7ZOKRpKXtSOCuvu1N3b5dfqaqtgMPAE8e8NzH2Kuj2n7C/fdcVZ/+ziIv83Tgu6OIZ5F+ht6XPWl+Hzvz+9jZtHwf4Hcy1yi+j2N2rPyE+9ddNfvJwwc876AkG/q2V1fV6kXGMrS9mniq6ojFXiPJ3VV18ijiWWQcq6vqgimIw+9j5zj8PnaOYyq+D/A72UUcI/0+qmrliC61GTi6b/uobt+uPrMpyf70kui9A577GC00tf140gF0rph0AB2/j535fexsWr4P8DuZa1q+j7nWAyuSHJdkOb3BAmvmfGYNcH63/jrg6qqqbv953ai344AVwNfmu2ELD5BOQ6lOVU3LXxq/j535fexsKr4P8DuZa4q+j51U1fYkFwLrgGXApVV1S5JVwIaqWgP8HfCxJBuB++glJ7rPfRK4FdgOvK2qZua7Z2rK5/pJcsEk2yKnjd/Hzvw+dub38Vh+J9Nn6hOPJGnfMrV9PEmeluTrSbZ0nVmTiuPUJNcluTbJ+ycYx7O7OL6S5O+7MfQTk+QdSa6d4P2PTfLDJNck+ddJxdHF8qYkX+ximXco6V6KYWV3/2uSfD/JaycRRxfL45J8rovls0kOnFAc+ye5PMmXkrxvAvd/zL9hSd7f/Qz/zbjjmSZTm3jotSOeAXx1wnF8B3hFVb0EeEqS50wojtuq6kVV9dJue2KjdLp/SE6a1P37XFlVp1XVqyYVQJdoXl5VZ3SxzDuiZ2+oqi909z+N3tDhqyYRR2clcH0Xy9e67Un4VeCmqjodODjJ88Z8/53+DUvyi8Dju5/h5UleMOZ4psbUJp6q2lJV909BHD+oqi3d5jZg3o6zvRTHtr7Nrez80Na4vZn/eop5kk7v/vf4jgnGcBawrKt4PjDIdCF7U5LjgR9W1X9OMIw7gEO69UPpDbudhOOBm7v1G4EXjfPmu/g37JeBK7v1q4AXjjOeaTK1iWfaJHkucERV3TrBGM5J8n+Bn2VCP8xJDgBOq6qrJ3H/Pt8HngmcDpzZ/flMws8Cy6vqDOCnPHaqkXH7NeCfJhzD7cALk9xCrzK/bkJx3Aa8vFs/nV4SnKRDgQe79QeYfDwTY+IZQJLDgL+l9z/9iamqNVX1bHrTUvzKhMJ4I3DZhO79qG5uqIe66Tv+BXj2hEJ5APhyt3418KwJxbHD2Tz2GYxxOx+4oqpOBD4HvGFCcVxBr4nti/RaCX44oTh2eAB4Yrf+RKbn+aKxM/HMo+sU/Djwrqr6wQTj6O+gfRDjA6DYAAABF0lEQVR4eEKh/ALw1iRfAE5M8vZJBJHkCX2bL6bXvDMJ1wE7qq2TgG9NKA6S/BzwSFVNqmnr0VDo9W8A3EPvKfexq6qZqnp7V43O0HtOZZL+jV6fD8CZTL7/emKmNvEkOSDJVcDzgHVJTp1QKK8HXgC8rxulM6l22ZVJvpzky/SadyYykquq3l1VZ3XTddxSVR+YRBzAS5PckOQ6YHNVXT+JIKrqRuDhJNfQ+3vy6UnE0TkX+OwE77/DZcCvd9/Jfwc+MYkgkhzZ/cxeDVw37oEfc/8NAw4AtiT5CjBTVfM+4b+v8jkeSdJYTW3FI0naN5l4JEljZeKRJI2ViUeSNFYmHknSWJl4JEljZeKRJI2ViUeSNFb/H/iA+sVQsFHlAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 1008x576 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"def get_value_grid(V_array, nvmdp, s_to_idx):\n",
" \"\"\"Returns value over states space grid.\n",
" \"\"\"\n",
" V = np.zeros((nvmdp.height, nvmdp.width), dtype=np.float32)\n",
" for row in range(nvmdp.height):\n",
" for col in range(nvmdp.width):\n",
" x, y = nvmdp._rowcol_to_xy(row, col)\n",
" V[row, col] = V_array[s_to_idx[NavigationWorldState(x, y)]]\n",
" return V\n",
" \n",
"## Greedy Policy\n",
"# v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, lambda Q: np.max(Q), 100)\n",
"## Boltzmann Policy\n",
"boltz_temp = 100\n",
"v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, \n",
" lambda Q, t=boltz_temp: np.exp((1/t)*Q) / (np.exp((1/t)*Q).sum()), 100)\n",
"V = get_value_grid(v, nvmdp, s_to_idx)\n",
"\n",
"plt.figure(figsize=(14,8))\n",
"nvmdp.visualize_grid(V, trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0. , 0. , 0. , 0. , 0.01, 0.01, 0.04, 0.13, 0.39, 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0.01, 0.03, 0.08, 0.19, 0.39],\n",
" [0. , 0. , 0. , 0. , 0. , 0.01, 0.02, 0.04, 0.08, 0.13],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0.01, 0.02, 0.03, 0.04],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.01, 0.01, 0.01],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0.01],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ],\n",
" [0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. , 0. ]],\n",
" dtype=float32)"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V.round(2)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Low Temperature"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAGCCAYAAAAsfY4lAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHahJREFUeJzt3XuwZWV55/Hvry+AAkoIeAMUzHRSIgKTIHijgECGJhEwmehoRk1mrHTFGa0pM1oxmQljmPwTUzVOxlhJemoy8YbEmEpsYsdWRIyGiGACJJAhdMgFUJG7gvTtnGf+2Lvb3Yc+5+yzz9pnrXXO90OtYq29dq/19Opz9rOf933Xu1JVSJLUpHVtByBJWn1MLpKkxplcJEmNM7lIkhpncpEkNc7kIklqnMlFa1aSk5NUkg1txyKtNiYX9VqSTye58hCvX57kGyYOqR0mF/XdB4E3Jsmc198EfLSq9rUQk7TmmVzUd38MfC9w7v4XknwP8GrgQ0l+LMlfJflWknuSvGe+AyX5xyQXjWy/J8lHRrZfluSGJI8muTXJ+dP4C0mrgclFvVZVTwIfB9488vLrgP9XVbcCTwz3HQP8GPDWJK9Z6nmSnAB8CvhV4FjgncAfJjl+eX8DaXUyuWg1+CDwk0mOGG6/efgaVXV9Vf11Vc1W1W3Ax4DzJjjHG4HtVbV9eKzPAjcDP9pA/NKqY3JR71XVl4AHgdck+T7gbOAqgCTnJPl8kgeSPAb8HHDcBKd5AfDaYZPYo0keBV4FPLeZv4W0ujiSRqvFhxhULD8A7Kiq+4evXwX8JnBJVe1K8j+ZP7k8ATx9ZPs5I+v3AB+uqp9tNmxpdbJy0WrxIeAi4GcZNokNHQ08PEwsZwM/tcAxbgFen2RjkrOAnxzZ9xHg0iQXJ1mf5Igk5yc5seG/h7QqmFy0KlTVPwI3AEcC20Z2/QfgyiTfBq5g0Pk/n18Gvg94BPgVhk1rw+PfA1wO/BLwAINK5l34OyQdUnxYmCSpaX7rkiQ1zuQiSWtckt9N8s0kfzPP/iT5X0l2JrktyQ8udkyTiyTp94DNC+y/BNg0XLYAv7XYAU0ukrTGVdWfAQ8v8JbLgQ/VwJeBY5IseI+XyUWStJgTGIyQ3O/e4WvzmupNlIdtPLKOOPyYaZ5iLLVu7oS57agN3YgDYHZ92xEMVEdu453d2HYEA+sOm2k7BACOOfzJtkM44IQN3Yjl7257+uJvmrJdPMGe2h2Aiy84sh56eLyfl6/etvt2YNfIS1urausUQjxgqr/aRxx+DOec/nPTPMVY9h7VjU+OPcd05JMU2HVMN4rWJ4/vRsJ98rnd+FA/6vnfajsEAC47+ZD9uq341Wf9ddshAHDx885sOwRurM8dWH/o4Rm+suP5Y/259c+9a1dVnbWMU98HnDSyfeLwtXl14xNGkrQkBcyO+V8DtgFvHo4aexnwWFV9faE/0J2v0pKkJShmqpHEQZKPAecDxyW5F/hvwEaAqvptYDuDGcB3At8B/t1ixzS5SNIaV1VvWGR/Af9xKcc0uUhSDw2axbo7fZfJRZJ6qqH+lKkwuUhSDxXFTIcnHna0mCSpcVYuktRT9rlIkhpVwIzJRZLUtC5XLva5SNKU7TvyMG79lUvYd+RhbYeyYkwukjRlD7ziZB4494U88PKTGztmATNVYy1tWDS5JHlekr9MsivJhuFr70vyxSS/Mf0QJanfvrb5RYP/X/KiRo87O+bShnH6XB4GLgT+CGD4eMujqurcJL+V5KVVddM0g5SkPvnqr1/GIz/03UmEs2cw6/ajL34u11733VlUvuer9/BD79o20TmK6nSH/qKVS1XtqqpHRl56GfDZ4fq1wMunEZgk9dUpH/0q63btPbBdh60/6P8A63bt5ZSP3Lzisa2USfpcjgH2P3TiseH2AUm2JLkryQO7dj+63PgkqXeOveU+zvylTx2UYEat27WXM3/xTzj21q9NfpKCmTGXNkySXB4DnjFcfwZwUAapqq1Vtamqju/CUyglqQ3H3nIfL7lyB+t27zvo9XW79/GSK3csL7Gwf+LK7va5TJJc/oJBHwzARcCXmwtHklaPfUcdTmZmYWZ2UMXMzJKZWfYddXgDRw8zYy5tGGe02MYk1wJnADsYPEBmV5IvAjNV9ZUpxyhJvfS1S17EzBEbOfruhzjjl7dz9N0PMXPExsZHjXXRoqPFqmovgwpl1I3TCUeSVo/1T+xh02//Oc//w1tJwbFv/QP++SfO4JHTn7vsYxcw293BYk7/IknTcuYVf3rQdmaLF3ziFl7wiVsaOX5bTV7jMLlIUg8NJq7sbnJx+hdJUuOsXCSpp2aru5WLyUWSeqjrzWImF0nqoSLMdLhno7uRSZJ6y8pFknrKPhdJUqPsc5EkTUGYqe72bHQ3MklSb021cslsse7xPdM8xVg2th1AJ3WlaO3K95v1i79lBTx+4GkW7drGaW2H0Dk7vtbMlC3LcfbF3zmwPphyvyu/P0/VlU8YSdIS2eciSWpUlX0ukqQ1xspFknpq1mYxSVKTBve5dLfxyeQiSb1kn4skaY2xcpGkHvI+F0nSVMw4caUkqUk+z0WStOZYuUhST812eLSYyUWSesj7XCRJjSvS6Q79idJekg1Jrk7y+STvbTooSVK/TVpT/Thwa1VdADwtyRkNxiRJGsMs68Za2jBps9gLgduG67cArwBubSQiSdKiqliV07/cCZw3XL8AOGZ0Z5JLk2zdO7NrObFJkuYVZsdc2jBpcrmGQXPY54DdwP2jO6vqmqrasnH9EcuNT5LUQxM1i1XVDPB2gCRbgR1NBiVJWljR7WaxiZJLkhOAjwKzwIeq6r5Go5IkLWrV3ecyTCbnNxuKJGlcRZhdbfe5SJK0EO/Ql6SeWnXNYpKkdhVOXClJalyYaekelnF0N+1JknrLykWSeshmMUnSVHS5WczkIkk9VJVOVy7djUyStCKSbE5yZ5KdSd59iP3PHz6/66+S3JbkRxc7ppWLJPVUE3OLJVkPfAD4EeBe4KYk26rqjpG3/Vfg41X1W0lOBbYDJy90XCsXSeqhgqam3D8b2FlVd1fVHuBq4PJDnO4Zw/VnAl9b7KDTrVxmZ8mu3VM9xTjWdSSFbmw7gE7qSvHckR8S1rcdAACPH/gcad82Tms7hM64b9+DI1tpalbkE4B7RrbvBc6Z8573AJ9J8nbgSOCixQ7ald8oSdL0HJfk5pFlyxL//BuA36uqE4EfBT6cZMH80ZWvjZKkJRjc5zL2UOQHq+qsefbdB5w0sn3i8LVRbwE2A1TVXyQ5AjgO+OZ8J7RykaSemmHdWMsibgI2JTklyWHA64Ftc97zz8CFAEleBBwBPLDQQa1cJKmHmnqeS1XtS/I2Bk8UXg/8blXdnuRK4Oaq2gb8Z+B/J3kHg6LpZ6qqFjquyUWS1riq2s5gePHoa1eMrN8BvHIpxzS5SFJPzXa4Z8PkIkk9VAUzHX7MsclFknqqiT6XaeluTSVJ6i0rF0nqocFose7WByYXSeopn+ciSWrUEu/QX3HdrakkSb1l5SJJvbQK+1ySPB34AwZTLz8GvK6q2p9bX5LWkDGe1dKaSdPeZuDGqjof+MpwW5K0QvbfRDnO0oZJk8vfM6haAI4BHmomHEnSajBpcrkLeHmS24GzgBv270iyJcldSR54ct+3mohRknQIs7VurKUNk571p4FrqurFwKeAN+7fUVVbq2pTVR3/tA3deVSqJK0m+6fcH2dpw6SjxQI8PFx/EHhmM+FIksbV5Q79SZPLVcDvJ3kTsBf4N82FJEnqu4mSS1U9ClzccCySpDF1/Q59b6KUpJ5adTdRSpJa1mJn/Ti6m/YkSb1l5SJJPVSsztFikqSWdblZzOQiST3U9dFi9rlIkhpn5SJJPdXlysXkIkk9tH9usa4yuUhST3V5tJh9LpKkxlm5SFIf1Vruc6kiu/dO9RRjSTf+AbpUJm5sO4DO6cr3rK78lKxvO4ADHqcbz4Xaxmlth8Cju798YL3rQ5G78hslSVqiLieXrnxNkiStIlYuktRDDkWWJE1FmVwkSU3zPhdJ0ppi5SJJPVRr+j4XSdLU2OciSWpYt0eL2eciSWqclYsk9ZTNYpKkRjm3mCSpeTUYMdZVE/W5JNmc5Prh8vUkr2k6MElSf01UuVTVp4FPAyS5Ebi2yaAkSYvr8h36y2oWS/JC4P6qeryheCRJYyhWd4f+TwB/1EQgkqSl6PZ9LstNLpcySDAHSXIpcOnTNzxzmYeXJPXRxDdRJnkOsKeqHpq7r6quqaotG9YdvqzgJEnzqxpvacNyKpfLgU82FYgkaWlWZZ9LVf1Ok4FIksY3qEq6m1ycW0yS1Djv0JeknlrNo8UkSS3p8vQvJhdJ6in7XCRJa4qViyT1UJFOVy4mF0nqqQ53uZhcJKmXvM9FkrTWmFwkqa9qzGURwwdA3plkZ5J3z/Oe1yW5I8ntSa5a7Jg2i0lSTzXRLJZkPfAB4EeAe4GbkmyrqjtG3rMJ+EXglVX1SJJnLXZcKxdJ6qmGZkU+G9hZVXdX1R7gagYTE4/6WeADVfXI4Lz1zcUOOt3KZbaoPXumeopxdLfLqz1d+Vaxse0AOqcrjQld+QkBWN92AAA8zjPaDoHZPVO5FicA94xs3wucM+c93w+Q5M8Z/IO8Z/i4+3l15SdZkrQES3zM8XFJbh7Z3lpVW5dwug3AJuB84ETgz5K8pKoeXegPSJL6poDxk8uDVXXWPPvuA04a2T5x+Nqoe4Ebq2ov8A9J/o5BsrlpvhN2qfaVJC1BQ30uNwGbkpyS5DDg9cC2Oe/5YwZVC0mOY9BMdvdCBzW5SNIaVlX7gLcBO4C/BT5eVbcnuTLJZcO37QAeSnIH8HngXYd6xP0om8Ukqa8amv+lqrYD2+e8dsXIegE/P1zGYnKRpF5y4kpJ0jR0eOZK+1wkSY2zcpGkPur4rMgmF0nqqw43i5lcJKm3ulu52OciSWqclYsk9VWHm8UmrlySvDnJ55Jcn+SEJoOSJI2hoYeFTcNElcswmZxXVRc2HI8kaRxLm7hyxU1auVwMrB9WLu8fPslMkiRg8uTybOCwYeXyHZ761DJJ0pQ1NCvyVEyaXB4DvjBcvw540f4dSbYkuSvJA0/OfHu58UmS5tPhPpdJk8sNwOnD9TOBf9i/o6q2VtWmqjr+aeuPXm58kqT5VMZbWjBRh35V3ZLkySTXAw8C72s0KklSr018n0tVvbPJQCRJS5MO3+fiTZSS1Ect9qeMw+QiSb3UXn/KOJxbTJLUOCsXSeorm8UkSY0zuUiSGtfh5GKfiySpcVYuktRHHZ8V2eQiST3lTZSSpOZ1OLnY5yJJapzJRZLUuOk2i1XBnr1TPcVYYbQdwFB3u97a05VvNxvbDqBzutRi3pWfkvYfuLtuzsepfS6SpOZ1eLRYV74SSJJWESsXSeojp9yXJE2FyUWS1LQud+jb5yJJapyViyT1VYcrF5OLJPWVyUWS1KSUfS6SpDXGykWS+qrDd+ibXCSprzrcLGZykaSeWnV9LklOTnJ/kuuTfKbpoCRJ/bacyuWzVfXGxiKRJC3Naqtchi5I8sUk72gsGknSeOq7w5EXW9owaXL5OvD9wAXARUlOby4kSdJYasylBRMll6raXVVPVNU+4E+A00b3J7k0yda9taeJGCVJPTNph/7RI5uvBP5+dH9VXVNVWzbmsOXEJklayGqrXIBzk3w1yQ3AfVV1Y5NBSZIW1+U+l4lGi1XVdmB7w7FIklYJ5xaTJDXOO/Qlqa86fJ+LyUWS+qjjU+6bXCSprzqcXOxzkSQ1zspFkvqqw5WLyUWSeijY5yJJmoYOJxf7XCRJjTO5SFIfNTjlfpLNSe5MsjPJuxd4379OUknOWuyYJhdJ6qsGJq5Msh74AHAJcCrwhiSnHuJ9RwP/CRhrLkmTiyT1VTOzIp8N7Kyqu6tqD3A1cPkh3vffgV8Ddo0T2nQ79KuoffumeopxpO0AhrrU99aVa9IVXfmWtbHtADqpK+OO2v8pyXQ+Tk8A7hnZvhc456DzJj8InFRVn0ryrnEO2pV/NUnSEi1hKPJxSW4e2d5aVVvHOkeyDvgfwM8sJTaTiyT11fjJ5cGqmq8T/j7gpJHtE4ev7Xc0g6cNX58E4DnAtiSXVdVowjqIyUWS+qi5p0zeBGxKcgqDpPJ64KcOnKbqMeC4/dtJrgfeuVBigS40IkqSWlNV+4C3ATuAvwU+XlW3J7kyyWWTHtfKRZJ6qqnpXw71dOGqumKe954/zjFNLpLUV10agjqHyUWSeqrLE1fa5yJJapyViyT1VYcrF5OLJPVRc0ORp8LkIkk9FLo9jZN9LpKkxlm5SFJf2SwmSWraqh2KnOQdSb7UVDCSpCVo5nkuUzFxcklyOHBmg7FIklaJ5VQubwE+2FQgkqQlWm2VS5KNwPlVdd0h9m1JcleSB56sJ5YdoCTpEGrQ5zLO0oZJK5c3AVcdakdVba2qTVV1/NNy5OSRSZIWttoqF+AHgLcm+TTw4iRvbzAmSVLPTTQUuap+Yf96ki9V1fubC0mSNI4uD0Ve9n0uVfWqJgKRJC3Rak4ukqR2dLlycW4xSVLjrFwkqY+ccl+SNBUmF0lSk4J9LpKkNcbKRZL6qsOVi8lFknoq1d3sYnKRpD7q+Ggx+1wkSY2zcpGknuryaLHpJpcq2Lt3qqcYK4y2AxhK2wGM8Jp0U1eaEja2HUAntf9dfN3MnBe68ot8CO1fLUnSRLpcuXTli5IkaRWxcpGkvupw5WJykaQ+qm43i5lcJKmvOpxc7HORJDXOykWSeqjrsyKbXCSpr5xbTJLUtC5XLva5SJIaZ+UiSX3U8VmRTS6S1FOZbTuC+ZlcJKmvOly52OciSWrcRJVLktOArcAMsBP491UdHhMnSavQahwtdmdVvaKqzh1un9VUQJKkMRSD+1zGWVowUeVSVaNPANsN3NNMOJKkca3GyoUklyX5G+DZwENz9l2aZOte9iw3PklSD02cXKpqW1WdBtwLvHrOvmuqastGDltufJKk+dSYSwsm7dA/vKp2Dze/BTzZXEiSpMWs1okrNyf5+eH6XcBnGopHkjSOFjvrxzFph/4ngU82HIskaZXwDn1J6qnV2CwmSWqbyUWS1LQuVy7OLSZJapyViyT1UQGz3S1dTC6S1FfdzS0mF0nqK/tcJElrislFkvqqoSn3k2xOcmeSnUnefYj9P5/kjiS3JflckhcsdkyTiyT1VGq8ZcFjJOuBDwCXAKcCb0hy6py3/RVwVlWdDnwCeO9isZlcJKmPxp0RefHC5WxgZ1XdXVV7gKuByw86VdXnq+o7w80vAycudlCTiyStbSdw8AMf7x2+Np+3AH+62EGnOlqsgJqZneYpxhL2Lv6mFdClgR1pO4ChrlyTrlyPrujSt86NbQfQIdn33d+YwZT7Y/8GHZfk5pHtrVW1dcnnT97I4LH25y32XociS1Jfjf/d/cGqOmueffcBJ41snzh87SBJLgL+C3DeyPO85mVykaSeWkLlspCbgE1JTmGQVF4P/NRB50n+JfA7wOaq+uY4B+1S9StJWmFVtQ94G7AD+Fvg41V1e5Irk1w2fNuvA0cBf5DkliTbFjuulYsk9dF4I8HGO1TVdmD7nNeuGFm/aKnHNLlIUi+twsccS5La59xikqQ1xcpFkvrKZjFJUqMK0v496vMyuUhSX3W4crHPRZLUOCsXSeqr7hYuJhdJ6quGpn+ZiomaxZKck+SGJF9K8r6mg5IkjaGhJ1FOw6R9Lv8E/HBVvQp4VpKXNBiTJKnnJmoWq6pvjGzuBWaaCUeSNJZiKVPur7hljRZLcjpwfFXdMfLaliR3JXlgVz2x7AAlSU8VitR4SxsmTi5JjgV+k8EjLw+oqq1Vtamqjj8iRy43PknSfFZbn0uSDcBHgHfOaSKTJGniyuW1wEuB9ya5PsnLG4xJkjSODlcuk3bofwz4WMOxSJLG1fEOfW+ilKSeWnU3UUqStBArF0nqqw5XLiYXSeql9jrrx2FykaQ+KjqdXOxzkSQ1zspFkvrKociSpKZ1eSiyyUWS+qrDycU+F0lS46xcJKmPCpjtbuVicpGkXlrT97kUVPvDGaojz8kMe9sO4YCu/Eim7QCGvB5zdOhDa137HyEAbGw7ACBzK5UO/TvNZZ+LJKlxNotJUl91uHIxuUhSH9mhL0lqXjf6tOdjn4skqXFWLpLUV/a5SJIaZZ+LJGkqOly52OciSWqclYsk9VWHKxeTiyT10pqeW0ySNBUFzHqfiyRpDZkouSR5XpK/TLIridWPJLWharylBZNWLg8DFwJfbjAWSdJSdDi5TFR1VNUuYFfSmadPSNIaU2vvJsoklwKXPo2jpnF4SVLHTaVDv6quqaotG9KFZ7dJ0ipUUDU71tIGO+Mlqa863Cw26WixjUmuBc4AdiQ5p9mwJEmLWoUd+nuBixqORZK0StgsJkl9VNXpO/RNLpLUV84tJklqWnW4cnFuMUlS46xcJKmXnHJfktS0otP3uZhcJKmvWrr7fhz2uUiSGmdykaQeKqBma6xlMUk2J7kzyc4k7z7E/sOT/P5w/41JTl7smCYXSeqjqkGz2DjLApKsBz4AXAKcCrwhyalz3vYW4JGq+hfA+4BfWyw8k4sk9VRDlcvZwM6quruq9gBXA5fPec/lwAeH658ALswiD/QyuUjS2nYCcM/I9r3D1w75nqraBzwGfO9CB53qaLFv1yMPfnbv1f+0zMM8H/jnJuJZpmcyuKBta+Z6PLnsI6yu67F8Xo+n8pocrInr8YL9K9/mkR3Xzn78uDH/3BFJbh7Z3lpVW5cZy4Kmmlyq6vjlHiPJA1V1VhPxLDOOrVW1pQNxeD0OjsPrcXAcnbge4DU5RByNXo+q2tzQoe4DThrZPnH42qHec2+SDQwS5UMLHbQPzWKPth3A0DVtBzDk9TiY1+NgXbke4DWZqyvXY66bgE1JTklyGPB6YNuc92wDfnq4/pPAdVULTw/Qh5sou1BWU1Vd+cHwehzM63GwTlwP8JrM1aHrcZCq2pfkbcAOYD3wu1V1e5IrgZurahvwf4APJ9kJPMwgAS2oD8llqu2CPeT1OJjX42Bej6fymiyiqrYD2+e8dsXI+i7gtUs5ZhapbCRJWrLO9rkkeV6Sv0yya9iB1FYc5yS5IcmXkryvxThOG8bxxST/d7Ex5isQzzuSfKnF85+c5P4k1yf5TFtxDGN5c5LPDWOZO4RzpWLYPDz/9Um+nuQ1bcQxjOXpST41jOWTSQ5vKY4NSa5O8vkk723h/E/5DEvyvuHv8G+sdDwrrbPJhUG73oXAl1uO45+AH66qVwHPSvKSluK4s6peUVXnDrdbG/0y/LA4s63zj/hsVZ1fVf+qrQCGyeS8qrpwGMvcUTYroqo+PTz/+QyG3V7bRhxDm4Ebh7F8Zbjdhh8Hbq2qC4CnJTljhc9/0GdYkh8Ejhr+Dh+W5KUrHM+K6mxyqapdVfVIB+L4xrC9EWAvMNNSHHtHNndz8E1PK+0tfPdu3TZdMPwW+I4WY7gYWD+sXN4/nEqjNUleCNxfVY+3GMbfA0cO149hkSGrU/RC4Lbh+i3AK1by5If4DHsZ8Nnh+rXAy1cynpXW2eTSNUlOB46vqjtajOGyJH8DPJuWfmGTbATOr6rr2jj/iK8D3w9cAFw0/Pdpw7OBw6rqQuA7PHXajJX2E8AftRzDXcDLk9zOoMK+oaU47gTOG65fwCDRtekY4FvD9cdoP56pMrmMIcmxwG8y+MbemqraVlWnMZie4dUthfEm4KqWzn1AVe2uqieGU1H8CXBaS6E8BnxhuH4d8KKW4tjvUp56j8JK+2ngmqp6MfAp4I0txXENg+awzzGo9u9vKY79HgOeMVx/Bt25/2YqTC6LGHbEfQR4Z1V9o8U4RjtFv0UTE7hM5geAtyb5NPDiJG9vI4gkR49svpJBU0wbbgD2V01nAv/QUhwkeQ6wp6raaoY6EAqD/gaABxnczb3iqmqmqt4+rCpnGNzH0aa/YNAHA3AR7fcnT1Vnk0uSjUmuBc4AdiQ5p6VQXgu8FHjvcPRLW+2km5N8IckXGDTFtDJCqqp+oaouHk49cXtVvb+NOIBzk3w1yQ3AfVV1YxtBVNUtwJNJrmfwc/KJNuIYuhz4ZIvn3+8q4HXDa/JvgY+2EUSSE4a/s9cBN6z0YIu5n2HARmBXki8CM1X1lZWMZ6V5n4skqXGdrVwkSf1lcpEkNc7kIklqnMlFktQ4k4skqXEmF0lS40wukqTGmVwkSY37/+wk9ZyRv3jkAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 1008x576 with 2 Axes>"
]
},
"metadata": {
"needs_background": "light"
},
"output_type": "display_data"
}
],
"source": [
"boltz_temp = 0.01\n",
"v = value_iter(S, A, nvmdp.reward_func, nvmdp.transition_func, 0.8, lambda Q, t=boltz_temp: np.exp((1/t)*Q) / (np.exp((1/t)*Q).sum()), 100)\n",
"V = get_value_grid(v, nvmdp, s_to_idx)\n",
"\n",
"plt.figure(figsize=(14,8))\n",
"nvmdp.visualize_grid(V, trajectories=None, show_colorbar=True, show_rewards_colorbar=False, goal_marker=\"*c\", new_fig=False, subplot_str=\"122\", \n",
" state_space_cmap=False, title=\"Value\")"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([[0.16, 0.21, 0.26, 0.33, 0.41, 0.51, 0.64, 0.8 , 1. , 0. ],\n",
" [0.13, 0.17, 0.21, 0.26, 0.33, 0.41, 0.51, 0.64, 0.8 , 1. ],\n",
" [0.1 , 0.13, 0.17, 0.21, 0.26, 0.33, 0.41, 0.51, 0.64, 0.8 ],\n",
" [0.08, 0.11, 0.13, 0.17, 0.21, 0.26, 0.33, 0.41, 0.51, 0.64],\n",
" [0.06, 0.08, 0.11, 0.13, 0.17, 0.21, 0.26, 0.33, 0.41, 0.51],\n",
" [0.05, 0.07, 0.08, 0.11, 0.13, 0.17, 0.21, 0.26, 0.33, 0.41],\n",
" [0.04, 0.05, 0.07, 0.08, 0.11, 0.13, 0.17, 0.21, 0.26, 0.33],\n",
" [0.03, 0.04, 0.05, 0.07, 0.08, 0.11, 0.13, 0.17, 0.21, 0.26],\n",
" [0.02, 0.03, 0.04, 0.05, 0.07, 0.08, 0.11, 0.13, 0.17, 0.21],\n",
" [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.08, 0.1 , 0.13, 0.16]],\n",
" dtype=float32)"
]
},
"execution_count": 8,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"V.round(2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "irl_python3",
"language": "python",
"name": "irl_python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
@yrevar
Copy link
Author

yrevar commented Dec 15, 2018

PDF

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment