Skip to content

Instantly share code, notes, and snippets.

@franroldans

franroldans/config_sac_her_pnp.yaml

Last active March 2, 2022 07:30
Show Gist options
  • Save franroldans/5223cef6b834a2286ee03d0fdd2ad29c to your computer and use it in GitHub Desktop.
Save franroldans/5223cef6b834a2286ee03d0fdd2ad29c to your computer and use it in GitHub Desktop.
Download ZIP
Copy of SAC_HER_Mujoco.ipynb
Raw
config_sac_her_pnp.yaml
# environment
info: Train_OfflineHER_FetchPickAndPlace
env: FetchPickAndPlace-v1
use_her: True
norm_obs: False
seed: 5
strategy: offline
curriculum: False
# buffer
use_PER: False
per_alpha: 0.7
per_beta: 0.5
buffer_size: 6400000
replay_k: 8
# network
hidden_sizes: [256, 256]
# train
actor_lr: 1.0e-3
critic_lr: 1.0e-3
start_timesteps: 64000 # warmup
epoch: 10 # save times
step_per_epoch: 64000 # save interval (50repeat)
step_per_collect: 6400 #(64path) after collect #, update
update_per_step: 0.1 # repeat update time
estimation_step: 1 # look ahead time steps
batch_size: 4096
# parallel
device: cuda # cuda
training_num: 64
test_num: 8
# Algorithm
alpha: 0.2 # entropy regularization coefficient
auto_alpha: True
alpha_lr: 0.0003 # works if open auto alpha
tau: 0.005
gamma: 0.95
# dir
logdir: log
resume_path: null
# render
watch_train: False # if show demo directly
record_test: False
render: 0.00 # render time rate
Display the source blob
Display the rendered blob
Raw
copy-of-sac_her_mujoco.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "Copy of SAC_HER_Mujoco.ipynb",
"provenance": [],
"authorship_tag": "ABX9TyOQbGij7AIwbEmXfdkpOJAE",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/franroldans/5223cef6b834a2286ee03d0fdd2ad29c/copy-of-sac_her_mujoco.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "YdcBciGqNFuu",
"outputId": "1921073e-74d0-41e6-9dd0-2c1486ba6c70"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Selecting previously unselected package libgl1-mesa-glx:amd64.\n",
"(Reading database ... 155320 files and directories currently installed.)\n",
"Preparing to unpack .../0-libgl1-mesa-glx_20.0.8-0ubuntu1~18.04.1_amd64.deb ...\n",
"Unpacking libgl1-mesa-glx:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
"Selecting previously unselected package libglew2.0:amd64.\n",
"Preparing to unpack .../1-libglew2.0_2.0.0-5_amd64.deb ...\n",
"Unpacking libglew2.0:amd64 (2.0.0-5) ...\n",
"Selecting previously unselected package libglew-dev:amd64.\n",
"Preparing to unpack .../2-libglew-dev_2.0.0-5_amd64.deb ...\n",
"Unpacking libglew-dev:amd64 (2.0.0-5) ...\n",
"Selecting previously unselected package libglfw3:amd64.\n",
"Preparing to unpack .../3-libglfw3_3.2.1-1_amd64.deb ...\n",
"Unpacking libglfw3:amd64 (3.2.1-1) ...\n",
"Selecting previously unselected package patchelf.\n",
"Preparing to unpack .../4-patchelf_0.9-1_amd64.deb ...\n",
"Unpacking patchelf (0.9-1) ...\n",
"Selecting previously unselected package libosmesa6:amd64.\n",
"Preparing to unpack .../5-libosmesa6_20.0.8-0ubuntu1~18.04.1_amd64.deb ...\n",
"Unpacking libosmesa6:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
"Selecting previously unselected package libosmesa6-dev:amd64.\n",
"Preparing to unpack .../6-libosmesa6-dev_20.0.8-0ubuntu1~18.04.1_amd64.deb ...\n",
"Unpacking libosmesa6-dev:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
"Setting up patchelf (0.9-1) ...\n",
"Setting up libosmesa6:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
"Setting up libglfw3:amd64 (3.2.1-1) ...\n",
"Setting up libgl1-mesa-glx:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
"Setting up libglew2.0:amd64 (2.0.0-5) ...\n",
"Setting up libglew-dev:amd64 (2.0.0-5) ...\n",
"Setting up libosmesa6-dev:amd64 (20.0.8-0ubuntu1~18.04.1) ...\n",
"Processing triggers for man-db (2.8.3-2ubuntu0.1) ...\n",
"Processing triggers for libc-bin (2.27-3ubuntu1.3) ...\n",
"/sbin/ldconfig.real: /usr/local/lib/python3.7/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
"\n",
"/sbin/ldconfig.real: /usr/local/lib/python3.7/dist-packages/ideep4py/lib/libmkldnn.so.0 is not a symbolic link\n",
"\n",
"Collecting mujoco-py<2.2,>=2.1\n",
" Downloading mujoco_py-2.1.2.14-py3-none-any.whl (2.4 MB)\n",
"\u001b[K |████████████████████████████████| 2.4 MB 4.4 MB/s \n",
"\u001b[?25hCollecting fasteners~=0.15\n",
" Downloading fasteners-0.17.3-py3-none-any.whl (18 kB)\n",
"Requirement already satisfied: Cython>=0.27.2 in /usr/local/lib/python3.7/dist-packages (from mujoco-py<2.2,>=2.1) (0.29.28)\n",
"Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.7/dist-packages (from mujoco-py<2.2,>=2.1) (1.21.5)\n",
"Requirement already satisfied: imageio>=2.1.2 in /usr/local/lib/python3.7/dist-packages (from mujoco-py<2.2,>=2.1) (2.4.1)\n",
"Requirement already satisfied: cffi>=1.10 in /usr/local/lib/python3.7/dist-packages (from mujoco-py<2.2,>=2.1) (1.15.0)\n",
"Collecting glfw>=1.4.0\n",
" Downloading glfw-2.5.0-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38-none-manylinux2014_x86_64.whl (205 kB)\n",
"\u001b[K |████████████████████████████████| 205 kB 51.4 MB/s \n",
"\u001b[?25hRequirement already satisfied: pycparser in /usr/local/lib/python3.7/dist-packages (from cffi>=1.10->mujoco-py<2.2,>=2.1) (2.21)\n",
"Requirement already satisfied: pillow in /usr/local/lib/python3.7/dist-packages (from imageio>=2.1.2->mujoco-py<2.2,>=2.1) (7.1.2)\n",
"Installing collected packages: glfw, fasteners, mujoco-py\n",
"Successfully installed fasteners-0.17.3 glfw-2.5.0 mujoco-py-2.1.2.14\n",
"Compiling /usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.pyx because it changed.\n",
"[1/1] Cythonizing /usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.pyx\n",
"running build_ext\n",
"building 'mujoco_py.cymj' extension\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/gl\n",
"x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-pX47U3/python3.7-3.7.12=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-pX47U3/python3.7-3.7.12=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/mujoco_py -I/root/.mujoco/mujoco210/include -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/local/lib/python3.7/dist-packages/mujoco_py/vendor/egl -I/usr/include/python3.7m -c /usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.c -o /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.o -fopenmp -w\n",
"x86_64-linux-gnu-gcc -pthread -Wno-unused-result -Wsign-compare -DNDEBUG -g -fwrapv -O2 -Wall -g -fdebug-prefix-map=/build/python3.7-pX47U3/python3.7-3.7.12=. -fstack-protector-strong -Wformat -Werror=format-security -g -fdebug-prefix-map=/build/python3.7-pX47U3/python3.7-3.7.12=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 -fPIC -I/usr/local/lib/python3.7/dist-packages/mujoco_py -I/root/.mujoco/mujoco210/include -I/usr/local/lib/python3.7/dist-packages/numpy/core/include -I/usr/local/lib/python3.7/dist-packages/mujoco_py/vendor/egl -I/usr/include/python3.7m -c /usr/local/lib/python3.7/dist-packages/mujoco_py/gl/eglshim.c -o /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/gl/eglshim.o -fopenmp -w\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/lib.linux-x86_64-3.7\n",
"creating /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/lib.linux-x86_64-3.7/mujoco_py\n",
"x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -Wl,-Bsymbolic-functions -Wl,-z,relro -g -fdebug-prefix-map=/build/python3.7-pX47U3/python3.7-3.7.12=. -fstack-protector-strong -Wformat -Werror=format-security -Wdate-time -D_FORTIFY_SOURCE=2 /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/cymj.o /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/temp.linux-x86_64-3.7/usr/local/lib/python3.7/dist-packages/mujoco_py/gl/eglshim.o -L/root/.mujoco/mujoco210/bin -Wl,--enable-new-dtags,-R/root/.mujoco/mujoco210/bin -lmujoco210 -lglewegl -o /usr/local/lib/python3.7/dist-packages/mujoco_py/generated/_pyxbld_2.1.2.14_37_linuxgpuextensionbuilder/lib.linux-x86_64-3.7/mujoco_py/cymj.cpython-37m-x86_64-linux-gnu.so -fopenmp\n"
]
}
],
"source": [
"#Include this at the top of your colab code\n",
"import os\n",
"if not os.path.exists('.mujoco_setup_complete'):\n",
" # Get the prereqs\n",
" !apt-get -qq update\n",
" !apt-get -qq install -y libosmesa6-dev libgl1-mesa-glx libglfw3 libgl1-mesa-dev libglew-dev patchelf\n",
" # Get Mujoco\n",
" !mkdir ~/.mujoco\n",
" !wget -q https://mujoco.org/download/mujoco210-linux-x86_64.tar.gz -O mujoco.tar.gz\n",
" !tar -zxf mujoco.tar.gz -C \"$HOME/.mujoco\"\n",
" !rm mujoco.tar.gz\n",
" # Add it to the actively loaded path and the bashrc path (these only do so much)\n",
" !echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.mujoco/mujoco210/bin' >> ~/.bashrc\n",
" !echo 'export LD_PRELOAD=$LD_PRELOAD:/usr/lib/x86_64-linux-gnu/libGLEW.so' >> ~/.bashrc\n",
" !echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/nvidia' >> ~/.bashrc\n",
" # THE ANNOYING ONE, FORCE IT INTO LDCONFIG SO WE ACTUALLY GET ACCESS TO IT THIS SESSION\n",
" !echo \"/root/.mujoco/mujoco210/bin\" > /etc/ld.so.conf.d/mujoco_ld_lib_path.conf\n",
" !ldconfig\n",
" # Install Mujoco-py\n",
" !pip3 install -U 'mujoco-py<2.2,>=2.1'\n",
" # run once\n",
" !touch .mujoco_setup_complete\n",
"\n",
"try:\n",
" if _mujoco_run_once:\n",
" pass\n",
"except NameError:\n",
" _mujoco_run_once = False\n",
"if not _mujoco_run_once:\n",
" # Add it to the actively loaded path and the bashrc path (these only do so much)\n",
" try:\n",
" os.environ['LD_LIBRARY_PATH']=os.environ['LD_LIBRARY_PATH'] + ':/root/.mujoco/mujoco210/bin'\n",
" except KeyError:\n",
" os.environ['LD_LIBRARY_PATH']='/root/.mujoco/mujoco210/bin'\n",
" try:\n",
" os.environ['LD_PRELOAD']=os.environ['LD_PRELOAD'] + ':/usr/lib/x86_64-linux-gnu/libGLEW.so'\n",
" except KeyError:\n",
" os.environ['LD_PRELOAD']='/usr/lib/x86_64-linux-gnu/libGLEW.so'\n",
" try:\n",
" os.environ['LD_LIBRARY_PATH']=os.environ['LD_LIBRARY_PATH'] + ':/usr/lib/nvidia'\n",
" except KeyError:\n",
" os.environ['LD_LIBRARY_PATH']=':/usr/lib/nvidia'\n",
" # presetup so we don't see output on first env initialization\n",
"import mujoco_py\n",
"_mujoco_run_once = True"
]
},
{
"cell_type": "code",
"source": [
" !pip3 install tianshou"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "0-bkYDD2NJXW",
"outputId": "71d5d7f1-b22f-4607-8518-4fd84936afe4"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Collecting tianshou\n",
" Downloading tianshou-0.4.5-py3-none-any.whl (129 kB)\n",
"\u001b[K |████████████████████████████████| 129 kB 3.7 MB/s \n",
"\u001b[?25hRequirement already satisfied: h5py>=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tianshou) (3.1.0)\n",
"Requirement already satisfied: torch>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from tianshou) (1.10.0+cu111)\n",
"Requirement already satisfied: numpy>1.16.0 in /usr/local/lib/python3.7/dist-packages (from tianshou) (1.21.5)\n",
"Requirement already satisfied: gym<0.20,>=0.15.4 in /usr/local/lib/python3.7/dist-packages (from tianshou) (0.17.3)\n",
"Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from tianshou) (4.62.3)\n",
"Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.7/dist-packages (from tianshou) (0.51.2)\n",
"Requirement already satisfied: tensorboard>=2.5.0 in /usr/local/lib/python3.7/dist-packages (from tianshou) (2.8.0)\n",
"Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym<0.20,>=0.15.4->tianshou) (1.5.0)\n",
"Requirement already satisfied: cloudpickle<1.7.0,>=1.2.0 in /usr/local/lib/python3.7/dist-packages (from gym<0.20,>=0.15.4->tianshou) (1.3.0)\n",
"Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym<0.20,>=0.15.4->tianshou) (1.4.1)\n",
"Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.10.0->tianshou) (1.5.2)\n",
"Requirement already satisfied: llvmlite<0.35,>=0.34.0.dev0 in /usr/local/lib/python3.7/dist-packages (from numba>=0.51.0->tianshou) (0.34.0)\n",
"Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from numba>=0.51.0->tianshou) (57.4.0)\n",
"Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym<0.20,>=0.15.4->tianshou) (0.16.0)\n",
"Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (0.6.1)\n",
"Requirement already satisfied: grpcio>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (1.43.0)\n",
"Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (0.37.1)\n",
"Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (1.0.1)\n",
"Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (2.23.0)\n",
"Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (1.8.1)\n",
"Requirement already satisfied: protobuf>=3.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (3.17.3)\n",
"Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (1.35.0)\n",
"Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (3.3.6)\n",
"Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (1.0.0)\n",
"Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard>=2.5.0->tianshou) (0.4.6)\n",
"Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from absl-py>=0.4->tensorboard>=2.5.0->tianshou) (1.15.0)\n",
"Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.5.0->tianshou) (4.8)\n",
"Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.5.0->tianshou) (4.2.4)\n",
"Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard>=2.5.0->tianshou) (0.2.8)\n",
"Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.5.0->tianshou) (1.3.1)\n",
"Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard>=2.5.0->tianshou) (4.11.1)\n",
"Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard>=2.5.0->tianshou) (3.10.0.2)\n",
"Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard>=2.5.0->tianshou) (3.7.0)\n",
"Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.5.0->tianshou) (0.4.8)\n",
"Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard>=2.5.0->tianshou) (3.0.4)\n",
"Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard>=2.5.0->tianshou) (2.10)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard>=2.5.0->tianshou) (2021.10.8)\n",
"Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard>=2.5.0->tianshou) (1.24.3)\n",
"Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.5.0->tianshou) (3.2.0)\n",
"Installing collected packages: tianshou\n",
"Successfully installed tianshou-0.4.5\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"!pip3 install stable-baselines3"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "j6odEXOuNMah",
"outputId": "b3bd1adb-106d-4473-878b-e13ac5fd715a"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Collecting stable-baselines3\n",
" Downloading stable_baselines3-1.4.0-py3-none-any.whl (176 kB)\n",
"\u001b[?25l\r\u001b[K |█▉ | 10 kB 24.2 MB/s eta 0:00:01\r\u001b[K |███▊ | 20 kB 12.6 MB/s eta 0:00:01\r\u001b[K |█████▋ | 30 kB 8.9 MB/s eta 0:00:01\r\u001b[K |███████▍ | 40 kB 3.6 MB/s eta 0:00:01\r\u001b[K |█████████▎ | 51 kB 3.5 MB/s eta 0:00:01\r\u001b[K |███████████▏ | 61 kB 4.1 MB/s eta 0:00:01\r\u001b[K |█████████████ | 71 kB 4.5 MB/s eta 0:00:01\r\u001b[K |██████████████▉ | 81 kB 4.5 MB/s eta 0:00:01\r\u001b[K |████████████████▊ | 92 kB 4.9 MB/s eta 0:00:01\r\u001b[K |██████████████████▌ | 102 kB 4.2 MB/s eta 0:00:01\r\u001b[K |████████████████████▍ | 112 kB 4.2 MB/s eta 0:00:01\r\u001b[K |██████████████████████▎ | 122 kB 4.2 MB/s eta 0:00:01\r\u001b[K |████████████████████████ | 133 kB 4.2 MB/s eta 0:00:01\r\u001b[K |██████████████████████████ | 143 kB 4.2 MB/s eta 0:00:01\r\u001b[K |███████████████████████████▉ | 153 kB 4.2 MB/s eta 0:00:01\r\u001b[K |█████████████████████████████▋ | 163 kB 4.2 MB/s eta 0:00:01\r\u001b[K |███████████████████████████████▌| 174 kB 4.2 MB/s eta 0:00:01\r\u001b[K |████████████████████████████████| 176 kB 4.2 MB/s \n",
"\u001b[?25hRequirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from stable-baselines3) (1.3.0)\n",
"Requirement already satisfied: torch>=1.8.1 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3) (1.10.0+cu111)\n",
"Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from stable-baselines3) (1.21.5)\n",
"Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from stable-baselines3) (3.2.2)\n",
"Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from stable-baselines3) (1.3.5)\n",
"Requirement already satisfied: gym<0.20,>=0.17 in /usr/local/lib/python3.7/dist-packages (from stable-baselines3) (0.17.3)\n",
"Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.7/dist-packages (from gym<0.20,>=0.17->stable-baselines3) (1.5.0)\n",
"Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from gym<0.20,>=0.17->stable-baselines3) (1.4.1)\n",
"Requirement already satisfied: future in /usr/local/lib/python3.7/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym<0.20,>=0.17->stable-baselines3) (0.16.0)\n",
"Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch>=1.8.1->stable-baselines3) (3.10.0.2)\n",
"Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3) (1.3.2)\n",
"Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3) (0.11.0)\n",
"Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3) (2.8.2)\n",
"Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->stable-baselines3) (3.0.7)\n",
"Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.1->matplotlib->stable-baselines3) (1.15.0)\n",
"Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->stable-baselines3) (2018.9)\n",
"Installing collected packages: stable-baselines3\n",
"Successfully installed stable-baselines3-1.4.0\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import time\n",
"import warnings\n",
"from os import listdir, makedirs\n",
"from typing import Any, Dict, Optional\n",
"import gym\n",
"import cv2\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import pandas as pd\n",
"import seaborn as sns\n",
"import torch\n",
"from stable_baselines3.common.callbacks import BaseCallback\n",
"from tensorflow.python.summary.summary_iterator import summary_iterator\n",
"from tianshou.data import (Batch, to_numpy)\n",
"from tqdm import tqdm\n",
"\n",
"\n",
"class LogStepsCallback(BaseCallback):\n",
" def __init__(self, log_dir, verbose=0):\n",
" self.log_dir = log_dir\n",
" super(LogStepsCallback, self).__init__(verbose)\n",
"\n",
" def _on_training_start(self) -> None:\n",
" self.results = pd.DataFrame(columns=['Reward', 'Done'])\n",
" print(\"Τraining starts!\")\n",
"\n",
" def _on_step(self) -> bool:\n",
" if 'reward' in self.locals:\n",
" keys = ['reward', 'done']\n",
" else:\n",
" keys = ['rewards', 'dones']\n",
" self.results.loc[len(self.results)] = [self.locals[keys[0]][0], self.locals[keys[1]][0]]\n",
" return True\n",
"\n",
" def _on_training_end(self) -> None:\n",
" self.results.to_csv(self.log_dir + 'training_data.csv', index=False)\n",
" print(\"Τraining ends!\")\n",
"\n",
"\n",
"class TqdmCallback(BaseCallback):\n",
" def __init__(self):\n",
" super().__init__()\n",
" self.progress_bar = None\n",
"\n",
" def _on_training_start(self):\n",
" self.progress_bar = tqdm(total=self.locals['total_timesteps'])\n",
"\n",
" def _on_step(self):\n",
" self.progress_bar.update(1)\n",
" return True\n",
"\n",
" def _on_training_end(self):\n",
" self.progress_bar.close()\n",
" self.progress_bar = None\n",
"\n",
"\n",
"def save_dict_to_file(dict, path, txt_name='hyperparameter_dict'):\n",
" f = open(path + '/' + txt_name + '.txt', 'w')\n",
" f.write(str(dict))\n",
" f.close()\n",
"\n",
"\n",
"def calc_episode_rewards(training_data):\n",
" # Calculate the rewards for each training episode\n",
" episode_rewards = []\n",
" temp_reward_sum = 0\n",
"\n",
" for step in range(training_data.shape[0]):\n",
" reward, done = training_data.iloc[step, :]\n",
" temp_reward_sum += reward\n",
" if done:\n",
" episode_rewards.append(temp_reward_sum)\n",
" temp_reward_sum = 0\n",
"\n",
" result = pd.DataFrame(columns=['Reward'])\n",
" result['Reward'] = episode_rewards\n",
" return result\n",
"\n",
"\n",
"def learning_curve(episode_rewards, log_dir, window=10):\n",
" # Calculate rolling window metrics\n",
" rolling_average = episode_rewards.rolling(window=window, min_periods=window).mean().dropna()\n",
" rolling_max = episode_rewards.rolling(window=window, min_periods=window).max().dropna()\n",
" rolling_min = episode_rewards.rolling(window=window, min_periods=window).min().dropna()\n",
"\n",
" # Change column name\n",
" rolling_average.columns = ['Average Reward']\n",
" rolling_max.columns = ['Max Reward']\n",
" rolling_min.columns = ['Min Reward']\n",
" rolling_data = pd.concat([rolling_average, rolling_max, rolling_min], axis=1)\n",
"\n",
" # Plot\n",
" sns.set()\n",
" plt.figure(0)\n",
" ax = sns.lineplot(data=rolling_data)\n",
" ax.fill_between(rolling_average.index, rolling_min.iloc[:, 0], rolling_max.iloc[:, 0], alpha=0.2)\n",
" ax.set_title('Learning Curve')\n",
" ax.set_ylabel('Reward')\n",
" ax.set_xlabel('Episodes')\n",
"\n",
" # Save figure\n",
" plt.savefig(log_dir + 'learning_curve' + str(window) + '.png')\n",
"\n",
"\n",
"def learning_curve_baselines(log_dir, window=10):\n",
" # Read data\n",
" training_data = pd.read_csv(log_dir + 'training_data.csv', index_col=None)\n",
"\n",
" # Calculate episode rewards\n",
" episode_rewards = calc_episode_rewards(training_data)\n",
"\n",
" learning_curve(episode_rewards=episode_rewards, log_dir=log_dir, window=window)\n",
"\n",
"\n",
"def learning_curve_tianshou(log_dir, window=10):\n",
" # Find event file\n",
" files = listdir(log_dir)\n",
" for f in files:\n",
" if 'events' in f:\n",
" event_file = f\n",
" break\n",
"\n",
" # Read episode rewards\n",
" episode_rewards_list = []\n",
" episode_rewards = pd.DataFrame(columns=['Reward'])\n",
" try:\n",
" for e in summary_iterator(log_dir + event_file):\n",
" if len(e.summary.value) > 0:\n",
" if e.summary.value[0].tag == 'train/reward':\n",
" episode_rewards_list.append(e.summary.value[0].simple_value)\n",
" except Exception as e:\n",
" pass\n",
" episode_rewards['Reward'] = episode_rewards_list\n",
"\n",
" # Learning curve\n",
" learning_curve(episode_rewards, log_dir, window=window)\n",
"\n",
"\n",
"def learning_curve_tianshou_multiple_runs(log_dirs, window=10):\n",
" episode_rewards_list = []\n",
" episode_rewards = pd.DataFrame(columns=['Reward'])\n",
"\n",
" for log_dir in log_dirs:\n",
" # Find event file\n",
" files = listdir(log_dir)\n",
" for f in files:\n",
" if 'events' in f:\n",
" event_file = f\n",
" break\n",
"\n",
" # Read episode rewards\n",
" try:\n",
" for e in summary_iterator(log_dir + event_file):\n",
" if len(e.summary.value) > 0:\n",
" if e.summary.value[0].tag == 'train/reward':\n",
" episode_rewards_list.append(e.summary.value[0].simple_value)\n",
" except Exception as e:\n",
" pass\n",
" episode_rewards['Reward'] = episode_rewards_list\n",
"\n",
" # Learning curve\n",
" learning_curve(episode_rewards, log_dir, window=window)\n",
"\n",
"\n",
"def collect_and_record(self, video_dir, n_step: Optional[int] = None, n_episode: Optional[int] = None,\n",
" random: bool = False, render: Optional[float] = None, no_grad: bool = True,\n",
" ) -> Dict[str, Any]:\n",
" \"\"\"Collect a specified number of step or episode.\n",
" To ensure unbiased sampling result with n_episode option, this function will\n",
" first collect ``n_episode - env_num`` episodes, then for the last ``env_num``\n",
" episodes, they will be collected evenly from each env.\n",
" :param int n_step: how many steps you want to collect.\n",
" :param int n_episode: how many episodes you want to collect.\n",
" :param bool random: whether to use random policy for collecting data. Default\n",
" to False.\n",
" :param float render: the sleep time between rendering consecutive frames.\n",
" Default to None (no rendering).\n",
" :param bool no_grad: whether to retain gradient in policy.forward(). Default to\n",
" True (no gradient retaining).\n",
" .. note::\n",
" One and only one collection number specification is permitted, either\n",
" ``n_step`` or ``n_episode``.\n",
" :return: A dict including the following keys\n",
" * ``n/ep`` collected number of episodes.\n",
" * ``n/st`` collected number of steps.\n",
" * ``rews`` array of episode reward over collected episodes.\n",
" * ``lens`` array of episode length over collected episodes.\n",
" * ``idxs`` array of episode start index in buffer over collected episodes.\n",
" \"\"\"\n",
" assert not self.env.is_async, \"Please use AsyncCollector if using async venv.\"\n",
" if n_step is not None:\n",
" assert n_episode is None, (\n",
" f\"Only one of n_step or n_episode is allowed in Collector.\"\n",
" f\"collect, got n_step={n_step}, n_episode={n_episode}.\"\n",
" )\n",
" assert n_step > 0\n",
" if not n_step % self.env_num == 0:\n",
" warnings.warn(\n",
" f\"n_step={n_step} is not a multiple of #env ({self.env_num}), \"\n",
" \"which may cause extra transitions collected into the buffer.\"\n",
" )\n",
" ready_env_ids = np.arange(self.env_num)\n",
" elif n_episode is not None:\n",
" assert n_episode > 0\n",
" ready_env_ids = np.arange(min(self.env_num, n_episode))\n",
" self.data = self.data[:min(self.env_num, n_episode)]\n",
" else:\n",
" raise TypeError(\n",
" \"Please specify at least one (either n_step or n_episode) \"\n",
" \"in AsyncCollector.collect().\"\n",
" )\n",
"\n",
" start_time = time.time()\n",
"\n",
" step_count = 0\n",
" episode_count = 0\n",
" episode_rews = []\n",
" episode_lens = []\n",
" episode_start_indices = []\n",
" img_array_list = []\n",
"\n",
" while True:\n",
" assert len(self.data) == len(ready_env_ids)\n",
" # restore the state: if the last state is None, it won't store\n",
" last_state = self.data.policy.pop(\"hidden_state\", None)\n",
"\n",
" # get the next action\n",
" if random:\n",
" self.data.update(\n",
" act=[self._action_space[i].sample() for i in ready_env_ids]\n",
" )\n",
" else:\n",
" if no_grad:\n",
" with torch.no_grad(): # faster than retain_grad version\n",
" # self.data.obs will be used by agent to get result\n",
" result = self.policy(self.data, last_state)\n",
" else:\n",
" result = self.policy(self.data, last_state)\n",
" # update state / act / policy into self.data\n",
" policy = result.get(\"policy\", Batch())\n",
" assert isinstance(policy, Batch)\n",
" state = result.get(\"state\", None)\n",
" if state is not None:\n",
" policy.hidden_state = state # save state into buffer\n",
" act = to_numpy(result.act)\n",
" if self.exploration_noise:\n",
" act = self.policy.exploration_noise(act, self.data)\n",
" self.data.update(policy=policy, act=act)\n",
"\n",
" # get bounded and remapped actions first (not saved into buffer)\n",
" action_remap = self.policy.map_action(self.data.act)\n",
" # step in env\n",
" result = self.env.step(action_remap, ready_env_ids) # type: ignore\n",
" obs_next, rew, done, info = result\n",
"\n",
" self.data.update(obs_next=obs_next, rew=rew, done=done, info=info)\n",
" if self.preprocess_fn:\n",
" self.data.update(\n",
" self.preprocess_fn(\n",
" obs_next=self.data.obs_next,\n",
" rew=self.data.rew,\n",
" done=self.data.done,\n",
" info=self.data.info,\n",
" policy=self.data.policy,\n",
" env_id=ready_env_ids,\n",
" )\n",
" )\n",
"\n",
" if render:\n",
" img_array = self.env.render(mode='rgb_array')\n",
" img_array = np.array(img_array)[0, :, :, :]\n",
" img_array = cv2.cvtColor(img_array, cv2.COLOR_BGR2RGB)\n",
" img_array_list.append(img_array)\n",
"\n",
" if render > 0 and not np.isclose(render, 0):\n",
" time.sleep(render)\n",
"\n",
" # add data into the buffer\n",
" ptr, ep_rew, ep_len, ep_idx = self.buffer.add(\n",
" self.data, buffer_ids=ready_env_ids\n",
" )\n",
"\n",
" # collect statistics\n",
" step_count += len(ready_env_ids)\n",
"\n",
" if np.any(done):\n",
" env_ind_local = np.where(done)[0]\n",
" env_ind_global = ready_env_ids[env_ind_local]\n",
" episode_count += len(env_ind_local)\n",
" episode_lens.append(ep_len[env_ind_local])\n",
" episode_rews.append(ep_rew[env_ind_local])\n",
" episode_start_indices.append(ep_idx[env_ind_local])\n",
" # now we copy obs_next to obs, but since there might be\n",
" # finished episodes, we have to reset finished envs first.\n",
" obs_reset = self.env.reset(env_ind_global)\n",
" if self.preprocess_fn:\n",
" obs_reset = self.preprocess_fn(\n",
" obs=obs_reset, env_id=env_ind_global\n",
" ).get(\"obs\", obs_reset)\n",
" self.data.obs_next[env_ind_local] = obs_reset\n",
" for i in env_ind_local:\n",
" self._reset_state(i)\n",
"\n",
" # remove surplus env id from ready_env_ids\n",
" # to avoid bias in selecting environments\n",
" if n_episode:\n",
" surplus_env_num = len(ready_env_ids) - (n_episode - episode_count)\n",
" if surplus_env_num > 0:\n",
" mask = np.ones_like(ready_env_ids, dtype=bool)\n",
" mask[env_ind_local[:surplus_env_num]] = False\n",
" ready_env_ids = ready_env_ids[mask]\n",
" self.data = self.data[mask]\n",
"\n",
" self.data.obs = self.data.obs_next\n",
"\n",
" if (n_step and step_count >= n_step) or \\\n",
" (n_episode and episode_count >= n_episode):\n",
" break\n",
"\n",
" # generate statistics\n",
" self.collect_step += step_count\n",
" self.collect_episode += episode_count\n",
" self.collect_time += max(time.time() - start_time, 1e-9)\n",
"\n",
" if n_episode:\n",
" self.data = Batch(\n",
" obs={}, act={}, rew={}, done={}, obs_next={}, info={}, policy={}\n",
" )\n",
" self.reset_env()\n",
"\n",
" if episode_count > 0:\n",
" rews, lens, idxs = list(\n",
" map(\n",
" np.concatenate,\n",
" [episode_rews, episode_lens, episode_start_indices]\n",
" )\n",
" )\n",
" else:\n",
" rews, lens, idxs = np.array([]), np.array([], int), np.array([], int)\n",
"\n",
" # Save video\n",
" width, height = img_array_list[0].shape[0], img_array_list[0].shape[1]\n",
" fourcc = cv2.VideoWriter_fourcc(*'mp4v')\n",
" makedirs(video_dir)\n",
" video = cv2.VideoWriter(video_dir + 'video.mp4', fourcc, 60, (width, height))\n",
" for img in img_array_list:\n",
" video.write(img)\n",
" video.release()\n",
" save_dict_to_file({'reward': rews[0], 'length': lens[0]}, video_dir, txt_name='episode_stats')\n",
"\n",
" return {\n",
" \"n/ep\": episode_count,\n",
" \"n/st\": step_count,\n",
" \"rews\": rews,\n",
" \"lens\": lens,\n",
" \"idxs\": idxs,\n",
" }\n",
"\n",
"\n",
"class Wrapper(gym.Wrapper):\n",
" \"\"\"Env wrapper for reward scale, action repeat and removing done penalty\"\"\"\n",
"\n",
" def __init__(self, env, action_repeat=3, reward_scale=5, rm_done=True):\n",
" super().__init__(env)\n",
" self.action_repeat = action_repeat\n",
" self.reward_scale = reward_scale\n",
" self.rm_done = rm_done\n",
"\n",
" def step(self, action):\n",
" r = 0.0\n",
" for _ in range(self.action_repeat):\n",
" obs, reward, done, info = self.env.step(action)\n",
" # remove done reward penalty\n",
" if not done or not self.rm_done:\n",
" r = r + reward\n",
" if done:\n",
" break\n",
" # scale reward\n",
" return obs, self.reward_scale * r, done, info"
],
"metadata": {
"id": "6IBbuBHENQXA"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import time\n",
"import warnings\n",
"from typing import Any, Callable, Dict, Optional\n",
"\n",
"import gym.spaces as space\n",
"import numpy as np\n",
"import torch\n",
"\n",
"from tianshou.data import Batch, Collector, ReplayBuffer, to_numpy\n",
"from tianshou.env import BaseVectorEnv\n",
"from tianshou.policy import BasePolicy\n",
"\n",
"\n",
"class HERCollector(Collector):\n",
" \"\"\"Hindsight Experience Replay Collector.\n",
" The collector will construct hindsight trajectory from achieved goals\n",
" after one trajectory is fully collected.\n",
" HER Collector provides two methods for relabel: `online` and `offline`.\n",
" For details, please refer to https://arxiv.org/abs/1707.01495\n",
" :param policy: an instance of the :class:`~tianshou.policy.BasePolicy` class.\n",
" :param env: a ``gym.Env`` environment or an instance of the\n",
" :class:`~tianshou.env.BaseVectorEnv` class.\n",
" :param dict_observation_space: a ``gym.spaces.Dict`` instance, which is\n",
" used to get goal and achieved goal in the flattened observation\n",
" :param function reward_fn: a function called to calculate reward.\n",
" Often defined as `env.compute_reward()`\n",
" :param str strategy: can be `online` or `offline`. `offline` strategy will add\n",
" relabeled data directly back to the buffer, while `online` strategy will store\n",
" the future achieved goal in `batch.info.achieved_goal`,\n",
" which can be used in `process_fn`to relabel data during the training process.\n",
" :param int replay_k: proportion of data to be relabeled.\n",
" For example, if `replay_k` is set to 4, then the collector will\n",
" generate 4 new trajectory with relabeled data.\n",
" :param buffer: an instance of the :class:`~tianshou.data.ReplayBuffer` class.\n",
" If set to None, it will not store the data. Default to None.\n",
" :param function preprocess_fn: a function called before the data has been added to\n",
" the buffer, see issue #42 and :ref:`preprocess_fn`. Default to None.\n",
" :param bool exploration_noise: determine whether the action needs to be modified\n",
" with corresponding policy's exploration noise. If so, \"policy.\n",
" exploration_noise(act, batch)\" will be called automatically to add the\n",
" exploration noise into action. Default to False.\n",
" .. note::\n",
" 1. According to the result reported in the paper, only future replay\n",
" is implemented in this collector.\n",
" 2. Make use your environment's `info` has `achieved_goal` attribution\n",
" before use `online` replay strategy. it will be used for a Batch place holder.\n",
" 3. Observation normalization in the environment is not recommended,\n",
" which bias the relabel.\n",
" 4. Success rate is also provided in the return to monitor the training\n",
" progress.\n",
" \"\"\"\n",
"\n",
" def __init__(\n",
" self,\n",
" policy: BasePolicy,\n",
" env: BaseVectorEnv,\n",
" dict_observation_space: space.Dict,\n",
" reward_fn: Callable[[np.ndarray, np.ndarray, Optional[dict]], np.ndarray],\n",
" replay_k: int = 4,\n",
" strategy: str = 'offline',\n",
" buffer: Optional[ReplayBuffer] = None,\n",
" preprocess_fn: Optional[Callable[..., Batch]] = None,\n",
" exploration_noise: bool = False,\n",
" ) -> None:\n",
" # HER need dict observation space\n",
" self.dict_observation_space = dict_observation_space\n",
" self.reward_fn = reward_fn\n",
" assert replay_k > 0, f'Replay k = {replay_k}, it must be a positive integer'\n",
" self.replay_k = replay_k\n",
" assert strategy == 'offline' or strategy == 'online', \\\n",
" f'Unsupported {strategy} replay strategy'\n",
" self.strategy = strategy\n",
" # Record the index of goal, achieved goal, and observation in obs,\n",
" # which save the 80% of time to get goal compared to\n",
" # use OpenAI gym's unflatten() function\n",
" current_idx = 0\n",
" self.obs_index_range = {}\n",
" for (key, s) in dict_observation_space.spaces.items():\n",
" self.obs_index_range[key] = np.arange(\n",
" current_idx, current_idx + s.shape[0]\n",
" )\n",
" current_idx += s.shape[0]\n",
" # assert type in base class\n",
" self.data: Batch\n",
" self.buffer: ReplayBuffer\n",
" super().__init__(policy, env, buffer, preprocess_fn, exploration_noise)\n",
"\n",
" def collect(\n",
" self,\n",
" n_step: Optional[int] = None,\n",
" n_episode: Optional[int] = None,\n",
" random: bool = False,\n",
" render: Optional[float] = None,\n",
" no_grad: bool = True,\n",
" ) -> Dict[str, Any]:\n",
" if n_step is not None:\n",
" assert n_episode is None, (\n",
" f\"Only one of n_step or n_episode is allowed in Collector.\"\n",
" f\"collect, got n_step={n_step}, n_episode={n_episode}.\"\n",
" )\n",
" assert n_step > 0\n",
" if not n_step % self.env_num == 0:\n",
" warnings.warn(\n",
" f\"n_step={n_step} is not a multiple of #env ({self.env_num}), \"\n",
" \"which may cause extra transitions collected into the buffer.\"\n",
" )\n",
" ready_env_ids = np.arange(self.env_num)\n",
" elif n_episode is not None:\n",
" assert n_episode > 0\n",
" ready_env_ids = np.arange(min(self.env_num, n_episode))\n",
" self.data = self.data[:min(self.env_num, n_episode)]\n",
" else:\n",
" raise TypeError(\n",
" \"Please specify at least one (either n_step or n_episode) \"\n",
" \"in AsyncCollector.collect().\"\n",
" )\n",
"\n",
" start_time = time.time()\n",
"\n",
" step_count = 0\n",
" episode_count = 0\n",
" episode_rews = []\n",
" episode_success = []\n",
" episode_lens = []\n",
" episode_start_indices = []\n",
"\n",
" while True:\n",
" assert len(self.data) == len(ready_env_ids)\n",
" # restore the state: if the last state is None, it won't store\n",
" last_state = self.data.policy.pop(\"hidden_state\", None)\n",
"\n",
" # get the next action\n",
" if random:\n",
" self.data.update(\n",
" act=[self._action_space[i].sample() for i in ready_env_ids]\n",
" )\n",
" else:\n",
" if no_grad:\n",
" with torch.no_grad(): # faster than retain_grad version\n",
" # self.data.obs will be used by agent to get result\n",
" result = self.policy(self.data, last_state)\n",
" else:\n",
" result = self.policy(self.data, last_state)\n",
" # update state / act / policy into self.data\n",
" policy = result.get(\"policy\", Batch())\n",
" assert isinstance(policy, Batch)\n",
" state = result.get(\"state\", None)\n",
" if state is not None:\n",
" policy.hidden_state = state # save state into buffer\n",
" act = to_numpy(result.act)\n",
" if self.exploration_noise:\n",
" act = self.policy.exploration_noise(act, self.data)\n",
" self.data.update(policy=policy, act=act)\n",
"\n",
" # get bounded and remapped actions first (not saved into buffer)\n",
" action_remap = self.policy.map_action(self.data.act)\n",
" # step in env\n",
" result = self.env.step(action_remap, ready_env_ids) # type: ignore\n",
" obs_next, rew, done, info = result\n",
"\n",
" self.data.update(obs_next=obs_next, rew=rew, done=done, info=info)\n",
" if self.preprocess_fn:\n",
" self.data.update(\n",
" self.preprocess_fn(\n",
" obs_next=self.data.obs_next,\n",
" rew=self.data.rew,\n",
" done=self.data.done,\n",
" info=self.data.info,\n",
" policy=self.data.policy,\n",
" env_id=ready_env_ids,\n",
" )\n",
" )\n",
"\n",
" if render:\n",
" self.env.render(mode='rgb_array')\n",
" if render > 0 and not np.isclose(render, 0):\n",
" time.sleep(render)\n",
"\n",
" # add data into the buffer\n",
" ptr, ep_rew, ep_len, ep_idx = self.buffer.add(\n",
" self.data, buffer_ids=ready_env_ids\n",
" )\n",
"\n",
" # collect statistics\n",
" step_count += len(ready_env_ids)\n",
"\n",
" if np.any(done):\n",
" env_ind_local = np.where(done)[0]\n",
" env_ind_global = ready_env_ids[env_ind_local]\n",
" episode_count += len(env_ind_local)\n",
" episode_lens.append(ep_len[env_ind_local])\n",
" episode_rews.append(ep_rew[env_ind_local])\n",
" episode_success.append(self.data[env_ind_local].info.is_success)\n",
" episode_start_indices.append(ep_idx[env_ind_local])\n",
" # now we copy obs_next to obs, but since there might be\n",
" # finished episodes, we have to reset finished envs first.\n",
" obs_reset = self.env.reset(env_ind_global)\n",
" if self.preprocess_fn:\n",
" obs_reset = self.preprocess_fn(\n",
" obs=obs_reset, env_id=env_ind_global\n",
" ).get(\"obs\", obs_reset)\n",
" self.data.obs_next[env_ind_local] = obs_reset\n",
" for i in env_ind_local:\n",
" self._reset_state(i)\n",
"\n",
" # remove surplus env id from ready_env_ids\n",
" # to avoid bias in selecting environments\n",
" if n_episode:\n",
" surplus_env_num = len(ready_env_ids) - (n_episode - episode_count)\n",
" if surplus_env_num > 0:\n",
" mask = np.ones_like(ready_env_ids, dtype=bool)\n",
" mask[env_ind_local[:surplus_env_num]] = False\n",
" ready_env_ids = ready_env_ids[mask]\n",
" self.data = self.data[mask]\n",
"\n",
" # use HER to create more trajectory\n",
" for env_id in env_ind_global: # enumerate env\n",
" # get recently collected data from buffer\n",
" env_buffer = self.buffer.buffers[env_id]\n",
" env_buffer_len = env_buffer.last_index[0] + 1\n",
" traj_len = ep_len[env_id]\n",
" obs_index_range = np.arange(\n",
" env_buffer_len - traj_len, env_buffer_len\n",
" ) % len(env_buffer)\n",
" original_trajectory = env_buffer[obs_index_range]\n",
" if self.strategy == 'offline':\n",
" new_trajactory_len = (\n",
" np.random.random(size=self.replay_k) * traj_len\n",
" ).astype(int) + 1\n",
" # relabel data and add back\n",
" for length in new_trajactory_len:\n",
" trajectory = Batch(original_trajectory[:length], copy=True)\n",
" new_goal = trajectory.obs_next[\n",
" length - 1, self.obs_index_range['achieved_goal']]\n",
" new_goals = np.repeat([new_goal], length, axis=0)\n",
" trajectory.obs[:, self.\n",
" obs_index_range['desired_goal']] = new_goals\n",
" trajectory.obs_next[:, self.obs_index_range['desired_goal']\n",
" ] = new_goals\n",
" trajectory.rew = self.reward_fn(\n",
" trajectory.obs_next[:, self.\n",
" obs_index_range['achieved_goal']],\n",
" new_goals, None\n",
" )\n",
" trajectory.done[-1] = True\n",
" for i in range(length):\n",
" env_buffer.add(trajectory[i])\n",
" elif self.strategy == 'online':\n",
" # record the achieved goal of future steps,\n",
" # to reduce the relabel time during the trainning\n",
" ag = original_trajectory.obs_next[:, self.obs_index_range[\n",
" 'achieved_goal']]\n",
" for i, idx in enumerate(obs_index_range):\n",
" env_buffer.info.achieved_goal[idx] = ag[i:]\n",
" \n",
" self.data.obs = self.data.obs_next\n",
"\n",
" if (n_step and step_count >= n_step) or \\\n",
" (n_episode and episode_count >= n_episode):\n",
" break\n",
"\n",
" # generate statistics\n",
" self.collect_step += step_count\n",
" self.collect_episode += episode_count\n",
" self.collect_time += max(time.time() - start_time, 1e-9)\n",
"\n",
" if n_episode:\n",
" self.data = Batch(\n",
" obs={}, act={}, rew={}, done={}, obs_next={}, info={}, policy={}\n",
" )\n",
" self.reset_env()\n",
"\n",
" if episode_count > 0:\n",
" rews, success, lens, idxs = list(\n",
" map(\n",
" np.concatenate, [\n",
" episode_rews, episode_success, episode_lens,\n",
" episode_start_indices\n",
" ]\n",
" )\n",
" )\n",
" rew_mean, rew_std = rews.mean(), rews.std()\n",
" len_mean, len_std = lens.mean(), lens.std()\n",
" else:\n",
" rews, success, lens, idxs = np.array([]), np.array(\n",
" []\n",
" ), np.array([], int), np.array([], int)\n",
" rew_mean = rew_std = len_mean = len_std = 0\n",
"\n",
" return {\n",
" \"n/ep\": episode_count,\n",
" \"n/st\": step_count,\n",
" \"rews\": rews,\n",
" \"success\": success,\n",
" \"lens\": lens,\n",
" \"idxs\": idxs,\n",
" \"rew\": rew_mean,\n",
" \"len\": len_mean,\n",
" \"rew_std\": rew_std,\n",
" \"len_std\": len_std,\n",
" }"
],
"metadata": {
"id": "MiA-k7XNOKpP"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"from typing import Any, Callable, Optional, Tuple, Union\n",
"\n",
"import gym.spaces as space\n",
"import numpy as np\n",
"import torch\n",
"\n",
"from tianshou.data import Batch, ReplayBuffer, to_numpy, to_torch_as\n",
"from tianshou.exploration import BaseNoise\n",
"from tianshou.policy import BasePolicy, SACPolicy\n",
"\n",
"\n",
"class SACHERPolicy(SACPolicy):\n",
" \"\"\"Implementation of Hindsight Experience Replay Based on SAC. arXiv:1707.01495.\n",
" The key difference is that we redesigned the process_fn to get relabel return,\n",
" if the replay strategy is `offline`, then it will behave the same as `SACPolicy`.\n",
" :param torch.nn.Module actor: the actor network following the rules in\n",
" :class:`~tianshou.policy.BasePolicy`. (s -> logits)\n",
" :param torch.optim.Optimizer actor_optim: the optimizer for actor network.\n",
" :param torch.nn.Module critic1: the first critic network. (s, a -> Q(s, a))\n",
" :param torch.optim.Optimizer critic1_optim: the optimizer for the first\n",
" critic network.\n",
" :param torch.nn.Module critic2: the second critic network. (s, a -> Q(s, a))\n",
" :param torch.optim.Optimizer critic2_optim: the optimizer for the second\n",
" critic network.\n",
" :param float tau: param for soft update of the target network. Default to 0.005.\n",
" :param float gamma: discount factor, in [0, 1]. Default to 0.99.\n",
" :param (float, torch.Tensor, torch.optim.Optimizer) or float alpha: entropy\n",
" regularization coefficient. Default to 0.2.\n",
" If a tuple (target_entropy, log_alpha, alpha_optim) is provided, then\n",
" alpha is automatically tuned.\n",
" :param bool reward_normalization: normalize the reward to Normal(0, 1).\n",
" Default to False.\n",
" :param BaseNoise exploration_noise: add a noise to action for exploration.\n",
" Default to None. This is useful when solving hard-exploration problem.\n",
" :param bool deterministic_eval: whether to use deterministic action (mean\n",
" of Gaussian policy) instead of stochastic action sampled by the policy.\n",
" Default to True.\n",
" :param bool action_scaling: whether to map actions from range [-1, 1] to range\n",
" [action_spaces.low, action_spaces.high]. Default to True.\n",
" :param str action_bound_method: method to bound action to range [-1, 1], can be\n",
" either \"clip\" (for simply clipping the action) or empty string for no bounding.\n",
" Default to \"clip\".\n",
" :param Optional[gym.Space] action_space: env's action space, mandatory if you want\n",
" to use option \"action_scaling\" or \"action_bound_method\". Default to None.\n",
" .. seealso::\n",
" Please refer to :class:`~tianshou.policy.SACPolicy` for more detailed\n",
" explanation.\n",
" \"\"\"\n",
"\n",
" def __init__(\n",
" self,\n",
" actor: torch.nn.Module,\n",
" actor_optim: torch.optim.Optimizer,\n",
" critic1: torch.nn.Module,\n",
" critic1_optim: torch.optim.Optimizer,\n",
" critic2: torch.nn.Module,\n",
" critic2_optim: torch.optim.Optimizer,\n",
" reward_fn: Callable[[np.ndarray, np.ndarray, Optional[dict]], np.ndarray],\n",
" tau: float = 0.005,\n",
" gamma: float = 0.99,\n",
" alpha: Union[float, Tuple[float, torch.Tensor, torch.optim.Optimizer]] = 0.2,\n",
" reward_normalization: bool = False,\n",
" estimation_step: int = 1,\n",
" exploration_noise: Optional[BaseNoise] = None,\n",
" deterministic_eval: bool = True,\n",
" dict_observation_space: space.Dict = None,\n",
" future_k: float = 4,\n",
" strategy: str = 'offline',\n",
" **kwargs: Any,\n",
" ) -> None:\n",
" super().__init__(\n",
" actor, actor_optim, critic1, critic1_optim, critic2, critic2_optim, tau,\n",
" gamma, alpha, reward_normalization, estimation_step, exploration_noise,\n",
" deterministic_eval, **kwargs\n",
" )\n",
" self.future_k = future_k\n",
" self.strategy = strategy\n",
" self.future_p = 1 - (1. / (1 + future_k))\n",
" self.reward_fn = reward_fn\n",
" # get index information of observation\n",
" self.dict_observation_space = dict_observation_space\n",
" current_idx = 0\n",
" self.index_range = {}\n",
" for (key, s) in dict_observation_space.spaces.items():\n",
" self.index_range[key] = np.arange(current_idx, current_idx + s.shape[0])\n",
" current_idx += s.shape[0]\n",
"\n",
" def process_fn(\n",
" self, batch: Batch, buffer: ReplayBuffer, indices: np.ndarray\n",
" ) -> Batch:\n",
" # Step1: get all index needed\n",
" if self.strategy == 'offline':\n",
" return super(SACHERPolicy, self).process_fn(batch, buffer, indices)\n",
" assert not self._rew_norm, \\\n",
" \"Reward normalization in computing n-step returns is unsupported now.\"\n",
" end_flag = buffer.done.copy()\n",
" end_flag[buffer.unfinished_index()\n",
" ] = True # consider unfinished case: remove it\n",
" bsz = len(indices) # get indice of sampled transitions\n",
" indices = [indices] # turn to list, prepare for expand next state e.g. [1,3]\n",
" for _ in range(self._n_step - 1):\n",
" indices.append(\n",
" buffer.next(indices[-1])\n",
" ) # append next state index e.g. [[1,3][2,4]]\n",
" indices = np.stack(indices)\n",
" terminal = indices[-1] # next state\n",
"\n",
" # Step2: sample new goal\n",
" batch = buffer[terminal] # batch.obs: s_{t+n}\n",
" new_goal = batch.obs_next[:, self.index_range['desired_goal']]\n",
" for i in range(bsz):\n",
" if np.random.random() < self.future_p:\n",
" goals = batch.info.achieved_goal[i]\n",
" if len(goals) != 0:\n",
" new_goal[i] = goals[int(np.random.random() * len(goals))]\n",
"\n",
" # Step3: relabel batch's obs, obs_next, reward, calculate Q\n",
" batch.obs[:, self.index_range['desired_goal']] = new_goal\n",
" batch.obs_next[:, self.index_range['desired_goal']] = new_goal\n",
" batch.rew = self.reward_fn(\n",
" batch.obs_next[:, self.index_range['achieved_goal']], new_goal, None\n",
" )\n",
" with torch.no_grad():\n",
" obs_next_result = self(batch, input='obs_next')\n",
" a_ = obs_next_result.act\n",
" target_q_torch = torch.min(\n",
" self.critic1_old(batch.obs_next, a_),\n",
" self.critic2_old(batch.obs_next, a_),\n",
" ) - self._alpha * obs_next_result.log_prob\n",
" target_q = to_numpy(target_q_torch.reshape(bsz, -1))\n",
" target_q = target_q * BasePolicy.value_mask(buffer, terminal).reshape(-1, 1)\n",
"\n",
" # Step4: calculate N step return\n",
" gamma_buffer = np.ones(self._n_step + 1)\n",
" for i in range(1, self._n_step + 1):\n",
" gamma_buffer[i] = gamma_buffer[i - 1] * self._gamma\n",
" target_shape = target_q.shape\n",
" bsz = target_shape[0]\n",
" # change target_q to 2d array\n",
" target_q = target_q.reshape(bsz, -1)\n",
" returns = np.zeros(target_q.shape) # n_step returrn\n",
" gammas = np.full(indices[0].shape, self._n_step)\n",
" for n in range(self._n_step - 1, -1, -1):\n",
" now = indices[n]\n",
" gammas[end_flag[now] > 0] = n + 1\n",
" returns[end_flag[now] > 0] = 0.0\n",
" new_rew = []\n",
" old_obs_next = buffer.obs_next[now]\n",
" new_rew.append(\n",
" self.reward_fn(\n",
" old_obs_next[:, self.index_range['achieved_goal']], new_goal, None\n",
" )\n",
" )\n",
" returns = np.array(new_rew).reshape(bsz, 1) + self._gamma * returns\n",
" target_q = target_q * gamma_buffer[gammas].reshape(bsz, 1) + returns\n",
" target_q = target_q.reshape(target_shape)\n",
" # return values\n",
" batch.returns = to_torch_as(target_q, target_q_torch)\n",
" if hasattr(batch, \"weight\"): # prio buffer update\n",
" batch.weight = to_torch_as(batch.weight, target_q_torch)\n",
" return batch"
],
"metadata": {
"id": "s-qFtIteOebg"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"import argparse\n",
"import os\n",
"import pprint\n",
"from functools import partial\n",
"\n",
"import gym\n",
"import numpy as np\n",
"import torch\n",
"import yaml\n",
"from torch.utils.tensorboard import SummaryWriter\n",
"import tianshou as ts\n",
"from gym.wrappers import FilterObservation, FlattenObservation\n",
"\n",
"from tianshou.data import (\n",
" Collector,\n",
" PrioritizedReplayBuffer,\n",
" PrioritizedVectorReplayBuffer,\n",
" ReplayBuffer,\n",
" VectorReplayBuffer,\n",
")\n",
"from tianshou.env import SubprocVectorEnv\n",
"from tianshou.trainer import offpolicy_trainer\n",
"from tianshou.utils import TensorboardLogger\n",
"from tianshou.utils.net.common import Net\n",
"from tianshou.utils.net.continuous import ActorProb, Critic\n",
"\n",
"if __name__ == '__main__':\n",
" '''\n",
" load param\n",
" '''\n",
" with open('/content/config_sac_her_pnp.yaml', \"r\") as stream:\n",
" try:\n",
" config = yaml.safe_load(stream)\n",
" except yaml.YAMLError as exc:\n",
" print(exc)\n",
" '''\n",
" make env\n",
" '''\n",
"\n",
" def make_env():\n",
" return gym.wrappers.FlattenObservation(gym.make(config['env']))\n",
"\n",
" def make_test_env(i):\n",
" if config['record_test']:\n",
" return gym.wrappers.RecordVideo(\n",
" gym.wrappers.FlattenObservation(gym.make(config['env'])),\n",
" video_folder='log/' + config['env'] + '/video' + str(i),\n",
" episode_trigger=lambda x: True\n",
" )\n",
" else:\n",
" return gym.wrappers.FlattenObservation(gym.make(config['env']))\n",
"\n",
" env = gym.make(config['env'])\n",
" dict_observation_space = env.observation_space\n",
" env = gym.wrappers.FlattenObservation(env)\n",
" obs = env.reset()\n",
" state_shape = len(obs)\n",
" action_shape = env.action_space.shape or env.action_space.n\n",
" train_envs = SubprocVectorEnv(\n",
" [make_env for _ in range(config['training_num'])], norm_obs=config['norm_obs']\n",
" )\n",
" if config['norm_obs']:\n",
" print('updating env norm...')\n",
" train_envs.reset()\n",
" for _ in range(1000):\n",
" _, _, done, _ = train_envs.step(\n",
" [env.action_space.sample() for _ in range(config['training_num'])]\n",
" )\n",
" if np.any(done):\n",
" env_ind = np.where(done)[0]\n",
" train_envs.reset(env_ind)\n",
" print('updating done!')\n",
" train_envs.update_obs_rms = False\n",
" test_envs = SubprocVectorEnv(\n",
" [partial(make_test_env, i) for i in range(config['test_num'])],\n",
" norm_obs=config['norm_obs'],\n",
" obs_rms=train_envs.obs_rms,\n",
" update_obs_rms=False\n",
" )\n",
" np.random.seed(config['seed'])\n",
" torch.manual_seed(config['seed'])\n",
" train_envs.seed(config['seed'])\n",
" test_envs.seed(config['seed'])\n",
" '''\n",
" build and init network\n",
" '''\n",
" if not (torch.cuda.is_available()):\n",
" config['device'] = 'cpu'\n",
" # actor\n",
" net_a = Net(\n",
" state_shape, hidden_sizes=config['hidden_sizes'], device=config['device']\n",
" )\n",
" actor = ActorProb(\n",
" net_a,\n",
" action_shape,\n",
" max_action=env.action_space.high[0],\n",
" device=config['device'],\n",
" unbounded=True,\n",
" conditioned_sigma=True\n",
" ).to(config['device'])\n",
" actor_optim = torch.optim.Adam(actor.parameters(), lr=config['actor_lr'])\n",
" # critic\n",
" net_c1 = Net(\n",
" state_shape,\n",
" action_shape,\n",
" hidden_sizes=config['hidden_sizes'],\n",
" concat=True,\n",
" device=config['device']\n",
" )\n",
" net_c2 = Net(\n",
" state_shape,\n",
" action_shape,\n",
" hidden_sizes=config['hidden_sizes'],\n",
" concat=True,\n",
" device=config['device']\n",
" )\n",
" critic1 = Critic(net_c1, device=config['device']).to(config['device'])\n",
" critic1_optim = torch.optim.Adam(critic1.parameters(), lr=config['critic_lr'])\n",
" critic2 = Critic(net_c2, device=config['device']).to(config['device'])\n",
" critic2_optim = torch.optim.Adam(critic2.parameters(), lr=config['critic_lr'])\n",
" # auto alpha\n",
" if config['auto_alpha']:\n",
" target_entropy = -np.prod(env.action_space.shape)\n",
" log_alpha = torch.zeros(1, requires_grad=True, device=config['device'])\n",
" alpha_optim = torch.optim.Adam([log_alpha], lr=config['alpha_lr'])\n",
" config['alpha'] = (target_entropy, log_alpha, alpha_optim)\n",
" '''\n",
" set up policy\n",
" '''\n",
" policy = SACHERPolicy(\n",
" actor,\n",
" actor_optim,\n",
" critic1,\n",
" critic1_optim,\n",
" critic2,\n",
" critic2_optim,\n",
" tau=config['tau'],\n",
" gamma=config['gamma'],\n",
" alpha=config['alpha'],\n",
" estimation_step=config['estimation_step'],\n",
" action_space=env.action_space,\n",
" reward_normalization=False,\n",
" dict_observation_space=dict_observation_space,\n",
" reward_fn=env.compute_reward,\n",
" future_k=config['replay_k'],\n",
" strategy=config['strategy']\n",
" )\n",
" # load policy\n",
" if config['resume_path']:\n",
" policy.load_state_dict(\n",
" torch.load(config['resume_path'], map_location=config['device'])\n",
" )\n",
" print(\"Loaded agent from: \", config['resume_path'])\n",
" '''\n",
" set up collector\n",
" '''\n",
" if config['training_num'] > 1:\n",
" if config['use_PER']:\n",
" buffer = PrioritizedVectorReplayBuffer(\n",
" total_size=config['buffer_size'],\n",
" buffer_num=len(train_envs),\n",
" alpha=config['per_alpha'],\n",
" beta=config['per_beta']\n",
" )\n",
" else:\n",
" buffer = VectorReplayBuffer(config['buffer_size'], len(train_envs))\n",
" else:\n",
" if config['use_PER']:\n",
" buffer = PrioritizedReplayBuffer(\n",
" size=config['buffer_size'],\n",
" alpha=config['per_alpha'],\n",
" beta=config['per_beta']\n",
" )\n",
" else:\n",
" buffer = ReplayBuffer(config['buffer_size'])\n",
" train_collector = HERCollector(\n",
" policy=policy,\n",
" env=train_envs,\n",
" buffer=buffer,\n",
" exploration_noise=True,\n",
" dict_observation_space=dict_observation_space,\n",
" reward_fn=env.compute_reward,\n",
" replay_k=config['replay_k'],\n",
" strategy=config['strategy']\n",
" )\n",
" test_collector = Collector(policy, test_envs)\n",
" # warm up\n",
" train_collector.collect(n_step=config['start_timesteps'], random=True)\n",
" '''\n",
" logger\n",
" '''\n",
" log_file = config['info']\n",
" log_path = os.path.join(config['logdir'], config['env'], 'sac', log_file)\n",
" writer = SummaryWriter(log_path)\n",
" writer.add_text(\"args\", str(config))\n",
" logger = TensorboardLogger(writer, update_interval=100, train_interval=100)\n",
"\n",
" # save function\n",
" def save_fn(policy):\n",
" torch.save(policy.state_dict(), os.path.join(log_path, 'policy.pth'))\n",
"\n",
" \n",
" # trainer\n",
" result = offpolicy_trainer(\n",
" policy,\n",
" train_collector,\n",
" test_collector,\n",
" config['epoch'],\n",
" config['step_per_epoch'],\n",
" config['step_per_collect'],\n",
" config['test_num'],\n",
" config['batch_size'],\n",
" save_fn=save_fn,\n",
" logger=logger,\n",
" update_per_step=config['update_per_step'],\n",
" test_in_train=False\n",
" )\n",
" pprint.pprint(result)\n",
"\n",
" \n",
" # Learning Curve\n",
" learning_curve_tianshou(log_dir=log_path, window=25)\n",
" # Load model, optimisers and buffer\n",
" #checkpoint = torch.load('/content/policy.pth')\n",
" # Record Episode Video\n",
" num_episodes = 10\n",
" for episode in range(num_episodes):\n",
" env = ts.env.DummyVectorEnv([lambda: FlattenObservation(FilterObservation(gym.make(\"FetchPickAndPlace-v1\"))) for _ in range(1)])\n",
" policy.eval()\n",
" collector = ts.data.Collector(policy, env, exploration_noise=False)\n",
" collector.collect_and_record = collect_and_record\n",
" collector.collect_and_record(self=collector, video_dir=log_path + f'final_agent/video{episode}/', n_episode=1,\n",
" render=1 / 60)\n",
"\n",
" # Execution Time\n",
" end = time.perf_counter()\n",
" print(f\"\\nExecution time = {end - start:.2f} second(s)\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"id": "RDk0d9YBOfQf",
"outputId": "15c3c490-7e21-4c78-ed3f-8081ba80afdb"
},
"execution_count": null,
"outputs": [
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #1: 64001it [36:23, 29.31it/s, alpha=0.151, env_step=64000, len=50, loss/actor=-6.112, loss/alpha=-12.014, loss/critic1=1.792, loss/critic2=1.781, n/ep=128, n/st=6400, rew=-49.22]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #1: test_reward: -50.000000 ± 0.000000, best_reward: -43.750000 ± 16.535946 in #0\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #2: 64001it [35:05, 30.39it/s, alpha=0.023, env_step=128000, len=50, loss/actor=1.153, loss/alpha=-21.468, loss/critic1=0.590, loss/critic2=0.426, n/ep=128, n/st=6400, rew=-48.44]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #2: test_reward: -50.000000 ± 0.000000, best_reward: -43.750000 ± 16.535946 in #0\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #3: 64001it [35:56, 29.67it/s, alpha=0.005, env_step=192000, len=50, loss/actor=3.174, loss/alpha=-21.170, loss/critic1=0.171, loss/critic2=0.378, n/ep=128, n/st=6400, rew=-48.96]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #3: test_reward: -49.250000 ± 1.984313, best_reward: -43.750000 ± 16.535946 in #0\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #4: 64001it [35:14, 30.27it/s, alpha=0.002, env_step=256000, len=50, loss/actor=4.053, loss/alpha=1.421, loss/critic1=0.183, loss/critic2=0.337, n/ep=128, n/st=6400, rew=-49.82]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #4: test_reward: -50.000000 ± 0.000000, best_reward: -43.750000 ± 16.535946 in #0\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #5: 64001it [35:47, 29.80it/s, alpha=0.002, env_step=320000, len=50, loss/actor=3.982, loss/alpha=-1.024, loss/critic1=0.212, loss/critic2=0.214, n/ep=128, n/st=6400, rew=-48.76]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #5: test_reward: -49.000000 ± 1.802776, best_reward: -43.750000 ± 16.535946 in #0\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #6: 64001it [34:37, 30.81it/s, alpha=0.002, env_step=384000, len=50, loss/actor=4.317, loss/alpha=-1.323, loss/critic1=0.234, loss/critic2=0.421, n/ep=128, n/st=6400, rew=-49.18]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #6: test_reward: -49.625000 ± 0.992157, best_reward: -43.750000 ± 16.535946 in #0\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #7: 64001it [34:57, 30.52it/s, alpha=0.003, env_step=448000, len=50, loss/actor=4.009, loss/alpha=-1.446, loss/critic1=0.240, loss/critic2=0.247, n/ep=128, n/st=6400, rew=-47.52]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #7: test_reward: -43.875000 ± 13.660504, best_reward: -43.750000 ± 16.535946 in #0\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #8: 64001it [34:58, 30.50it/s, alpha=0.002, env_step=512000, len=50, loss/actor=4.032, loss/alpha=-0.186, loss/critic1=0.265, loss/critic2=0.279, n/ep=128, n/st=6400, rew=-46.93]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #8: test_reward: -47.250000 ± 7.275816, best_reward: -43.750000 ± 16.535946 in #0\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stderr",
"output_type": "stream",
"text": [
"Epoch #9: 64001it [34:46, 30.67it/s, alpha=0.003, env_step=576000, len=50, loss/actor=3.889, loss/alpha=0.185, loss/critic1=0.339, loss/critic2=0.343, n/ep=128, n/st=6400, rew=-43.03]\n"
]
},
{
"metadata": {
"tags": null
},
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch #9: test_reward: -42.500000 ± 12.688578, best_reward: -42.500000 ± 12.688578 in #9\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"Epoch #10: 64001it [35:01, 30.45it/s, alpha=0.002, env_step=640000, len=50, loss/actor=3.906, loss/alpha=0.641, loss/critic1=0.337, loss/critic2=0.330, n/ep=128, n/st=6400, rew=-36.36]\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Epoch #10: test_reward: -42.000000 ± 9.987492, best_reward: -42.000000 ± 9.987492 in #10\n",
"{'best_result': '-42.00 ± 9.99',\n",
" 'best_reward': -42.0,\n",
" 'duration': '21183.35s',\n",
" 'test_episode': 88,\n",
" 'test_speed': '339.81 step/s',\n",
" 'test_step': 4400,\n",
" 'test_time': '12.95s',\n",
" 'train_episode': 12800,\n",
" 'train_speed': '30.23 step/s',\n",
" 'train_step': 640000,\n",
" 'train_time/collector': '1803.78s',\n",
" 'train_time/model': '19366.62s'}\n",
"WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/summary/summary_iterator.py:27: tf_record_iterator (from tensorflow.python.lib.io.tf_record) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use eager execution and: \n",
"`tf.data.TFRecordDataset(path)`\n"
]
},
{
"output_type": "error",
"ename": "TypeError",
"evalue": "ignored",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-8-a6ba4a8521e3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m 222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 223\u001b[0m \u001b[0;31m# Learning Curve\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 224\u001b[0;31m \u001b[0mlearning_curve_tianshou\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlog_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwindow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 225\u001b[0m \u001b[0;31m# Load model, optimisers and buffer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 226\u001b[0m \u001b[0;31m#checkpoint = torch.load('/content/policy.pth')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-4-3e7f1b0ab405>\u001b[0m in \u001b[0;36mlearning_curve_tianshou\u001b[0;34m(log_dir, window)\u001b[0m\n\u001b[1;32m 134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 135\u001b[0m \u001b[0;31m# Learning curve\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 136\u001b[0;31m \u001b[0mlearning_curve\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepisode_rewards\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlog_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwindow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwindow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 138\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m<ipython-input-4-3e7f1b0ab405>\u001b[0m in \u001b[0;36mlearning_curve\u001b[0;34m(episode_rewards, log_dir, window)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0max\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlineplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrolling_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfill_between\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrolling_average\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrolling_min\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrolling_max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 97\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_title\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Learning Curve'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 98\u001b[0m \u001b[0max\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_ylabel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Reward'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/matplotlib/__init__.py\u001b[0m in \u001b[0;36minner\u001b[0;34m(ax, data, *args, **kwargs)\u001b[0m\n\u001b[1;32m 1563\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1564\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1565\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msanitize_sequence\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1566\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1567\u001b[0m \u001b[0mbound\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_sig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0max\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/matplotlib/axes/_axes.py\u001b[0m in \u001b[0;36mfill_between\u001b[0;34m(self, x, y1, y2, where, interpolate, step, **kwargs)\u001b[0m\n\u001b[1;32m 5148\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5149\u001b[0m \u001b[0;31m# Convert the arrays so we can work with them\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5150\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasked_invalid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_xunits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 5151\u001b[0m \u001b[0my1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasked_invalid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_yunits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5152\u001b[0m \u001b[0my2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmasked_invalid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconvert_yunits\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.7/dist-packages/numpy/ma/core.py\u001b[0m in \u001b[0;36mmasked_invalid\u001b[0;34m(a, copy)\u001b[0m\n\u001b[1;32m 2367\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2368\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2369\u001b[0;31m \u001b[0mcondition\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m~\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfinite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2370\u001b[0m \u001b[0mcls\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMaskedArray\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2371\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mTypeError\u001b[0m: ufunc 'isfinite' not supported for the input types, and the inputs could not be safely coerced to any supported types according to the casting rule ''safe''"
]
},
{
"output_type": "display_data",
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD/CAYAAAD4xAEfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASD0lEQVR4nO3cW2yU5dqH8X9nECSLEu1kWqeiaViaOjHgASQmxhpsKy0wZaNIkyIRCeUAIwkmRjTSjRi1Jh4IQgwHsrEkKgeCjE0hxoNaw0YNSRtHINYiIENbp6thI6bl7fMd9Fud1YWueWlnU/tcv4SDJs+UmzvNXDPvWybLGGMEALCOJ9MDAAAygwAAgKUIAABYigAAgKUIAABYigAAgKUSBqChoUHFxcUqLCzUmTNn/vSM4ziqr69XaWmpnnjiCe3fvz/pgwIAkithAEpKSrRv3z7dfffdf3nm0KFDOnfunI4cOaJPPvlE27Zt04ULF5I6KAAguRIGYO7cuQoEAv/zTFNTk55++ml5PB7l5OSotLRUzc3NSRsSAJB8SbkHEI1GlZ+fP/x1IBDQpUuXkvGtAQApwk1gALDUpGR8k0AgoIsXL2r27NmSbn5H4Na//nVNg4N8NJHPN02x2NVMjzEusIs4dhHHLoZ4PFm6885/jPrxSQlAeXm59u/fr/nz56uvr09ffvml9u3bd8vfZ3DQEID/xx7i2EUcu4hjF2OX8BLQG2+8occee0yXLl3Sc889p0WLFkmSqqur1d7eLklasmSJZsyYofnz52vFihV6/vnndc8996R2cgDAmGSNp4+DjsWuUnVJfn+2enquZHqMcYFdxLGLOHYxxOPJks83bfSPT+IsAIC/EQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgKQIAAJYiAABgqUluDnV2dmrTpk3q6+vTHXfcoYaGBhUUFIw4E4vF9MorrygajerGjRt6+OGH9dprr2nSJFd/BQAgzVy9A6itrVVVVZUOHz6sqqoq1dTU3HTmgw8+0D//+U8dOnRIn3/+uX744QcdOXIk6QMDAJIjYQBisZgikYhCoZAkKRQKKRKJqLe3d8S5rKwsXbt2TYODg+rv79fAwIDy8vJSMzUAYMwSXp+JRqPKy8uT1+uVJHm9XuXm5ioajSonJ2f43Pr16/XCCy/o0Ucf1fXr17Vy5UrNmTPnlobx+abd4vgTl9+fnekRxg12Eccu4tjF2CXtAn1zc7MKCwu1Z88eXbt2TdXV1WpublZ5ebnr7xGLXdXgoEnWSH9bfn+2enquZHqMcYFdxLGLOHYxxOPJGtML54SXgAKBgLq6uuQ4jiTJcRx1d3crEAiMONfY2KjFixfL4/EoOztbxcXFOn78+KgHAwCkVsIA+Hw+BYNBhcNhSVI4HFYwGBxx+UeSZsyYoZaWFklSf3+/jh49qvvvvz8FIwMAksHVbwHV1dWpsbFRZWVlamxsVH19vSSpurpa7e3tkqRXX31V33//vSoqKrR06VIVFBRoxYoVqZscADAmWcaYcXPRnXsAQ7i+Gccu4thFHLsYkvJ7AACAiYkAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClXAWgs7NTlZWVKisrU2Vlpc6ePfun55qamlRRUaFQKKSKigr99ttvyZwVAJBEk9wcqq2tVVVVlZYsWaKDBw+qpqZGe/fuHXGmvb1d77//vvbs2SO/368rV65o8uTJKRkaADB2Cd8BxGIxRSIRhUIhSVIoFFIkElFvb++Ic7t379aaNWvk9/slSdnZ2ZoyZUoKRgYAJEPCAESjUeXl5cnr9UqSvF6vcnNzFY1GR5zr6OjQ+fPntXLlSi1btkw7duyQMSY1UwMAxszVJSA3HMfR6dOntWvXLvX392vt2rXKz8/X0qVLXX8Pn29assb52/P7szM9wrjBLuLYRRy7GLuEAQgEAurq6pLjOPJ6vXIcR93d3QoEAiPO5efnq7y8XJMnT9bkyZNVUlKitra2WwpALHZVg4O8a/D7s9XTcyXTY4wL7CKOXcSxiyEeT9aYXjgnvATk8/kUDAYVDoclSeFwWMFgUDk5OSPOhUIhtba2yhijgYEBHTt2TA888MCoBwMApJarXwOtq6tTY2OjysrK1NjYqPr6eklSdXW12tvbJUmLFi2Sz+fTwoULtXTpUt13331avnx56iYHAIxJlhlHd2q5BDSEt7dx7CKOXcSxiyEpvwQEAJiYCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClXAWgs7NTlZWVKisrU2Vlpc6ePfuXZ3/++Wc99NBDamhoSNaMAIAUcBWA2tpaVVVV6fDhw6qqqlJNTc2fnnMcR7W1tSotLU3qkACA5EsYgFgspkgkolAoJEkKhUKKRCLq7e296ezOnTs1b948FRQUJH1QAEByTUp0IBqNKi8vT16vV5Lk9XqVm5uraDSqnJyc4XOnTp1Sa2ur9u7dqx07doxqGJ9v2qgeNxH5/dmZHmHcYBdx7CKOXYxdwgC4MTAwoM2bN+utt94aDsVoxGJXNThokjHS35rfn62eniuZHmNcYBdx7CKOXQzxeLLG9MI5YQACgYC6urrkOI68Xq8cx1F3d7cCgcDwmZ6eHp07d07r1q2TJF2+fFnGGF29elVbtmwZ9XAAgNRJGACfz6dgMKhwOKwlS5YoHA4rGAyOuPyTn5+v48ePD3+9bds2/f7773r55ZdTMzUAYMxc/RZQXV2dGhsbVVZWpsbGRtXX10uSqqur1d7entIBAQCpkWWMGTcX3bkHMITrm3HsIo5dxLGLIWO9B8D/BAYASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALAUAQAASxEAALDUJDeHOjs7tWnTJvX19emOO+5QQ0ODCgoKRpzZvn27mpqa5PF4dNttt2njxo0qKipKxcwAgCRwFYDa2lpVVVVpyZIlOnjwoGpqarR3794RZ2bPnq01a9Zo6tSpOnXqlJ555hm1trbq9ttvT8ngAICxSXgJKBaLKRKJKBQKSZJCoZAikYh6e3tHnCsqKtLUqVMlSYWFhTLGqK+vLwUjAwCSIWEAotGo8vLy5PV6JUler1e5ubmKRqN/+ZgDBw7o3nvv1V133ZW8SQEASeXqEtCtOHHihN577z19+OGHt/xYn29assf52/L7szM9wrjBLuLYRRy7GLuEAQgEAurq6pLjOPJ6vXIcR93d3QoEAjedPXnypF566SXt2LFDM2fOvOVhYrGrGhw0t/y4icbvz1ZPz5VMjzEusIs4dhHHLoZ4PFljeuGc8BKQz+dTMBhUOByWJIXDYQWDQeXk5Iw419bWpo0bN2rr1q168MEHRz0QACA9sowxCV9yd3R0aNOmTbp8+bKmT5+uhoYGzZw5U9XV1dqwYYNmzZqlp556Sr/++qvy8vKGH/fOO++osLDQ9TC8AxjCq5s4dhHHLuLYxZCxvgNwFYB0IQBD+OGOYxdx7CKOXQxJ+SUgAMDERAAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAsRQAAwFIEAAAs5SoAnZ2dqqysVFlZmSorK3X27NmbzjiOo/r6epWWluqJJ57Q/v37kz0rACCJXAWgtrZWVVVVOnz4sKqqqlRTU3PTmUOHDuncuXM6cuSIPvnkE23btk0XLlxI+sAAgOSYlOhALBZTJBLRrl27JEmhUEhbtmxRb2+vcnJyhs81NTXp6aeflsfjUU5OjkpLS9Xc3Ky1a9e6HsbjyRrFP2FiYhdx7CKOXcSxi7HvIGEAotGo8vLy5PV6JUler1e5ubmKRqMjAhCNRpWfnz/8dSAQ0KVLl25pmDvv/MctnZ/IfL5pmR5h3GAXcewijl2MHTeBAcBSCQMQCATU1dUlx3EkDd3s7e7uViAQuOncxYsXh7+ORqO66667kjwuACBZEgbA5/MpGAwqHA5LksLhsILB4IjLP5JUXl6u/fv3a3BwUL29vfryyy9VVlaWmqkBAGOWZYwxiQ51dHRo06ZNunz5sqZPn66GhgbNnDlT1dXV2rBhg2bNmiXHcfT666/rm2++kSRVV1ersrIy5f8AAMDouAoAAGDi4SYwAFiKAACApQgAAFiKAACApdIaAD5ULs7NLrZv365FixapoqJCTz75pL7++uv0D5oGbnbxbz///LMeeughNTQ0pG/ANHK7i6amJlVUVCgUCqmiokK//fZbegdNAze7iMViWrdunSoqKrRgwQLV1dXpxo0b6R82hRoaGlRcXKzCwkKdOXPmT8+M+nnTpNGqVavMgQMHjDHGHDhwwKxateqmM5999plZs2aNcRzHxGIxU1RUZM6fP5/OMdPCzS5aWlrM77//bowx5scffzRz5swx169fT+uc6eBmF8YYc+PGDfPMM8+YF1980bz99tvpHDFt3Oyira3NLFiwwHR3dxtjjLl8+bL5448/0jpnOrjZxRtvvDH8s9Df32+WL19uvvjii7TOmWrffvutuXjxonn88cfN6dOn//TMaJ830/YO4N8fKhcKhSQNfahcJBJRb2/viHN/9aFyE4nbXRQVFWnq1KmSpMLCQhlj1NfXl/Z5U8ntLiRp586dmjdvngoKCtI8ZXq43cXu3bu1Zs0a+f1+SVJ2dramTJmS9nlTye0usrKydO3aNQ0ODqq/v18DAwPKy8vLxMgpM3fu3Js+eeG/jfZ5M20B+F8fKvff58b6oXLjndtd/KcDBw7o3nvvnXAfr+F2F6dOnVJra6tWr16dgSnTw+0uOjo6dP78ea1cuVLLli3Tjh07ZCbYf+dxu4v169ers7NTjz766PCfOXPmZGLkjBrt8yY3gf8GTpw4offee0/vvvtupkfJiIGBAW3evFn19fXDTwg2cxxHp0+f1q5du/TRRx+ppaVFBw8ezPRYGdHc3KzCwkK1traqpaVF33333YS7YpBKaQsAHyoX53YXknTy5Em99NJL2r59u2bOnJnuUVPOzS56enp07tw5rVu3TsXFxdqzZ48+/fRTbd68OVNjp4Tbn4v8/HyVl5dr8uTJmjZtmkpKStTW1paJkVPG7S4aGxu1ePFieTweZWdnq7i4WMePH8/EyBk12ufNtAWAD5WLc7uLtrY2bdy4UVu3btWDDz6YiVFTzs0u8vPzdfz4cX311Vf66quv9Oyzz2rFihXasmVLpsZOCbc/F6FQSK2trTLGaGBgQMeOHdMDDzyQiZFTxu0uZsyYoZaWFklSf3+/jh49qvvvvz/t82baqJ83k3q7OoGffvrJLF++3MyfP98sX77cdHR0GGOMWbt2rWlrazPGDP2mR01NjSkpKTElJSXm448/TueIaeNmF08++aR5+OGHzeLFi4f/nDp1KpNjp4SbXfynrVu3TtjfAnKzC8dxzJtvvmnKy8vNwoULzZtvvmkcx8nk2CnhZhe//PKLWb16tQmFQmbBggWmrq7ODAwMZHLspNuyZYspKioywWDQPPLII2bhwoXGmOQ8b/JhcABgKW4CA4ClCAAAWIoAAIClCAAAWIoAAIClCAAAWIoAAIClCAAAWOr/ABwJ/vdo2W6nAAAAAElFTkSuQmCC\n",
"text/plain": [
"<Figure size 432x288 with 1 Axes>"
]
},
"metadata": {}
}
]
},
{
"cell_type": "code",
"source": [
"!zip -r /content/SAC_HER_videos2.zip /content/log\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "n2gLPHkpqYXl",
"outputId": "68cbd981-cd4d-49d2-886b-e6164ab6d5da"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
" adding: content/log/ (stored 0%)\n",
" adding: content/log/FetchPickAndPlace-v1/ (stored 0%)\n",
" adding: content/log/FetchPickAndPlace-v1/sac/ (stored 0%)\n",
" adding: content/log/FetchPickAndPlace-v1/sac/Train_OfflineHER_FetchPickAndPlace/ (stored 0%)\n",
" adding: content/log/FetchPickAndPlace-v1/sac/Train_OfflineHER_FetchPickAndPlace/events.out.tfevents.1645624481.c5abb6993b2d.70.0 (deflated 71%)\n",
" adding: content/log/FetchPickAndPlace-v1/sac/Train_OfflineHER_FetchPickAndPlace/policy.pth (deflated 7%)\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"from google.colab import files\n",
"files.download(\"/content/SAC_HER_videos2.zip\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 34
},
"id": "SVL4qvQKqgJD",
"outputId": "43681b88-2684-4b69-cea9-ae02bdaeb332"
},
"execution_count": null,
"outputs": [
{
"output_type": "display_data",
"data": {
"application/javascript": [
"\n",
" async function download(id, filename, size) {\n",
" if (!google.colab.kernel.accessAllowed) {\n",
" return;\n",
" }\n",
" const div = document.createElement('div');\n",
" const label = document.createElement('label');\n",
" label.textContent = `Downloading \"${filename}\": `;\n",
" div.appendChild(label);\n",
" const progress = document.createElement('progress');\n",
" progress.max = size;\n",
" div.appendChild(progress);\n",
" document.body.appendChild(div);\n",
"\n",
" const buffers = [];\n",
" let downloaded = 0;\n",
"\n",
" const channel = await google.colab.kernel.comms.open(id);\n",
" // Send a message to notify the kernel that we're ready.\n",
" channel.send({})\n",
"\n",
" for await (const message of channel.messages) {\n",
" // Send a message to notify the kernel that we're ready.\n",
" channel.send({})\n",
" if (message.buffers) {\n",
" for (const buffer of message.buffers) {\n",
" buffers.push(buffer);\n",
" downloaded += buffer.byteLength;\n",
" progress.value = downloaded;\n",
" }\n",
" }\n",
" }\n",
" const blob = new Blob(buffers, {type: 'application/binary'});\n",
" const a = document.createElement('a');\n",
" a.href = window.URL.createObjectURL(blob);\n",
" a.download = filename;\n",
" div.appendChild(a);\n",
" a.click();\n",
" div.remove();\n",
" }\n",
" "
],
"text/plain": [
"<IPython.core.display.Javascript object>"
]
},
"metadata": {}
},
{
"output_type": "display_data",
"data": {
"application/javascript": [
"download(\"download_9de99e00-2c8d-4d4a-8b57-12add236e4f9\", \"SAC_HER_videos2.zip\", 1478992)"
],
"text/plain": [
"<IPython.core.display.Javascript object>"
]
},
"metadata": {}
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment