PPO_Tutorial_answer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "gaBbeW2TNnIK"
   },
   "source": [
    "# Deep Reinforcement Learning Tutorial - Proximal Policy Optimization\n",
    "Author: Yu Tang Liu <br>\n",
    "Email: yutang.liu@tuebingen.mpg.de \n",
    "\n",
    "In this tutorial you will practice\n",
    "- how to write your own PPO algorithm \n",
    "- python programming guideline \n",
    "\n",
    "reference<br>\n",
    "[1] openAI SpinningUp https://spinningup.openai.com/en/latest/algorithms/ppo.html<br>\n",
    "[2] What Matters In On-Policy Reinforcement Learning? A Large-Scale Empirical Study https://arxiv.org/abs/2006.05990"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAC8AAAALCAYAAADvGtSMAAAABHNCSVQICAgIfAhkiAAAAyxJREFUSEvNVU0orGEUfubOTWLGDI3FlPJflCaTpqRIsaAkylIkIgs/U2RQupFsbCZN+ZvJgvLXJCsLCyEb8pdBjBD5l0FG+Zs759zmq5n56HYXN2fzve855z3v857znPNJXG6Bj/T29kImk6G4uNjX9K32P33RvLy8YGRkBM/Pz3h6ekJVVZWvy7fZe4F/fX1FU1MTAycZGBjg72cPeHx8hMlkwuTkJHJzcxEcHIybmxs8PDygra0NERERmJqawtDQEFexqKjonx6+trbG94SHh6Orq0uI8cOzIuCNjY1YWFjwuoAeYLfbRS+Vy+VoaGhgW2VlJQwGA7q7uxEYGIjBwUHWFxQUIDExUfT83yq1Wi1ycnL83DnzxPH5+Xns7e15OWRmZiIjIwNxcXF+Bz9TvL29weFwICEhQdTl4uKCsxcbGwun08nVqqmpEXzHx8exubnJWd7f30dhYSGysrK8Yo2NjcFsNoPBz83N+WWXskeZVKlUoiB8lRSMgNhsNoSFhaGsrMzXhfcSiQQlJSVISUnhPVVuaWkJaWlpWFxcxOjoKCYmJiCVSrG1tYXZ2Vkv8PTgnZ0dTjjTpqenB5GRkRyMJDs7m4HX1tbi9PQUd3d3gu2zRXl5OdOO+E1Zr66uFnVVKpVYWVlBZ2cnU4ziHx4esi9RlihCwEmSkpJQX18vxLm/v+e4UVFRiImJ+QOeStTX1yc8oLS0FHl5ecjPz2fO0nd9fV0UjJgyPT2ds3Z7e+tntlgsODg4QGtrK2ddo9Hg4+OD/USmttf5q6sr6PV6DA8PM1OEhiV69Pf3o729XWgwD2Vo+lAVNjY2/MCIKYirCoUCoaGhfmbqh5CQEEF/dnYmrKm/6I7393fW0Zro4ZH4+HgkJyejrq4Ozc3NkP5yi8cYFBQEcvBIdHQ0rq+vsbu7C5pGxD8aifQDI6FRaTQasb29zeNxeXmZ+4eq1NLSArVajenpaT53eXnJlU1NTYXVamXqkN/R0RFOTk5Ad+l0Os4+8Z6adnV1FRUVFTg+PuZ/D32pnwICAjAzM8Ol+lLcJXV1dHS43A3mctPBdX5+/qX//zT+Bn+vBM203vhPAAAAAElFTkSuQmCC"
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Black\n",
    "- python auto-formating tool\n",
    "- run the first cell to install black and then refresh this notebook\n",
    "- press the black button at tab to format the code\n",
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: jupyter_contrib_nbextensions in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (0.5.1)\n",
      "Requirement already satisfied: ipython-genutils in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (0.2.0)\n",
      "Requirement already satisfied: tornado in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (6.1)\n",
      "Requirement already satisfied: lxml in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (4.8.0)\n",
      "Requirement already satisfied: nbconvert>=4.2 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (6.0.7)\n",
      "Requirement already satisfied: jupyter-nbextensions-configurator>=0.4.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (0.4.1)\n",
      "Requirement already satisfied: jupyter-contrib-core>=0.3.3 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (0.3.3)\n",
      "Requirement already satisfied: pyyaml in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (6.0)\n",
      "Requirement already satisfied: traitlets>=4.1 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (4.3.3)\n",
      "Requirement already satisfied: notebook>=4.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (6.4.8)\n",
      "Requirement already satisfied: jupyter-latex-envs>=1.3.8 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (1.4.6)\n",
      "Requirement already satisfied: jupyter-highlight-selected-word>=0.1.1 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (0.2.0)\n",
      "Requirement already satisfied: jupyter-core in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter_contrib_nbextensions) (4.9.2)\n",
      "Requirement already satisfied: setuptools in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter-contrib-core>=0.3.3->jupyter_contrib_nbextensions) (58.0.4)\n",
      "Requirement already satisfied: ipython in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter-latex-envs>=1.3.8->jupyter_contrib_nbextensions) (7.16.3)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (0.8.4)\n",
      "Requirement already satisfied: pygments>=2.4.1 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (2.11.2)\n",
      "Requirement already satisfied: jinja2>=2.4 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (3.0.3)\n",
      "Requirement already satisfied: nbformat>=4.4 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (5.1.3)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (0.5.9)\n",
      "Requirement already satisfied: jupyterlab-pygments in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (0.1.2)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (0.4)\n",
      "Requirement already satisfied: testpath in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (0.6.0)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (1.5.0)\n",
      "Requirement already satisfied: defusedxml in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (0.7.1)\n",
      "Requirement already satisfied: bleach in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbconvert>=4.2->jupyter_contrib_nbextensions) (4.1.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jinja2>=2.4->nbconvert>=4.2->jupyter_contrib_nbextensions) (2.0.1)\n",
      "Requirement already satisfied: async-generator in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert>=4.2->jupyter_contrib_nbextensions) (1.10)\n",
      "Requirement already satisfied: nest-asyncio in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert>=4.2->jupyter_contrib_nbextensions) (1.5.4)\n",
      "Requirement already satisfied: jupyter-client>=6.1.5 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbclient<0.6.0,>=0.5.0->nbconvert>=4.2->jupyter_contrib_nbextensions) (7.1.2)\n",
      "Requirement already satisfied: python-dateutil>=2.1 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter-client>=6.1.5->nbclient<0.6.0,>=0.5.0->nbconvert>=4.2->jupyter_contrib_nbextensions) (2.8.2)\n",
      "Requirement already satisfied: pyzmq>=13 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jupyter-client>=6.1.5->nbclient<0.6.0,>=0.5.0->nbconvert>=4.2->jupyter_contrib_nbextensions) (22.3.0)\n",
      "Requirement already satisfied: jsonschema!=2.5.0,>=2.4 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from nbformat>=4.4->nbconvert>=4.2->jupyter_contrib_nbextensions) (3.2.0)\n",
      "Requirement already satisfied: importlib-metadata in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4->nbconvert>=4.2->jupyter_contrib_nbextensions) (4.8.3)\n",
      "Requirement already satisfied: six>=1.11.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4->nbconvert>=4.2->jupyter_contrib_nbextensions) (1.16.0)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4->nbconvert>=4.2->jupyter_contrib_nbextensions) (0.18.0)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4->nbconvert>=4.2->jupyter_contrib_nbextensions) (21.4.0)\n",
      "Requirement already satisfied: Send2Trash>=1.8.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from notebook>=4.0->jupyter_contrib_nbextensions) (1.8.0)\n",
      "Requirement already satisfied: prometheus-client in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from notebook>=4.0->jupyter_contrib_nbextensions) (0.13.1)\n",
      "Requirement already satisfied: argon2-cffi in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from notebook>=4.0->jupyter_contrib_nbextensions) (21.3.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from notebook>=4.0->jupyter_contrib_nbextensions) (0.12.1)\n",
      "Requirement already satisfied: ipykernel in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from notebook>=4.0->jupyter_contrib_nbextensions) (5.5.6)\n",
      "Requirement already satisfied: ptyprocess in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from terminado>=0.8.3->notebook>=4.0->jupyter_contrib_nbextensions) (0.7.0)\n",
      "Requirement already satisfied: decorator in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from traitlets>=4.1->jupyter_contrib_nbextensions) (5.1.1)\n",
      "Requirement already satisfied: dataclasses in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from argon2-cffi->notebook>=4.0->jupyter_contrib_nbextensions) (0.8)\n",
      "Requirement already satisfied: argon2-cffi-bindings in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from argon2-cffi->notebook>=4.0->jupyter_contrib_nbextensions) (21.2.0)\n",
      "Requirement already satisfied: typing-extensions in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from argon2-cffi->notebook>=4.0->jupyter_contrib_nbextensions) (4.1.1)\n",
      "Requirement already satisfied: cffi>=1.0.1 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from argon2-cffi-bindings->argon2-cffi->notebook>=4.0->jupyter_contrib_nbextensions) (1.15.0)\n",
      "Requirement already satisfied: pycparser in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->notebook>=4.0->jupyter_contrib_nbextensions) (2.21)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: packaging in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from bleach->nbconvert>=4.2->jupyter_contrib_nbextensions) (21.3)\n",
      "Requirement already satisfied: webencodings in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from bleach->nbconvert>=4.2->jupyter_contrib_nbextensions) (0.5.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from importlib-metadata->jsonschema!=2.5.0,>=2.4->nbformat>=4.4->nbconvert>=4.2->jupyter_contrib_nbextensions) (3.6.0)\n",
      "Requirement already satisfied: pexpect in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from ipython->jupyter-latex-envs>=1.3.8->jupyter_contrib_nbextensions) (4.8.0)\n",
      "Requirement already satisfied: jedi<=0.17.2,>=0.10 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from ipython->jupyter-latex-envs>=1.3.8->jupyter_contrib_nbextensions) (0.17.2)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from ipython->jupyter-latex-envs>=1.3.8->jupyter_contrib_nbextensions) (3.0.28)\n",
      "Requirement already satisfied: backcall in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from ipython->jupyter-latex-envs>=1.3.8->jupyter_contrib_nbextensions) (0.2.0)\n",
      "Requirement already satisfied: pickleshare in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from ipython->jupyter-latex-envs>=1.3.8->jupyter_contrib_nbextensions) (0.7.5)\n",
      "Requirement already satisfied: parso<0.8.0,>=0.7.0 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from jedi<=0.17.2,>=0.10->ipython->jupyter-latex-envs>=1.3.8->jupyter_contrib_nbextensions) (0.7.1)\n",
      "Requirement already satisfied: wcwidth in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->jupyter-latex-envs>=1.3.8->jupyter_contrib_nbextensions) (0.2.5)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages (from packaging->bleach->nbconvert>=4.2->jupyter_contrib_nbextensions) (3.0.7)\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m jupyter contrib nbextension install --user\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing jupyter_contrib_nbextensions nbextension files to jupyter data directory\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/toc2 -> toc2\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/toc2.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/demo_dark.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/toc2.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/demo3.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/demo.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/image.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/demo2.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toc2/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/hide_header -> hide_header\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_header/hide_header.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_header/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_header/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/rubberband -> rubberband\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/rubberband/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/rubberband/rubberband.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/rubberband/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/rubberband/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/rubberband/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/printview -> printview\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/printview/printview-button.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/printview/printview.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/printview/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/printview/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/printview/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/code_font_size -> code_font_size\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_font_size/code_font_size.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_font_size/code_font_size.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_font_size/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/autoscroll -> autoscroll\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/autoscroll/autoscroll.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/autoscroll/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/autoscroll/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/autoscroll/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/navigation-hotkeys -> navigation-hotkeys\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/navigation-hotkeys/hotkeys.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/navigation-hotkeys/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/navigation-hotkeys/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/navigation-hotkeys/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/python-markdown -> python-markdown\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/untrusted.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/python-markdown-pre.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/trusted.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/python-markdown-post.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/python-markdown.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/python-markdown.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/python-markdown/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/livemdpreview -> livemdpreview\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/livemdpreview/livemdpreview.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/livemdpreview/livemdpreview.yml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/skill -> skill\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skill/skill.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skill/skill.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skill/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skill/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/scroll_down -> scroll_down\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scroll_down/config.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scroll_down/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scroll_down/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scroll_down/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/runtools -> runtools\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/gutter.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/runtools_marker.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/runtools_lock.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/annotations.odg\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/runtools_nb.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/cellstate.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/runtools.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/runtools_execute.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/demo.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/runtools_show_hide.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/runtools/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/select_keymap -> select_keymap\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/select_keymap/select_keymap.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/select_keymap/select_keymap.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/select_keymap/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/select_keymap/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/keyboard_shortcut_editor -> keyboard_shortcut_editor\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/keyboard_shortcut_editor.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_menu_item.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_add_new_select_action.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_comma.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/quickhelp_shim.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_shortcut_editor_blank.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/kse_components.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_add_new_link.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_undefined_key.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_shortcut_editor_success.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_conflict.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_dropdown.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/readme_reset_disabled.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/keyboard_shortcut_editor/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/splitcell -> splitcell\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/splitcell/splitcell.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/splitcell/splitcell.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/splitcell/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/splitcell/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/move_selected_cells -> move_selected_cells\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/move_selected_cells/move_selected_cells.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/move_selected_cells/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/move_selected_cells/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/hide_input -> hide_input\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input/hide-input.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/scratchpad -> scratchpad\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scratchpad/scratchpad.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scratchpad/LICENSE\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scratchpad/scratchpad.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scratchpad/demo.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scratchpad/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/scratchpad/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/export_embedded -> export_embedded\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/export_embedded/export_embedded.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/export_embedded/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/export_embedded/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/export_embedded/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/zenmode -> zenmode\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/zenmode.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/back22.jpg\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/back2.jpg\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/back3.jpg\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/back11.jpg\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/ipynblogo0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/back21.jpg\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/back12.jpg\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/ipynblogo1.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/zenmode/images/back1.jpg\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/exercise -> exercise\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise/image.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise/exercise.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise/history.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/ruler -> ruler\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/edit.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/ruler_editor.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/ruler.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/ruler -> ruler\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/edit.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/ruler_editor.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/ruler.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/ruler/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/datestamper -> datestamper\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/datestamper/main.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/datestamper/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/datestamper/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/datestamper/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/notify -> notify\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/notify/notify.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/notify/notify.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/notify/notification.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/notify/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/notify/notify.mp3\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/codemirror_mode_extensions -> codemirror_mode_extensions\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codemirror_mode_extensions/codemirror_mode_extensions.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codemirror_mode_extensions/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/addbefore -> addbefore\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/addbefore/addbefore.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/addbefore/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/addbefore/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/addbefore/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/execution_dependencies -> execution_dependencies\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execution_dependencies/execution_dependencies.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execution_dependencies/execution_dependencies.yml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execution_dependencies/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/execute_time -> execute_time\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execute_time/execution-timings-box.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execute_time/ExecuteTime.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execute_time/ExecuteTime.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execute_time/execution-timings-menu.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execute_time/ExecuteTime.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execute_time/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/execute_time/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/spellchecker -> spellchecker\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/download_new_dict.py\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/config.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/screenshot.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/typo/LICENSE.txt\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/typo/typo.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/spellchecker/__pycache__/download_new_dict.cpython-36.pyc\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/code_prettify -> code_prettify\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-jv.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-py.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/autopep8.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/code_prettify.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_isort.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-R.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/2to3.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/autopep8.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/code_prettify.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/isort.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/2to3.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_2to3.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_autopep8.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/isort.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/kernel_exec_on_cell.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo_2to3.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_code_prettify.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/code_prettify -> code_prettify\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-jv.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-py.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/autopep8.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/code_prettify.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_isort.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-R.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/2to3.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/autopep8.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/code_prettify.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/isort.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/2to3.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_2to3.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_autopep8.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/isort.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/kernel_exec_on_cell.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo_2to3.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_code_prettify.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/code_prettify -> code_prettify\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-jv.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-py.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/autopep8.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/code_prettify.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_isort.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-R.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/2to3.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/autopep8.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/code_prettify.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/isort.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/2to3.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_2to3.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_autopep8.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/isort.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/kernel_exec_on_cell.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo_2to3.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_code_prettify.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/code_prettify -> code_prettify\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-jv.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-py.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/autopep8.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/code_prettify.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_isort.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo-R.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/2to3.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/autopep8.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/code_prettify.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/isort.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/2to3.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_2to3.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_autopep8.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/isort.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/kernel_exec_on_cell.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/demo_2to3.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README_code_prettify.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/code_prettify/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/codefolding -> codefolding\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_indent_unfolded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/edit.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_indent_folded_2.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_firstline_folded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/firstline-fold.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_firstline_unfolded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/magic-unfolded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/magic-fold.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/foldgutter.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_editor.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_indent_folded_1.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_editor.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/magic-folded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/codefolding -> codefolding\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_indent_unfolded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/edit.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_indent_folded_2.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_firstline_folded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/firstline-fold.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_firstline_unfolded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/magic-unfolded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/magic-fold.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/foldgutter.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_editor.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_indent_folded_1.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/codefolding_editor.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/magic-folded.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/codefolding/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/equation-numbering -> equation-numbering\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/equation-numbering/button.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/equation-numbering/info.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/equation-numbering/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/equation-numbering/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/equation-numbering/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/contrib_nbextensions_help_item -> contrib_nbextensions_help_item\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/contrib_nbextensions_help_item/contrib_nbextensions_help_item.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/contrib_nbextensions_help_item/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/contrib_nbextensions_help_item/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/tree-filter -> tree-filter\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/tree-filter/tree-filter.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/tree-filter/index.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/tree-filter/demo.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/tree-filter/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/snippets -> snippets\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets/snippets.json\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets/snippets.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets/snippets-demo.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/snippets_menu -> snippets_menu\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/examples_for_custom.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_menu.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenu_python.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/thumbnail.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenu_markdown.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/screenshot2.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/config.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/screenshot3.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/screenshot1.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/pandas.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/astropy.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/python_regex.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/matplotlib.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/sympy_functions.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/scipy_constants.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/scipy_special.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/scipy.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/sympy.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/h5py.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/numpy_ufuncs.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/numpy.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/sympy_assumptions.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/numpy_polynomial.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/numba.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/snippets_menu/snippets_submenus_python/python.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/hinterland -> hinterland\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hinterland/hinterland.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hinterland/hinterland.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hinterland/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/init_cell -> init_cell\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/init_cell/cell_toolbar_menu.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/init_cell/init_cell.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/init_cell/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/init_cell/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/init_cell/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/load_tex_macros -> load_tex_macros\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/load_tex_macros/load_tex_macros.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/load_tex_macros/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/load_tex_macros/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/load_tex_macros/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/exercise2 -> exercise2\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise2/image.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise2/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise2/exercise2.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise2/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise2/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/exercise2/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/freeze -> freeze\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/freeze/config.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/freeze/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/freeze/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/freeze/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/skip-traceback -> skip-traceback\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skip-traceback/skip-traceback.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skip-traceback/skip-traceback.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skip-traceback/traceback.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skip-traceback/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skip-traceback/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/skip-traceback/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/nbTranslate -> nbTranslate\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/nbTranslate/mutils.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/nbTranslate/nbTranslate.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/nbTranslate/demo1.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/nbTranslate/nbTranslate.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/nbTranslate/languages.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/nbTranslate/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/nbTranslate/demo2.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/nbTranslate/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/help_panel -> help_panel\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/help_panel/help_panel_ext_fullscreen.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/help_panel/help_panel.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/help_panel/help_panel.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/help_panel/help_panel_ext.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/help_panel/help_panel.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/help_panel/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/help_panel/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/help_panel/img/handle-v.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/gist_it -> gist_it\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/gist_it/gist_it.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/gist_it/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/gist_it/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/gist_it/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/limit_output -> limit_output\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/limit_output/limit-output.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/limit_output/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/limit_output/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/limit_output/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/hide_input_all -> hide_input_all\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input_all/hide_input_all_hide.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input_all/hide_input_all_show.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input_all/hide_input_all.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input_all/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input_all/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/hide_input_all/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/varInspector -> varInspector\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/varInspector.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/var_list.py\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/jquery.tablesorter.min.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/var_list.r\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/demo.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/tablesorter_LICENSE.txt\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/varInspector/__pycache__/var_list.cpython-36.pyc\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/qtconsole -> qtconsole\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/qtconsole/qtconsole.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/qtconsole/qtconsole.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/qtconsole/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/collapsible_headings -> collapsible_headings\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/collapsible_headings/collapsible_headings.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/collapsible_headings/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/collapsible_headings/screenshot.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/collapsible_headings/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/collapsible_headings/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/collapsible_headings/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/autosavetime -> autosavetime\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/autosavetime/autosavetime.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/autosavetime/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/autosavetime/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/autosavetime/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/table_beautifier -> table_beautifier\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/table_beautifier/table_beautifier.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/table_beautifier/main.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/table_beautifier/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/table_beautifier/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/cell_filter -> cell_filter\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/cell_filter/cell_filter.yml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/cell_filter/cell_filter.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/cell_filter/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/comment-uncomment -> comment-uncomment\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/comment-uncomment/comment-uncomment.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/comment-uncomment/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/comment-uncomment/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/comment-uncomment/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/toggle_all_line_numbers -> toggle_all_line_numbers\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toggle_all_line_numbers/main.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toggle_all_line_numbers/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toggle_all_line_numbers/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/toggle_all_line_numbers/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_contrib_nbextensions/nbextensions/highlighter -> highlighter\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/highlighter.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/export_highlights.tex\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/demo_highlighter.html\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/tst_highlights.pdf\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/image.gif\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/export_highlights.pdf\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/export_highlights.html\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/export_highlights.ipynb\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/highlighter.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/tst_highlights.html\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/demo_highlighter.ipynb\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/tst_highlights.tex\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/highlighter.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/tst_highlights.ipynb\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/readme.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlighter/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/jupyter_highlight_selected_word/static/highlight_selected_word -> highlight_selected_word\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlight_selected_word/configurator.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlight_selected_word/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/highlight_selected_word/main.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing /home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/latex_envs/static -> latex_envs\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/bibtex2.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/initNb.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/thmsInNb4.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/latex_envs.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/configuration_toolbar.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/example_user_envs.json\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/envsLatex.json\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/example.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/envsLatex.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/latex_envs.css\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/latex_envs.yaml\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/main_toolbar.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/bibInNb4.js\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/icon.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/documentation.tex\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/LaTeX_envs_menu.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/config_toolbar.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc.tex\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/IEEEtran.bst\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/configuration_toolbar.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/documentation.pdf\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/header.tex\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/example.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/Toggles.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/biblio_ipython.bib\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc.html\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/main_toolbar.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc.ipynb\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/README.md\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/.ipynb_checkpoints/latex_env_doc-checkpoint.ipynb\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_52_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_50_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_44_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_41_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_46_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_27_2.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_45_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_47_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_42_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_24_2.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_43_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_26_2.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_49_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_48_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_23_2.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Up to date: /home/yliu_local/.local/share/jupyter/nbextensions/latex_envs/doc/latex_env_doc_files/latex_env_doc_51_0.png\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Installing jupyter_contrib_nbextensions items to config in /home/yliu_local/.jupyter\n",
      "Enabling: jupyter_nbextensions_configurator\n",
      "- Writing config: /home/yliu_local/.jupyter\n",
      "    - Validating...\n",
      "      jupyter_nbextensions_configurator 0.4.1 \u001b[32mOK\u001b[0m\n",
      "Enabling notebook nbextension nbextensions_configurator/config_menu/main...\n",
      "Enabling tree nbextension nbextensions_configurator/tree_tab/main...\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m Enabling notebook extension contrib_nbextensions_help_item/main...\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m       - Validating: \u001b[32mOK\u001b[0m\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Editing config: /home/yliu_local/.jupyter/jupyter_nbconvert_config.json\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m --  Configuring nbconvert template path\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m --  Configuring nbconvert preprocessors\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m - Writing config: /home/yliu_local/.jupyter/jupyter_nbconvert_config.json\n",
      "\u001b[32m[I 09:12:48 InstallContribNbextensionsApp]\u001b[m --  Writing updated config file /home/yliu_local/.jupyter/jupyter_nbconvert_config.json\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading: https://github.com/drillan/jupyter-black/archive/master.zip -> /tmp/tmp96ompsih/master.zip\n",
      "Extracting: /tmp/tmp96ompsih/master.zip -> /home/yliu_local/.local/share/jupyter/nbextensions\n",
      "Enabling notebook extension jupyter-black-master/jupyter-black...\n",
      "      - Validating: \u001b[32mOK\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install jupyter_contrib_nbextensions\n",
    "!jupyter contrib nbextension install --user\n",
    "!jupyter nbextension install https://github.com/drillan/jupyter-black/archive/master.zip --user\n",
    "!jupyter nbextension enable jupyter-black-master/jupyter-black"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tools \n",
    "\n",
    "- torch: map tensor on GPU, build neural network, compute gradient and backpropagation\n",
    "- gym: a RL test environment set\n",
    "- spinup: a RL tutorial package\n",
    "\n",
    "install spinup will also install torch and gym<br>\n",
    "in the terminal run the following command:\n",
    "```\n",
    "sudo apt-get update && sudo apt-get install libopenmpi-dev\n",
    "git clone https://github.com/openai/spinningup.git\n",
    "cd spinningup\n",
    "pip install -e .\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "8IkgWsb4_3A8"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.distributions.categorical import Categorical\n",
    "from torch.optim import Adam\n",
    "import numpy as np\n",
    "import gym\n",
    "from gym.spaces import Discrete, Box"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "let's import environment and take a look at the specification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "action space: Discrete(2)\n",
      "action dimension: 2\n",
      "observation space: Box(4,)\n",
      "observation dimension: 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/yliu_local/miniconda3/envs/spinningup/lib/python3.6/site-packages/gym/logger.py:30: UserWarning: \u001b[33mWARN: Box bound precision lowered by casting to float32\u001b[0m\n",
      "  warnings.warn(colorize('%s: %s'%('WARN', msg % args), 'yellow'))\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"CartPole-v0\")\n",
    "print(\"action space:\", env.action_space)\n",
    "print(\"action dimension:\", env.action_space.n)\n",
    "print(\"observation space:\", env.observation_space)\n",
    "print(\"observation dimension:\", env.observation_space.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we have \n",
    "- discrete action space with dimension 2 \n",
    "- continuous observation space with dimension 4"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "here we introduce some useful environment function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---call env.reset() to reset environment---\n",
      "current observation: [-0.00226647 -0.03311677  0.01117324 -0.0273437 ]\n",
      "---call env.action_space.sample() to sample random action\n",
      "action: 0\n",
      "---call env.step(act) to perform action to the environment\n",
      "new observation: [-0.00292881 -0.22839715  0.01062637  0.26884346]\n",
      "new reward: 1.0\n"
     ]
    }
   ],
   "source": [
    "obs = env.reset()\n",
    "print(\"---call env.reset() to reset environment---\")\n",
    "print(\"current observation:\", obs)\n",
    "\n",
    "act = env.action_space.sample()\n",
    "print(\"---call env.action_space.sample() to sample random action\")\n",
    "print(\"action:\", act)\n",
    "\n",
    "obs, reward, done, info = env.step(act)\n",
    "print(\"---call env.step(act) to perform action to the environment\")\n",
    "print(\"new observation:\", obs)\n",
    "print(\"new reward:\", reward)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create a policy model with the following components\n",
    "- linear layer: nn.Linear(in, out)\n",
    "- activation function: nn.ReLU(), nn.Tanh(), nn.Hardswish()\n",
    "\n",
    "For example:\n",
    "```python\n",
    "input_layer = nn.Linear(4, 10) \n",
    "activation_in = nn.ReLU()\n",
    "hidden_layer0 = nn.Linear(10,10)\n",
    "activation0 = nn.ReLU()\n",
    "output_layer = nn.Linear(10, 2)\n",
    "activation_out = nn.Identity()\n",
    "\n",
    "layers=[input_layer, activation_in, hidden_layer0, activation0, output_layer, activation_out]\n",
    "model = nn.Sequential(*layers)\n",
    "```\n",
    "Selecting activation function can be treaky. In general, ReLU allow faster convergence but Tanh final performance is usually better. Hardswish is a upgraded ReLU and sometimes can be found in model-based RL context."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mlp(sizes, activation=nn.Tanh, output_activation=nn.Identity):\n",
    "    # Build a feedforward neural network.\n",
    "    layers = []\n",
    "    for j in range(len(sizes) - 1):\n",
    "        act = activation if j < len(sizes) - 2 else output_activation\n",
    "        layers += [nn.Linear(sizes[j], sizes[j + 1]), act()]\n",
    "    return nn.Sequential(*layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (3): ReLU()\n",
       "  (4): Linear(in_features=64, out_features=2, bias=True)\n",
       "  (5): Identity()\n",
       ")"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs_dim = env.observation_space.shape[0]\n",
    "n_acts = env.action_space.n\n",
    "hidden_sizes = [64, 64]\n",
    "\n",
    "sizes = [4] + hidden_sizes + [2]\n",
    "activation = nn.ReLU\n",
    "\n",
    "model = mlp(sizes, activation=activation)\n",
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also easy to foward/infer this model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-0.0608,  0.0468], grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "obs=env.reset()\n",
    "obs_tensor = torch.as_tensor(obs, dtype=torch.float32)\n",
    "logits=model.forward(obs_tensor)\n",
    "print(logits)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "since the environment has 2 actions, model output also has 2 values, which represents the logits of each action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "we can use pytorch categoritcal distribution to turn logits into probability and sample from this policy distribution. This is similar to classifier in machine learning. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1)\n"
     ]
    }
   ],
   "source": [
    "distribution = Categorical(logits=logits)\n",
    "action=distribution.sample()\n",
    "print(action)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It is also convenient to compute log probability of an action when you want to compute the loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-0.6408, grad_fn=<SqueezeBackward1>)"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distribution.log_prob(action)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's build our first RL agent\n",
    "RL agent should have following attributes:\n",
    "- environment: gym env to play with, e.g. cartpole-v0\n",
    "- model: policy model, e.g. NN\n",
    "- optimizer: perform gradient descent, e.g. Adam\n",
    "\n",
    "and functions:\n",
    "- train(): defines the training loop\n",
    "- get_action(): get action from the policy \n",
    "- compute_loss(): calculate policy loss  \n",
    "\n",
    "note: python is a object oriented language, to reuse the code and scale up the code size\n",
    "- start with class template or abstract\n",
    "- inherit from this template "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Abstract_agent:\n",
    "    def __init__(\n",
    "        self, env, model, optimizer\n",
    "    ):\n",
    "        self.env = env\n",
    "        self.logits_net = model\n",
    "        self.optimizer = optimizer\n",
    "\n",
    "    def get_action(self, obs):\n",
    "        \"\"\"\n",
    "        sample action from the policy distribution\n",
    "        \"\"\"\n",
    "        raise NotImplemented\n",
    "\n",
    "    def compute_loss(self, obs, act, weights):\n",
    "        \"\"\"\n",
    "        compute policy loss\n",
    "        \"\"\"\n",
    "        raise NotImplemented\n",
    "\n",
    "    def train(self, n_epochs):\n",
    "        \"\"\"\n",
    "        trainining loop\n",
    "        \"\"\"\n",
    "        raise NotImplemented"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#note: you can always black the code cell\n",
    "class Agent(Abstract_agent):\n",
    "    def __init__(self,   env, model   , optimizer=    Adam,     batch_size =5000, \n",
    "                 render =   False):\n",
    "        super().__init__(env,    model, optimizer)\n",
    "                \n",
    "        self.batch_size  =    batch_size\n",
    "        self.render          =   render\n",
    "    \n",
    "    def get_policy(self, obs):\n",
    "        \"\"\"\n",
    "        return policy distribution by model inference, \n",
    "        e.g. logits = model(obs)\n",
    "        distribution=Categorical(logits)\n",
    "        \"\"\"\n",
    "        ######### Insert Code Here #########\n",
    "        logits = self.logits_net(obs)\n",
    "        distribution = Categorical(logits=logits)\n",
    "        ####################################\n",
    "        return distribution\n",
    "\n",
    "    def get_action(self, obs):\n",
    "        \"\"\"\n",
    "        sample action from the policy distribution\n",
    "        i.e. policy_distribution.sample().item()\n",
    "        \"\"\"\n",
    "        ######### Insert Code Here #########\n",
    "        action=self.get_policy(obs).sample().item()\n",
    "        ####################################\n",
    "        return action\n",
    "\n",
    "    def compute_loss(self, obs, act, weights):\n",
    "        \"\"\"\n",
    "        policy loss, i.e. averaging over -(logprob * weights)\n",
    "        note: do not confuse weights with NN's weight and bias, \n",
    "            here it represent the reward weight for credit assignment\n",
    "        \"\"\"\n",
    "        ######### Insert Code Here #########\n",
    "        logp = self.get_policy(obs).log_prob(act)\n",
    "        loss_pi = -(logp * weights).mean()\n",
    "        ####################################\n",
    "        return loss_pi\n",
    "    \n",
    "    def policy_update(self,batch_obs,batch_acts,batch_weights):\n",
    "        \"\"\"\n",
    "        update the policy model by performing a gradient descent\n",
    "        \"\"\"\n",
    "        ######### Insert Code Here #########\n",
    "        # take a single policy gradient update step\n",
    "        # you need to\n",
    "        # 1. clean the optimizer gradient by self.optimizer.zero_grad() (required by torch)\n",
    "        # 2. compute batch_loss by batch_loss=self.compute_loss(obs, act, weights)\n",
    "        # 3. update the gradient by gradient propagation, batch_loss.backward()\n",
    "        # 4. run the optimizer by self.optimizer.step()\n",
    "        \n",
    "        self.optimizer.zero_grad()\n",
    "        batch_loss = self.compute_loss(obs=torch.as_tensor(batch_obs, dtype=torch.float32),\n",
    "                                  act=torch.as_tensor(batch_acts, dtype=torch.int32),\n",
    "                                  weights=torch.as_tensor(batch_weights, dtype=torch.float32)\n",
    "                                  )\n",
    "        batch_loss.backward()\n",
    "        self.optimizer.step()\n",
    "        return batch_loss        \n",
    "\n",
    "    # for training policy\n",
    "    def train_one_epoch(self,):\n",
    "        \"\"\"\n",
    "        1. collect (state, action, reward) pair and save in batch\n",
    "        2. train model with the batch\n",
    "        \"\"\"\n",
    "        # make some empty lists for logging.\n",
    "        batch_obs = []          # for observations\n",
    "        batch_acts = []         # for actions\n",
    "        batch_weights = []      # for R(tau) weighting in policy gradient\n",
    "        batch_rets = []         # for measuring episode returns\n",
    "        batch_lens = []         # for measuring episode lengths\n",
    "\n",
    "        # reset episode-specific variables\n",
    "        obs = env.reset()       # first obs comes from starting distribution\n",
    "        done = False            # signal from environment that episode is over\n",
    "        ep_rews = []            # list for rewards accrued throughout ep\n",
    "\n",
    "        # render first episode of each epoch\n",
    "        finished_rendering_this_epoch = False\n",
    "\n",
    "        # collect experience by acting in the environment with current policy\n",
    "        while True:\n",
    "\n",
    "            # rendering\n",
    "            if (not finished_rendering_this_epoch) and render:\n",
    "                self.env.render()\n",
    "\n",
    "            # save obs\n",
    "            batch_obs.append(obs.copy())\n",
    "\n",
    "            # act in the environment\n",
    "            act = self.get_action(torch.as_tensor(obs, dtype=torch.float32))\n",
    "            obs, rew, done, _ = self.env.step(act)\n",
    "\n",
    "            # save action, reward\n",
    "            batch_acts.append(act)\n",
    "            ep_rews.append(rew)\n",
    "\n",
    "            if done:\n",
    "                # if episode is over, record info about episode\n",
    "                ep_ret, ep_len = sum(ep_rews), len(ep_rews)\n",
    "                batch_rets.append(ep_ret)\n",
    "                batch_lens.append(ep_len)\n",
    "\n",
    "                # the weight for each logprob(a|s) is R(tau)\n",
    "                batch_weights += [ep_ret] * ep_len\n",
    "\n",
    "                # reset episode-specific variables\n",
    "                obs, done, ep_rews = env.reset(), False, []\n",
    "\n",
    "                # won't render again this epoch\n",
    "                finished_rendering_this_epoch = True\n",
    "\n",
    "                # end experience loop if we have enough of it\n",
    "                if len(batch_obs) > self.batch_size:\n",
    "                    break\n",
    "\n",
    "        batch_loss=self.policy_update(batch_obs, batch_acts,batch_weights)\n",
    "        \n",
    "        return batch_loss, batch_rets, batch_lens\n",
    "    \n",
    "    def train(self, n_epoch):\n",
    "        for i in range(n_epoch):\n",
    "            batch_loss, batch_rets, batch_lens = self.train_one_epoch()\n",
    "            print('epoch: %3d \\t loss: %.3f \\t return: %.3f \\t ep_len: %.3f'%\n",
    "                    (i, batch_loss, np.mean(batch_rets), np.mean(batch_lens)))\n",
    "        print('training finished')\n",
    "        self.env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "ompXoML8P8jF",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:   0 \t loss: 16.108 \t return: 19.265 \t ep_len: 19.265\n",
      "epoch:   1 \t loss: 23.517 \t return: 25.556 \t ep_len: 25.556\n",
      "epoch:   2 \t loss: 26.540 \t return: 29.192 \t ep_len: 29.192\n",
      "epoch:   3 \t loss: 29.677 \t return: 33.026 \t ep_len: 33.026\n",
      "epoch:   4 \t loss: 30.353 \t return: 33.353 \t ep_len: 33.353\n",
      "epoch:   5 \t loss: 31.296 \t return: 35.458 \t ep_len: 35.458\n",
      "epoch:   6 \t loss: 33.570 \t return: 36.882 \t ep_len: 36.882\n",
      "epoch:   7 \t loss: 33.048 \t return: 39.310 \t ep_len: 39.310\n",
      "epoch:   8 \t loss: 39.996 \t return: 46.046 \t ep_len: 46.046\n",
      "epoch:   9 \t loss: 36.763 \t return: 46.324 \t ep_len: 46.324\n",
      "epoch:  10 \t loss: 43.342 \t return: 58.953 \t ep_len: 58.953\n",
      "epoch:  11 \t loss: 47.798 \t return: 62.062 \t ep_len: 62.062\n",
      "epoch:  12 \t loss: 48.083 \t return: 61.864 \t ep_len: 61.864\n",
      "epoch:  13 \t loss: 54.178 \t return: 74.029 \t ep_len: 74.029\n",
      "epoch:  14 \t loss: 52.344 \t return: 68.726 \t ep_len: 68.726\n",
      "epoch:  15 \t loss: 51.879 \t return: 71.845 \t ep_len: 71.845\n",
      "epoch:  16 \t loss: 57.220 \t return: 79.730 \t ep_len: 79.730\n",
      "epoch:  17 \t loss: 63.055 \t return: 86.776 \t ep_len: 86.776\n",
      "epoch:  18 \t loss: 58.027 \t return: 82.295 \t ep_len: 82.295\n",
      "epoch:  19 \t loss: 66.892 \t return: 97.327 \t ep_len: 97.327\n",
      "epoch:  20 \t loss: 75.662 \t return: 110.391 \t ep_len: 110.391\n",
      "epoch:  21 \t loss: 78.123 \t return: 113.489 \t ep_len: 113.489\n",
      "epoch:  22 \t loss: 88.378 \t return: 134.447 \t ep_len: 134.447\n",
      "epoch:  23 \t loss: 87.910 \t return: 136.053 \t ep_len: 136.053\n",
      "epoch:  24 \t loss: 98.367 \t return: 165.903 \t ep_len: 165.903\n",
      "epoch:  25 \t loss: 99.317 \t return: 166.645 \t ep_len: 166.645\n",
      "epoch:  26 \t loss: 98.188 \t return: 157.688 \t ep_len: 157.688\n",
      "epoch:  27 \t loss: 94.785 \t return: 161.125 \t ep_len: 161.125\n",
      "epoch:  28 \t loss: 93.766 \t return: 152.882 \t ep_len: 152.882\n",
      "epoch:  29 \t loss: 95.835 \t return: 160.344 \t ep_len: 160.344\n",
      "epoch:  30 \t loss: 97.426 \t return: 167.900 \t ep_len: 167.900\n",
      "epoch:  31 \t loss: 94.289 \t return: 160.969 \t ep_len: 160.969\n",
      "epoch:  32 \t loss: 94.612 \t return: 160.812 \t ep_len: 160.812\n",
      "epoch:  33 \t loss: 97.133 \t return: 172.967 \t ep_len: 172.967\n",
      "epoch:  34 \t loss: 97.238 \t return: 168.567 \t ep_len: 168.567\n",
      "epoch:  35 \t loss: 98.726 \t return: 175.793 \t ep_len: 175.793\n",
      "epoch:  36 \t loss: 101.255 \t return: 184.214 \t ep_len: 184.214\n",
      "epoch:  37 \t loss: 97.513 \t return: 172.233 \t ep_len: 172.233\n",
      "epoch:  38 \t loss: 96.419 \t return: 169.433 \t ep_len: 169.433\n",
      "epoch:  39 \t loss: 98.981 \t return: 184.857 \t ep_len: 184.857\n",
      "epoch:  40 \t loss: 98.727 \t return: 183.857 \t ep_len: 183.857\n",
      "epoch:  41 \t loss: 99.674 \t return: 184.250 \t ep_len: 184.250\n",
      "epoch:  42 \t loss: 97.261 \t return: 181.929 \t ep_len: 181.929\n",
      "epoch:  43 \t loss: 102.145 \t return: 190.556 \t ep_len: 190.556\n",
      "epoch:  44 \t loss: 99.231 \t return: 179.000 \t ep_len: 179.000\n",
      "epoch:  45 \t loss: 97.937 \t return: 174.621 \t ep_len: 174.621\n",
      "epoch:  46 \t loss: 99.811 \t return: 187.222 \t ep_len: 187.222\n",
      "epoch:  47 \t loss: 98.881 \t return: 185.357 \t ep_len: 185.357\n",
      "epoch:  48 \t loss: 99.943 \t return: 189.481 \t ep_len: 189.481\n",
      "epoch:  49 \t loss: 100.190 \t return: 187.481 \t ep_len: 187.481\n",
      "training finished\n"
     ]
    }
   ],
   "source": [
    "env_name = \"CartPole-v0\"\n",
    "hidden_sizes = [32]\n",
    "lr = 1e-2\n",
    "epochs = 50\n",
    "batch_size = 5000\n",
    "render = True\n",
    "\n",
    "# define env\n",
    "env=gym.make(env_name)\n",
    "\n",
    "# create model\n",
    "model = mlp(sizes=[obs_dim]+hidden_sizes+[n_acts])\n",
    "\n",
    "# make optimizer, almost always Adam\n",
    "optimizer = Adam(model.parameters(), lr=lr) \n",
    "\n",
    "# get agent\n",
    "agent=Agent(env, model, optimizer, batch_size, render)\n",
    "\n",
    "# start training \n",
    "agent.train(epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OX3sXWQeQBHd"
   },
   "source": [
    "## Reward-to-Go \n",
    "a simple trick to better estimate the value of the state\n",
    "\n",
    "change the weighting function from\n",
    "```python\n",
    "batch_weights += [ep_ret] * ep_len\n",
    "```\n",
    "to\n",
    "```python \n",
    "batch_weights += list(reward_to_go(ep_rews))\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([15, 14, 12,  9,  5])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def reward_to_go(rews):\n",
    "    n = len(rews)\n",
    "    rtgs = np.zeros_like(rews)\n",
    "    for i in reversed(range(n)):\n",
    "        rtgs[i] = rews[i] + (rtgs[i+1] if i+1 < n else 0)\n",
    "    return rtgs\n",
    "\n",
    "reward_to_go([1,2,3,4,5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RTG_Agent(Agent):\n",
    "    def train_one_epoch(self,):\n",
    "        \"\"\"\n",
    "        1. collect (state, action, reward) pair and save in batch\n",
    "        2. train model with the batch\n",
    "        \"\"\"\n",
    "        # make some empty lists for logging.\n",
    "        batch_obs = []          # for observations\n",
    "        batch_acts = []         # for actions\n",
    "        batch_weights = []      # for R(tau) weighting in policy gradient\n",
    "        batch_rets = []         # for measuring episode returns\n",
    "        batch_lens = []         # for measuring episode lengths\n",
    "\n",
    "        # reset episode-specific variables\n",
    "        obs = env.reset()       # first obs comes from starting distribution\n",
    "        done = False            # signal from environment that episode is over\n",
    "        ep_rews = []            # list for rewards accrued throughout ep\n",
    "\n",
    "        # render first episode of each epoch\n",
    "        finished_rendering_this_epoch = False\n",
    "\n",
    "        # collect experience by acting in the environment with current policy\n",
    "        while True:\n",
    "\n",
    "            # rendering\n",
    "            if (not finished_rendering_this_epoch) and render:\n",
    "                self.env.render()\n",
    "\n",
    "            # save obs\n",
    "            batch_obs.append(obs.copy())\n",
    "\n",
    "            # act in the environment\n",
    "            act = self.get_action(torch.as_tensor(obs, dtype=torch.float32))\n",
    "            obs, rew, done, _ = self.env.step(act)\n",
    "\n",
    "            # save action, reward\n",
    "            batch_acts.append(act)\n",
    "            ep_rews.append(rew)\n",
    "\n",
    "            if done:\n",
    "                # if episode is over, record info about episode\n",
    "                ep_ret, ep_len = sum(ep_rews), len(ep_rews)\n",
    "                batch_rets.append(ep_ret)\n",
    "                batch_lens.append(ep_len)\n",
    "\n",
    "                # the weight for each logprob(a|s) is R(tau)\n",
    "                batch_weights += list(reward_to_go(ep_rews)) ###\n",
    "\n",
    "                # reset episode-specific variables\n",
    "                obs, done, ep_rews = env.reset(), False, []\n",
    "\n",
    "                # won't render again this epoch\n",
    "                finished_rendering_this_epoch = True\n",
    "\n",
    "                # end experience loop if we have enough of it\n",
    "                if len(batch_obs) > self.batch_size:\n",
    "                    break\n",
    "\n",
    "        batch_loss=self.policy_update(batch_obs, batch_acts,batch_weights)\n",
    "        \n",
    "        return batch_loss, batch_rets, batch_lens\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch:   0 \t loss: 49.481 \t return: 184.036 \t ep_len: 184.036\n",
      "epoch:   1 \t loss: 46.450 \t return: 167.677 \t ep_len: 167.677\n",
      "epoch:   2 \t loss: 49.875 \t return: 184.321 \t ep_len: 184.321\n",
      "epoch:   3 \t loss: 48.317 \t return: 174.793 \t ep_len: 174.793\n",
      "epoch:   4 \t loss: 48.242 \t return: 182.607 \t ep_len: 182.607\n",
      "epoch:   5 \t loss: 49.514 \t return: 177.724 \t ep_len: 177.724\n",
      "epoch:   6 \t loss: 50.673 \t return: 191.074 \t ep_len: 191.074\n",
      "epoch:   7 \t loss: 50.149 \t return: 187.852 \t ep_len: 187.852\n",
      "epoch:   8 \t loss: 49.254 \t return: 184.107 \t ep_len: 184.107\n",
      "epoch:   9 \t loss: 50.325 \t return: 186.852 \t ep_len: 186.852\n",
      "epoch:  10 \t loss: 51.193 \t return: 196.115 \t ep_len: 196.115\n",
      "epoch:  11 \t loss: 50.378 \t return: 190.407 \t ep_len: 190.407\n",
      "epoch:  12 \t loss: 51.483 \t return: 198.462 \t ep_len: 198.462\n",
      "epoch:  13 \t loss: 50.631 \t return: 191.889 \t ep_len: 191.889\n",
      "epoch:  14 \t loss: 50.948 \t return: 195.000 \t ep_len: 195.000\n",
      "epoch:  15 \t loss: 51.415 \t return: 199.500 \t ep_len: 199.500\n",
      "epoch:  16 \t loss: 51.485 \t return: 196.346 \t ep_len: 196.346\n",
      "epoch:  17 \t loss: 51.704 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  18 \t loss: 52.147 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  19 \t loss: 51.539 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  20 \t loss: 51.503 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  21 \t loss: 51.195 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  22 \t loss: 50.874 \t return: 198.885 \t ep_len: 198.885\n",
      "epoch:  23 \t loss: 51.576 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  24 \t loss: 51.639 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  25 \t loss: 51.815 \t return: 199.654 \t ep_len: 199.654\n",
      "epoch:  26 \t loss: 51.989 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  27 \t loss: 51.417 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  28 \t loss: 52.056 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  29 \t loss: 51.809 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  30 \t loss: 51.922 \t return: 199.846 \t ep_len: 199.846\n",
      "epoch:  31 \t loss: 52.060 \t return: 198.000 \t ep_len: 198.000\n",
      "epoch:  32 \t loss: 52.017 \t return: 196.115 \t ep_len: 196.115\n",
      "epoch:  33 \t loss: 51.308 \t return: 194.462 \t ep_len: 194.462\n",
      "epoch:  34 \t loss: 52.169 \t return: 196.462 \t ep_len: 196.462\n",
      "epoch:  35 \t loss: 51.597 \t return: 197.962 \t ep_len: 197.962\n",
      "epoch:  36 \t loss: 50.628 \t return: 195.731 \t ep_len: 195.731\n",
      "epoch:  37 \t loss: 51.508 \t return: 190.667 \t ep_len: 190.667\n",
      "epoch:  38 \t loss: 51.594 \t return: 192.462 \t ep_len: 192.462\n",
      "epoch:  39 \t loss: 51.232 \t return: 198.385 \t ep_len: 198.385\n",
      "epoch:  40 \t loss: 51.880 \t return: 199.192 \t ep_len: 199.192\n",
      "epoch:  41 \t loss: 52.616 \t return: 198.385 \t ep_len: 198.385\n",
      "epoch:  42 \t loss: 51.685 \t return: 196.115 \t ep_len: 196.115\n",
      "epoch:  43 \t loss: 52.174 \t return: 197.923 \t ep_len: 197.923\n",
      "epoch:  44 \t loss: 52.482 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  45 \t loss: 52.180 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  46 \t loss: 52.597 \t return: 199.231 \t ep_len: 199.231\n",
      "epoch:  47 \t loss: 52.328 \t return: 199.846 \t ep_len: 199.846\n",
      "epoch:  48 \t loss: 51.684 \t return: 200.000 \t ep_len: 200.000\n",
      "epoch:  49 \t loss: 51.900 \t return: 200.000 \t ep_len: 200.000\n",
      "training finished\n"
     ]
    }
   ],
   "source": [
    "n_epoch=50\n",
    "agent=RTG_Agent(env, model, optimizer, batch_size, render)\n",
    "agent.train(n_epoch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Question: \n",
    "how does reward-to-go function improve training progress?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## It is convenient to define model and buffer to further reuse the code\n",
    "general rule of thumb: \n",
    "- break down the code to small piece can reuse the code easier\n",
    "- apply inheritence wisely to create different level of abstract \n",
    "- when writing the same code repeatedly, consider making them a class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.distributions.normal import Normal\n",
    "from torch.distributions.categorical import Categorical\n",
    "\n",
    "\n",
    "def combined_shape(length, shape=None):\n",
    "    \"\"\"a handy function to append shape to the length of batch\"\"\"\n",
    "    if shape is None:\n",
    "        return (length,)\n",
    "    return (length, shape) if np.isscalar(shape) else (length, *shape)\n",
    "\n",
    "\n",
    "def count_vars(module):\n",
    "    \"\"\" count the number of trainable variables in the nn\"\"\"\n",
    "    return sum([np.prod(p.shape) for p in module.parameters()])\n",
    "\n",
    "\n",
    "def discount_cumsum(x, discount):\n",
    "    \"\"\"\n",
    "    magic from rllab for computing discounted cumulative sums of vectors.\n",
    "\n",
    "    input:\n",
    "        vector x,\n",
    "        [x0,\n",
    "         x1,\n",
    "         x2]\n",
    "\n",
    "    output:\n",
    "        [x0 + discount * x1 + discount^2 * x2,\n",
    "         x1 + discount * x2,\n",
    "         x2]\n",
    "    \"\"\"\n",
    "    return scipy.signal.lfilter([1], [1, float(-discount)], x[::-1], axis=0)[::-1]\n",
    "\n",
    "\n",
    "class Actor(nn.Module):\n",
    "    def _distribution(self, obs):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def _log_prob_from_distribution(self, pi, act):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def forward(self, obs, act=None):\n",
    "        # Produce action distributions for given observations, and\n",
    "        # optionally compute the log likelihood of given actions under\n",
    "        # those distributions.\n",
    "        pi = self._distribution(obs)\n",
    "        logp_a = None\n",
    "        if act is not None:\n",
    "            logp_a = self._log_prob_from_distribution(pi, act)\n",
    "        return pi, logp_a\n",
    "\n",
    "\n",
    "class MLPCategoricalActor(Actor):\n",
    "    def __init__(self, obs_dim, act_dim, hidden_sizes, activation):\n",
    "        super().__init__()\n",
    "        self.logits_net = mlp([obs_dim] + list(hidden_sizes) + [act_dim], activation)\n",
    "\n",
    "    def _distribution(self, obs):\n",
    "        logits = self.logits_net(obs)\n",
    "        return Categorical(logits=logits)\n",
    "\n",
    "    def _log_prob_from_distribution(self, pi, act):\n",
    "        return pi.log_prob(act)\n",
    "\n",
    "\n",
    "class MLPGaussianActor(Actor):\n",
    "    def __init__(self, obs_dim, act_dim, hidden_sizes, activation):\n",
    "        super().__init__()\n",
    "        log_std = -0.5 * np.ones(act_dim, dtype=np.float32)\n",
    "        self.log_std = torch.nn.Parameter(torch.as_tensor(log_std))\n",
    "        self.mu_net = mlp([obs_dim] + list(hidden_sizes) + [act_dim], activation)\n",
    "\n",
    "    def _distribution(self, obs):\n",
    "        mu = self.mu_net(obs)\n",
    "        std = torch.exp(self.log_std)\n",
    "        return Normal(mu, std)\n",
    "\n",
    "    def _log_prob_from_distribution(self, pi, act):\n",
    "        return pi.log_prob(act).sum(\n",
    "            axis=-1\n",
    "        )  # Last axis sum needed for Torch Normal distribution\n",
    "\n",
    "\n",
    "class MLPCritic(nn.Module):\n",
    "    def __init__(self, obs_dim, hidden_sizes, activation):\n",
    "        super().__init__()\n",
    "        self.v_net = mlp([obs_dim] + list(hidden_sizes) + [1], activation)\n",
    "\n",
    "    def forward(self, obs):\n",
    "        return torch.squeeze(\n",
    "            self.v_net(obs), -1\n",
    "        )  # Critical to ensure v has right shape.\n",
    "\n",
    "\n",
    "class MLPActorCritic(nn.Module):\n",
    "    def __init__(\n",
    "        self, observation_space, action_space, hidden_sizes=(64, 64), activation=nn.Tanh\n",
    "    ):\n",
    "        super().__init__()\n",
    "\n",
    "        obs_dim = observation_space.shape[0]\n",
    "\n",
    "        # policy builder depends on action space\n",
    "        if isinstance(action_space, Box):\n",
    "            self.pi = MLPGaussianActor(\n",
    "                obs_dim, action_space.shape[0], hidden_sizes, activation\n",
    "            )\n",
    "        elif isinstance(action_space, Discrete):\n",
    "            self.pi = MLPCategoricalActor(\n",
    "                obs_dim, action_space.n, hidden_sizes, activation\n",
    "            )\n",
    "\n",
    "        # build value function\n",
    "        self.v = MLPCritic(obs_dim, hidden_sizes, activation)\n",
    "\n",
    "    def step(self, obs):\n",
    "        with torch.no_grad():\n",
    "            pi = self.pi._distribution(obs)\n",
    "            a = pi.sample()\n",
    "            logp_a = self.pi._log_prob_from_distribution(pi, a)\n",
    "            v = self.v(obs)\n",
    "        return a.numpy(), v.numpy(), logp_a.numpy()\n",
    "\n",
    "    def act(self, obs):\n",
    "        return self.step(obs)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MLPActorCritic(env.observation_space, env.action_space)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: don't forget to turn observation to tensortype before doing inference<br>\n",
    "Notice that the outcome can change everytime you run the following code as the policy is stochastic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(0)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obs=env.reset()\n",
    "obs_tensor=torch.as_tensor(obs, dtype=torch.float32)\n",
    "model.act(obs_tensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "actor and critic net can be called by the following command"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "action: tensor(0)\n",
      "value: tensor(0.0021, grad_fn=<SqueezeBackward1>)\n"
     ]
    }
   ],
   "source": [
    "action = model.pi._distribution(obs_tensor).sample()\n",
    "print(\"action:\", action)\n",
    "\n",
    "value = model.v(obs_tensor)\n",
    "print(\"value:\", value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import spinup.algos.pytorch.vpg.core as core\n",
    "from spinup.utils.logx import EpochLogger\n",
    "from spinup.utils.mpi_pytorch import setup_pytorch_for_mpi, sync_params, mpi_avg_grads\n",
    "from spinup.utils.mpi_tools import mpi_fork, mpi_avg, proc_id, mpi_statistics_scalar, num_procs\n",
    "\n",
    "class Buffer:\n",
    "    \"\"\"\n",
    "    A buffer for storing trajectories experienced by a VPG agent interacting\n",
    "    with the environment, and using Generalized Advantage Estimation (GAE-Lambda)\n",
    "    for calculating the advantages of state-action pairs.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95):\n",
    "        self.obs_buf = np.zeros(core.combined_shape(size, obs_dim), dtype=np.float32)\n",
    "        self.act_buf = np.zeros(core.combined_shape(size, act_dim), dtype=np.float32)\n",
    "        self.adv_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.rew_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.ret_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.val_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.logp_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.gamma, self.lam = gamma, lam\n",
    "        self.ptr, self.path_start_idx, self.max_size = 0, 0, size\n",
    "\n",
    "    def store(self, obs, act, rew, val, logp):\n",
    "        \"\"\"\n",
    "        Append one timestep of agent-environment interaction to the buffer.\n",
    "        \"\"\"\n",
    "        assert self.ptr < self.max_size     # buffer has to have room so you can store\n",
    "        self.obs_buf[self.ptr] = obs\n",
    "        self.act_buf[self.ptr] = act\n",
    "        self.rew_buf[self.ptr] = rew\n",
    "        self.val_buf[self.ptr] = val\n",
    "        self.logp_buf[self.ptr] = logp\n",
    "        self.ptr += 1\n",
    "\n",
    "    def finish_path(self, last_val=0):\n",
    "        \"\"\"\n",
    "        Call this at the end of a trajectory, or when one gets cut off\n",
    "        by an epoch ending. This looks back in the buffer to where the\n",
    "        trajectory started, and uses rewards and value estimates from\n",
    "        the whole trajectory to compute advantage estimates with GAE-Lambda,\n",
    "        as well as compute the rewards-to-go for each state, to use as\n",
    "        the targets for the value function.\n",
    "\n",
    "        The \"last_val\" argument should be 0 if the trajectory ended\n",
    "        because the agent reached a terminal state (died), and otherwise\n",
    "        should be V(s_T), the value function estimated for the last state.\n",
    "        This allows us to bootstrap the reward-to-go calculation to account\n",
    "        for timesteps beyond the arbitrary episode horizon (or epoch cutoff).\n",
    "        \"\"\"\n",
    "\n",
    "        path_slice = slice(self.path_start_idx, self.ptr)\n",
    "        rews = np.append(self.rew_buf[path_slice], last_val)\n",
    "        vals = np.append(self.val_buf[path_slice], last_val)\n",
    "        \n",
    "        # the next two lines implement GAE-Lambda advantage calculation\n",
    "        deltas = rews[:-1] + self.gamma * vals[1:] - vals[:-1]\n",
    "        self.adv_buf[path_slice] = core.discount_cumsum(deltas, self.gamma * self.lam)\n",
    "        \n",
    "        # the next line computes rewards-to-go, to be targets for the value function\n",
    "        self.ret_buf[path_slice] = core.discount_cumsum(rews, self.gamma)[:-1]\n",
    "        \n",
    "        self.path_start_idx = self.ptr\n",
    "\n",
    "    def get(self):\n",
    "        \"\"\"\n",
    "        Call this at the end of an epoch to get all of the data from\n",
    "        the buffer, with advantages appropriately normalized (shifted to have\n",
    "        mean zero and std one). Also, resets some pointers in the buffer.\n",
    "        \"\"\"\n",
    "        assert self.ptr == self.max_size    # buffer has to be full before you can get\n",
    "        self.ptr, self.path_start_idx = 0, 0\n",
    "        # the next two lines implement the advantage normalization trick\n",
    "        adv_mean, adv_std = mpi_statistics_scalar(self.adv_buf)\n",
    "        self.adv_buf = (self.adv_buf - adv_mean) / adv_std\n",
    "        data = dict(obs=self.obs_buf, act=self.act_buf, ret=self.ret_buf,\n",
    "                    adv=self.adv_buf, logp=self.logp_buf)\n",
    "        return {k: torch.as_tensor(v, dtype=torch.float32) for k,v in data.items()}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vanilla Policy Gradient\n",
    "define a buffer class to store and query data easier"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnAAAAHRCAYAAADqu4R1AAAABHNCSVQICAgIfAhkiAAAIABJREFUeF7svT2wJNeV57cAGTFcR1O0hhsycClDg1kZLFrDiVAELq3lGArWWEtZSMrQUhZrLHEsJCxRFotyREMRSHoca4oeRw4uHAVlsehscBWKQMISx2KNtRyL+v9en9s8fZFZlfVeva/ucyL+ON8f92RWZeK9Bvqtf/Wv/tU/C0GxgdhAbCA2EBuIDcQGYgPPZANvac53nsmsMWZsIDYQG4gNxAZiA7GB2EBsIDYQG4gNxAZiA7GB2EBs4Plt4AsnRl7L93vDVNhKxr8UkvDZVMA92uj9rvDbhT2y4vhJI78u5kxBpzeQ5AZ1v/VaP/R1npsym+M4F3DPdnbzNQFe7yl2dB/3Fj3A0ntdoQ9GzMX3xDjTET94KrPX+zi+B2YuWJhjA7GB12MDPBz7E0fp5CPmDydi7ss1Wl8eHp7SRMMs20FgTuTHIh4e31/YvFccM89dg518nAeeFtY8FbZ3dbJk9GIJSbwX7nqdO9UYrA5zUxMgY0/CEuoUNAq9C8YG7ptWavCx0AtrIQmdsBGKcG3KKjgIxRVOkj8SmOWxaasBuFemKJuvTDmvbOtUD5yjXgGP/T1wbsbwxwZiA7GBO22gsy+6T89UKRZ3JuzqbvoehdRUnnuY9Ip7rC/unXrz0GfeS16CeCkgfu5BTd1rEXtLrliWXJril8w+Nxd1p+p0sv9KmDtrW6+XAVTaSgD3Scz2O2E90WQj29S5JkIvNmVlFJeVJHO9lu5qScN+SdBEDHOcukez/GUi79qmqes/dz881vfAtc8c9WIDsYHYwOQG+GIeBb7seDjNUbGYOf9D2nmg8YCdol7Gx/7iLjbD1HxztlGOqQcR1yTNJV3BnlWjNHWu8YJC3ak6XLtz95ofp5cCHpL4TAwnGh5O+O7iykoudymwIJezXUprJXAfHoVuJjnLXmZ8920eZho89vfAzFhhjg3EBmIDl23g7YnwJBtfyoP5uomYc6asgPcEvuQh9PdNrixJIAYge8rOvnIyMcl08vBB9PnY9LmaFnqTU2Oqrdbw9qm4NJPr61xT3qtYuzfqM8foGrU78jPgYz9QMpj6ks3Z27hWpy47uysxI3RsCmXpqbHNqfWM3k+uv0/mcs/ZqfNtgesxR0Pj4EzJbO1s2Jmr+pvUGxVf3Uvrp17rQ586K3UA5GUz3fwKu61Vfad4lnMvDMLmVOCEL8lWe3IW4GnKhp+cZIGV1zxfI8vI9TpHxE1RrUW/dra5nKk6YYsNxAZiA/e2gakXuF7ddsJgXfkiTAsn4AuPX4V1wlvCVuCnYtgHIQkQ8l4gBiD/CIdRFi8C+b3wXYG6yfCheBHql+tGcv3/2X1TMiC2pSxDJ9Dzp8LHLoBaPxaKQL1O8HHo2KdyZb4X2qkqc9Vz0iQJR9cNH3P92jg5lbKEQcBGTBI64SOhEnoR4JfQ1oI/E98LzHFb6pTI9ShWgFmZeRSoOwgr87Usy0B/4isROwjk1r0gZ4HYo9AJUBIOwmDyjbEhciHi5mjnHFkyfbaG3jghGyEJn5itw+iIXtRKQjZ4fy+lCHUmfJ1AL84KR4fgfo4kvTd79a8kVDu+S2lQwtLvCHoRn4ROYLZR2AgQZxpMhu3MhtwJ/iyD9K2QzU4shA7o1RuSuKcspfY8SK5+7PSgViesBWyD8akcuYJiA7GB2MDjb4AvveLGGCT/QehnRiMWfyXi0KkDZdO3psN6s9UYbHxRkufj0D8ViOOLswhJgHoBf36hvmJzppdijd9NxKcJm5+DnHa2wWz+DK7MpFgsZ9J5wkje4PydZN+X+fy5DtLXLj5LPjY5/poRSn7f5BSnI/oc+vvrvJG+b+Kn1Gx5xCN3Qm9c7IaSUEyujHhmrNRLAJWyhOL09jxeZ/ajkFx85+QpsZex3dlUnLeRU8yQxAFUhK3JML9HZhsFeKWNhOJ0RPRstrX4ofGj1xq95OL8+YzelJpU6QkqjRK2E5FZtuLsO8m900fJvo6fmzDOgK1S7/QkGT+UhWJyZXPXC3vnYtuZqMW/dCYXQw72Sr0EEBQbiA3EBh5tA283nTfSB2er8vtXnLCzWkdXs35J098TP1Ujbi9kYXzFe7lCnZZSa5Be58FV5/S20XLWE7nXNg0q6H8dlNxM9NoZkLPgXwiwQcxez1Ftnp/yzcV/02om8XeE1VTgjI1+RRiE3ngN7SSUqhhH/35jO6US62v00tkTRO+90JnO3OfOXyx2bbyyJOGDBthaGmUAUCcMJmfxg1Dr4hsFP8+52TaK/1R4z4F7wM9Kj2sSPf/U9eOn40u+I1IzhD9blg+9PTs6vkr1LGMT60LOiuRW8v2qje8dH/OJiw8xNhAbiA08iQ20L3CdpgLF0IvzZZYEvrTP0c7ifyTOA4Uvdb78BpfIw36O1o1j6st1Lvd1te91sLeETmA/pTnoSnpv/oN4/VVyE3Z1Nasi15v+/OruWpQmCtX7gF5LqeYQj+z1XnpnhbL43uQ5djDHugkYpX8ocP5e+LGArVJxchU5Qy9sBOr664Wv1MCFnJmowT1S8bdNnePCWpeE8bmuYAfMkc4U6OXPFkP8KLADiLPPUXKOcS7oivaH6HHFcaNUbCA28CZuwL/A1S/UrEV49LaYJT8BSYrdCT8VeJjAs+AfIHzpt1S/vEvruIN+zVp3GOPOqexuL2wM7bkG2Sv8nrPs90WdCnO/bIVD0yTfsemo/NTUQOclxZ/vVBtiV02A10f5ADtt46bq0peXs7mfMtW5zs1Hr4+FXtgLxNf+yfQsfgkVBVMD7lHrXlIrKxicoo2czO7pIOUzAd8pWsvZCVshCT5+NJvYK0Qc9e9C+S7JkRsbiA3EBp7iBvwLHF+qu4khB7Nl8TTh96aVFB5y/PStAt3n9ZbQGYdVufqc62KRhwRzjBdnPt2EQaPxa9SpM31N9vqA49wQPJm8hNW8JbHEULv2RPf5+O5COyXnpmYnvb+gKLHkeNo0+iCdnxSXxj6nbuX4sjD1LzLrmaR2r0lxbwlHi8ePDcrCIGDzeXO1b5JEg0CMz9k0+ovIZf8cz4RR+zARs5Ntajc+lBm5j38l/E5IzklNkJ0NeTS7My8S/Ut8WpQxH+R3Ox8VnthAbCA28IAb+IJ6ZaEXkvANgb/2ZhSgJHxP4Mvwn4S1oYh3QjI7HNtRIH4jZAPyVvhM4At6FD4ROuFbAv53hb8WfiNAvVAfdPhGgdpQNtQvaHLwUfvfCPRPwg/NnsWBj6fn+oq2OptKfo56WbKwEthhMn0UP5Un90si9rtCL7Q5nIvz/JnA+X8mdEIRksliNzseha3Abjk/uyOX6/4lgdpJ6ARoLucgH3nJYn7pZPpP/ZVSnexZ+BdhJXxFoE5L5O4FriNxzAbtjHfiWfDzMgs6oGad51uSk7AWqOnnIu7rwiAspZ8okPu0E/5UYD6uC3v8R4GaUDZwxneFUTgKvxWwfUcg/y+EXwhZYGbqoH9PoPY3BM70H4SvCpyhNx91iKcm9h8I2NZmx9eZnsSpwwzYiMNWhFEgh15QMd6yJEMvcB7kIlTKEr4l1Pvq95KT0FkAOUVghr8T/kr4ptAJ/174uUDOL4TvCJydHGbamq8TZ85kviIOZWEjUBscbqwv9vI9k6lL/V5gRur/RiAvC+RhA9i+IiShCL3pczm1n8KCYgOxgdjA890AX2a7ZvwkfS98+nyP9SQm5wESdL0NJJXiYX1bIh8ELdsAuxomQrsZ+0RomGIDsYHYQGygbuDtK6+Cl4xjU3OUzosdPykKuv0G2r3evtKbm5l19N6Oj8y/WNyWRiWCoOUbmPqXEHYY9/byHUZkbCA2EBu42cAXrryHonr8KqUT/lzIwt8K/IqCX43wa4yg2MBjbYAXCO5PiF+h/faxBnkD+/KSxuef7wF+fcyvKf9GeFf4wRu4jzhybCA2EBuIDcQGYgOxgdhAbCA2EBuIDcQGYgOxgdhAbCA2EBuIDcQGYgNPdgP8F1n/45OdLgaLDcQGYgOxgdhAbCA2EBv43AZ4gfuvPmcNQ2wgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA282AD/ZeX6gZZBn6n/FcQDtY82d9wA146/IeV1vYYP8VngM5DueB2WpmcFPtRne+lMEfe8NvC6ftaf11V4TabtdY698AdhJ+RneC4+EB8Jl3wwLsnpVXuzcC/sbxDKwvi7hPVKPgrZFcHmdecK8YltYK15+MxBw5VmS6rzDxfWImfp5+eS74isunuhXDjPJeGdgkehvyTplrFFeUk4CFy7x6KtGneP1Tz63nkDmztXiAJPdgPX/h/5njtor4D6EOGLoZxLuAd/vkLNr6rG6sI6UzndTI2ltYvyh5ka1zb3KsjDxBNzLp312vM8x3rdIw3NNRoEPnMQ+trkuzL+HO2l98CXJ5r2EzZMS2sXxe5malzLPKgQuG/aqMFo6MQP993Q6tO3paX7b/PeZL1/IofnM/5Q984TOfKbNcYXFx43Ke4dYSv8WCgL855iGF9Sd5n/qPx84cGmcpJqgJb61vCEde6HoGUbSAoDj0E7NQWVuB9XVxhkVA0+T5fQVE6aKfCm3l9r7YM9QYeZ3VzbzP1A331TuL92ozegHnt8CpQ0RHs9n8JcMcOVNnDJT+D4q7Cm/s35SqO88nCb+wBwQ0KVt735EgKefGwnx1ztJu2keqoP/doZKNbm9Cc6TM2YFU/t25Lvf64OvabOMNV7qhY2/qzVNWjp3FnN6DtF1d76qX3qz4TV+Pba0WPqGtXeWUK7v746Z/hUDjWSxVde04lnhqnZagycvI0wOCO2axG1/Fnbmds90Nfn4P/RiWGI9UQ816y1N2F3Uql9yf3bztLqDFPnvu1gU9d53RTzeuvzofha/+7EYG0soWmiRi2BD2LmKpvpJKuxp3KYZe7a1Dz6Ak9TZ67+mofenhW9reXz2viNnN/2jRuZXm0OIdg9N/XGXr+j5uaosZ4T+7UpR9henw0sfYEbdeTDPR07qe5g4MbmA4DN9+tM34kj4x+ED4RKvYTfCfXD0UmmBhzKhiTeG7BD5JC7hHoFjcJUn072JBCzFSqhj4LPSdKzgK8ToCwUgXNWShI6YRQ2wiBcSr0SRgFOjSQUk8VeUicJjEIWemGO8BehcwEryXuBc35mPmxZKMJostiN7Sj4s2L31EsZBbifO7sgenXCaDG+HnHMg38r9MbFbvpTkznx+XsJXxE6gRh6EENtdJCEIqyEStiJGYVOIAdCTkIWetPFbuhUTpGfGp2wN57Ee+EgQNjXJk8x6pfGwQPhOBV8oa1X/KdC7d9JLkKdOUkeBOyVegk+h1j+xTAL+DoBygJn7F6oN/+kD+f5tfGd8y0Rk4LI+YPxmrOXUIQsUJ8+3BfEdsIcZTnIHSwgiZPD+Tz1Ujqh1syvul/RiMMPeiEZH8WZC8pCEegFJZN/Jc782WxFfCVUIp95K3UmwJOQhV5Ah7JAfO2DjXro1DoKvcn4skD8IHQCMRvTxWYpy1OE3pCM+74yvfw88cME+lAfSkIvsHf6dSaL3cyLXmXysEFJoEfNy2Yr4tTuBGKJoW4lbNRJAnIxnsXJg3pDMn0uB/9g6MTpQ22oF8jjfs/CFuMZoj9xWXjL5P5MTrjfkA0UnTM3Z91J7y44P/l8gXriJh0Fbr5KBwnZ6b1kenkqUjbOgJ6d3ksGlbKE4vQq0p+bfilRI7vgXjLzVkoSjk5HLEJ2tl4yaCnLUJyxl7x3Oj5/5iwd2zkixudxZmZMlpjFB5MrYyegUpGQnd5LBpWKhM7pzF31JJl+nnztxvVSLZLauUfZkkX04vSpVCRkpyMfhJWQDLjJ8f2ZDX8lehaBvErEYK9UJGRTkvjhpeeFgI4d6g2m3rAknMsplrAWZ5Ze8DMgZ4uZYkXGvdAbBvHjVOAtbcyfXW4v2Z8JX3F+RHTslXoJoKXWvlPA1gXRZ+30LLk4fU4kb+OcyCvTj+K1ZpKM7qmXAiplCcXpiP77jdqD89NnbOJbtZcBeCpSsjNQF5sn+mZnwF91+vKikpzfn7WXHbSUZSjOuJeMzdNBCvWhLPAvw+mFevNP5vK6c70Ue0nU8TRIwQ6tBf8v2Z10ZvFEn40ZKs/SfV4vHXia2tvOBVDL90Ku9QnbCr2Lb59vuM7ljIpZCyvjSXwQPHWN3qr4BzNSK5m8E88mB3uNNvDFK5ylqMZ4xzpH5b8jHFwdbC21tr0CtgL8LkTd3V0KKNfPPkr/0zvWq+mDq7OWzJcw/DZn5pyVkKnRCb3xsTqNc6YfCbvGPqUmGd8TsnNuJdeeo2TqdcIgrAX6L6Fag1jkUeiEXhiEStRk78nZEMmpqK6tiyGvPtxGs9eevje19y7Pi50UvrjZQSUeHNQenc2LnZRzOcUSDsbRmeEjARn4GaW+QszzdaHm95LrGVaSO/Mx5064lKZ61/qX1joX7+djXnbHGS4l6nRCnTNJrufIkkcBSsJdP8db1eDe8vdFnbv2fNHtsn/O5ZaZMll2fnI1Oj+2w0z8lJldf1vYNE5m6YSd2ds+n8iehNH8c6w906BA7vNeYM6/sUTm4LrAWxrNsDdO3jdNTuLvCF813TPiPPlZkGsvODtgrnpNuZ75lexXlSU5zFVnqJya9EH/ubA/0SPJtxXWFoNe44+Ss1CEjYBO3GCyWNBz3MDbVxiam+RwhTq3KUHfdJvEe8gZ76EmJfmwbYXe5M+u2GdULT7IUDLesupv7a0+lT8qiPkrDRLeN4W64x9dF0lc95Vl1P100pF5eKQF1cjfCj5vQdrJEHq/5fCh5HIy48W8l+RQb2M9qP8rIZ3pwb4qkTuYshPfC0Vgd53Znyq71jXjzFlIwloo7sDY2At7Ojj7XcT2vviuirHvhyTO2fa89HzUmKOVc4xzQbewJ5eTJe8E5vj1TK32TJx5I2wFZvxsJo+4JZQsyF9TZqH+HN0mh1r1nLy4fipwjjnCV2acSXbOlw3EDQaxoOe6gWu8wD3m2bOaH+4wAPlPgU7Nsbcz9uKjsLKBk/G7ML4g6v6o3RK95r4o29hap85X/V4fZPy6QN9jW+ACnfzR4os4GJwNV+YfJ6jItxMGYXRxycmXiLVGUZLHVI1sxktyap3O6m/F2cOPBWxLKCuIvRcL3oiPJsPRH5v6EwMM8lWMLi6dyJlysYO90AlZOFgQ+9yZHT9xlZKTLxFHBVOnONR+l9RpY5n1EqLn1E8TVxNF8oQNUzH7VM44k3MXM33q989WMmeGl6ZoOtGkl486O+Hg4sgBl1KtcVRicRhnCvWyX5pDKc6aBObOAt+bvTBHnJGZWsKehcH43gKIpUfQM97ANV7gNle4EbjJllBqgjrp/YnE3PgO0pPZfE9sPzpR5z5cfIDqDKfOz4eMuSuhQ/mlZbngc5LV2Fl6b7qvRjz2JcR5eJnYumDOVeetZvoNwt7FnROTC6Ae+mC2r4kX56cn1PZ1ITc+HmajGcmpvw5JzuZzzsmchzo1n/hsNuSjk+uM53LIaynJkJ3xIJnac+RfwLcK6lxg+0CvcxFCnI91aa+IPmcu5hL7OBNMH64154XQk8mVm7qI7RTVCaOLXjd6lv7P5k8uzovHxk6OJ/psGlvX6EvUUUGcuVJakuRi9pLfErKzcd5aZ5Rc61ebC30p8hn35yGHeOrPkZ97LgZ7ez9uZestIYkfXDKzV8I3R1mOMuFMsi2dq03/UAa/A/xdG2T6aPySHFKYjfNX4ux7p7dikSGZkd0QT42dsBGOQjZ/ZWOjh/rMNvCFhfNyI3xD4Mb4kuWMxr8nzs3xG9NPMW6k7wh8KVLztxa8FefLBRqFbwnvCvRCrzcfOYA/L/B3Qi/8Uqh0kPAD4T8Lfy38XvgPwr8IxKEngbNQ/2cClITvCnuBmFPEGeps9EP+lpCE3wrMygxz5yGHHr8ROuHPBPaBnoStAFFrNP498XomZmaGXwrUQU4CfakxR50cRVgLPAj/RsBGH4h8ZqtngaPvX7hv+rwrfEkYhSTUWGpg+4WQBXp8xXjNl3pDo4CP2CXUKYjZkgEd1Lk/k/wd4U8ErvkgMGsR2E8nMHOdGxu5XxbI48+vfF34qfA94TcCMchQjSeWHGgUOPu7BvZEzZ8JPxC+IiSBnr8UIOp2gr/e1J7LYYf0WJEsIh+qu6XHXwpfFXbmm2LMxZzUGoRah9he+BBBlIQsDDfaix2iM98cdXJQG2IHtQ8zj8KXBK5FEiBiOoEzVJ0dYP+OwOf6nwRmJIaZk8AZsL0ldALXjHMzWy8UAeoEelLTn/OF99V/UpPZfmjxeEfhGwK9OQN1atwvJb8rkEMPZj6an3jyyMHHWYj7e2EUyP2ekARi0ak7Rfi/JSSh9iFuFDqBs/+FQG/+OMK/FpiFnqfuT+J/JjDHV4QkMG8RoFHoBO7Peg2S5K0AMe8o/ELIwrtCEjhnJ1C/6nCIuTqB2Kqz0ynKMnIuzvznAt9PewPx9OeMfyLweWUOeifhY4F4zr8WfiMwD1RnYG9/ZToxzFGEdm/fkA1/3f1Kcmc6OdQuFkPsVwTOx6z4oX8W2HMSfibUXtRtc+jzA+EtgV6ck9mTQDyca8J3Gz3wT9Foxm+J8/3OLpB3AjNDWTgI4wv15lyDycFiA/e6gV7VQdDtNlCUlm+XetUsZkgXVCyKJSfo+hvgy7xSlsCXvaeu0UONDdzXBnoVLvdV/A2ru5k5LzvunG+ciQvzM9nA289kzhjzeW+AL5T6pZIkj8/7OK/N9LywdXYark/7AvfaHDQOEhuIDbz8c3CsIgtD7CQ28BAb6NTkVwbkoMs20Cv8d8I/CPmy1KtE83LADPB0QUVyHnPuC0Z9tqFZk3dCak5w6bV6tguIwR99A1kT8GvQT4Xu0ad53gOsNX46cQR8fLbziZhwxQZiA7GB2EBsIDYQG4gNxAZiA7GB2EBsIDYQG4gNxAZiA7GB2EBsIDYQG4gNxAZiA7GB2EBsIDYQG4gNxAZiA7GB2EBsIDYQG4gNxAZiA7GB2EBsIDYQG4gNxAYefwO9RijCXuC/Ll3d00hJdQfheE/158pyns6Q5oJO2MnPJ/xzLv7rog+Eb88FPLC9Uz92z/9EMyg2EBuIDcQGYgOxgWe8ga1m54HOS0qV1/d4Hl5q9vdYvy2dZBicEXnp+bJie6EYF1tM7DJZNPJBWC3Ovr/AotLxAnd/+43KsYHYQGwgNhAbOLkBXoR6QzoZedrJy5R/oC99uTlddd67k4sXmocizpddM86H7RLqFQyWUlYg8HSQ8pDnbtq/VIukeIGb207YYwOxgdhAbCA2cI8b4EVgZfWz+Oj0S9sWJTzkA50XGV6iHorqTxd9v0vP2ysZLCVixyZ4J70sLXCPccxw6fnvcZwoHRuIDcQGYgOxgTdnA0cddeOOi55vcXxeKsjlgV4EdGglfCTshV4opmOHOmEUyNsKg0Ad5FNEPnEQ8kbYGzfzVdla1aZeVrDlCzr1igVLibPR21ORslta4EpxSXX42x4GoTf+qXi7kyzbxwLzEXcQvi8ExQZiA7GB2EBsIDZwxQ3w4sNLQiUeyPWFoZOcne+cWBTQPtB5gGP3hM5DvhIzkLc3w9HJLuwVkRzimX1rnkE8vxL1R4V4ep5DP5NP3fZshGLDt5R6BYLbUlIi+4E/FLFj/tqroWl4kO53kkyv14NwconpUYJiA7GB2EBsIDYQG7j+BjqVLK5sL5kXn6VErn+gkzv18O7Nnq0wnLjO9CVspyBwSc6SunMxWY6n8AK31xzruSHvyd7b2dt7oTQ74XpMvdC2cfc0ZpSNDcQGYgOxgdjA89/A2xceISm+E/xDupfOC8Nt6dyLRusfL2iUFXsUqIF830SvxyZekHrh8MCDJOt3bgf1eo4z861m7GGODcQGYgOxgdhAbMA28MULNsGDdSNky0E/97BeUv7ci8Y5/1wP5ktCLyCPxpGhqdk3si/5s1ifKK63Op7VWenR1i8T8dc2bVVwEOocWfJD9OUcI/9YQMS9JyQBuaV2b60/9NhAbCA2EBuIDcQGFm6AFxJeDiolCdkUZPxLqSjQ/5qR3FH4uCnAS0h9EcGVhalfvTVpL9WNpL1z1p7Y75MGFfc92jmS/PnMAL38oKUsw7o1mk6f1rd1sZ3k1Ylc76OOr4Xv3P8cOCnmKOyaHgfp/npn07cujvrEtLlNqVBjA7GB2EBsIDYQG7hkA3sF84D14KELFcE/jM08yTpZPxWo84GQLWotXgT+C0bscPQkQFn4SCAP3mE8Q738fq4inT7dmby7upMK0Iv9AHZH30o7CUenezFJ4SeAvzIgY6tEreL0Km4k+GtT5a2LpWc/kYsJHzUqDRLoVanWd6ZJkbhR+JHAC9/O9HrdahJxB4vjenPeXljVgOCxgdhAbCA2EBuIDTyvDeQrjZuaOrwcXKv2uRHp1RmmXkruMsddcnlxeghKagKgtZCF9EJ95Z/sZso+ERqm2EBsIDYQG4gNxAZiA4+7ge4O7e+S+1AvcHc4XqTGBmIDsYHYQGwgNhAbeHobSBop33Is8si/DcXL2222FjmxgdhAbCA2EBuIDcQGtIGpX6kuXcxdcpf2iLjYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDz3kD/EcvW+G2f+6S/PfvkP+cdxezvzkbyFc46jVqXGGMKPHcNnDJX6X1lM7GQ4WHw1eFUUDfm5xNFpulJA/5WfimRfXi7wg/Fg5mexMYu+uEUUAeHujQ9cXg+ED9LmmTFDxeknBBLOfmf3JMD0/cc58IT2Efg+YYhd4PeKG8V3wWivFz56LX14QPBXbxulDSQcYnfhhmbL8PGfk5zP6Yq822o3YGPuNbYRS4l8/dz8RvhH1baELjz2X+AAAgAElEQVQnFjpO+O5q6lTgPYHvoeGuxR4oP6nP+EC9os0VNrBWDT4QuanVSS9C39jnVD4I/A0BnqbqzuWfs+dzAXf0pzvm13S+NJh1I/QnahJzTdqpWLlSwWvO1mum8Upz+TLcb9Tm3IB7r+58kHwUfid0wmNT+7m4yzxFyXlhAXawNHZhyUcPGzVB/+hTnB+A+9Nf92R6Pp/6RkawLz7HLWEfBDhUjJ9j1Ko5p2KpN5wKuMCXJmKZo5+wP0UT83PPrp/icFeYifuBM16FWFIn8NODxyIO9KmwmRng0puvfVAV1c0ztS8195cmXBCfrjhnu4O5Ma59nmvenNec7Zpz+V0epPAZgjqh3Tt9icGeCHpEame7yyhFyXlhgUtiF5Z89DCuOdf2OVB73fNzGPqRZtyq79R+Btk3bqZ2p3PjJjnIPUfEXeN+okaeaNbLBp4L5ecy6C3m5Gwnz/f2wqIU4YIPwpcF/u7KxyA+NG8J+5nmw4z9oc3dPTe87/rt+P4LqfXdVj8qcbxtcpN3zX1cc646ZpLQCwczoPNrCk/0zWboGl+oz3cDXHOu7XOk8hyHfqCZs/q0+0myYZ97Pp0abZST/HNE3DXup+5co2fiL89kztuMuTmX9MVzAebvxEehGF8vzLt2GAc69eHgyzK5pswJuOFXwuB8S8XOAms+tSptJVS9SB6FXvi+cBA+ED4TBmGKqLkRRiFZwGC8+uD02BvfiVP/HeE9gReBIsxRkoMe0CgUgXpJeF+ATs25Nf+hieukMwPnG4UsMCNxK4GecHphh1fKEpj9n4VdY0+m17zqXkvIpozi1ESvu+AM0E+F0WRyALVWwmD2LE7/twTq1JiDZHaShO8KlWqdqg8mJPEs1Provfk8G6WAStSjV0vU+XVrfEJ61izsraWfyzB1njZuib5S0MYCkQeTe/E/mFyvcSe93oM1zkJuGH7+XB3xSchCL1Df52Fv70fi/lT4W4tP4tDOeBav9xA29CSwhyJAnTDVh/tuLoc8v4PR9CS+F9A9Ya/37IeS1wL5R4F4T0lKFqp/eNX9itZJY/b22mKHqFGEg9AJxEJLrw2x1ODzO0V8Dtr5p+KubcsqyHWdonq2tZzst6VeBj9zagPO6OwyC2UmDl+953rJSajXnvuUuYihzqnd7eRn74NAvU+EIniiFkgC8UfnrD5M2E/1IqazuCRehIOQhXqWQXKtSS3vl/ryGZokc16Ia9EJfEaRyUHnPvxMGIUs1HoSX36vJMlFIAfqBJ+XpIOdcBSgXvDfByuzD+JZSAafQ8jagEwt5klCvW7flbwRfD3kDwSuET3nrpFcl1OnlOLSGLA2v7zaZRl/UHi/MCUrbnCxSXJpcqnnCX92hoNkzgcloZgM40Jk0zn/x87XSwbnaNcEVJ169IZDWSgmw5CxnaOkgMEFUY+5a11c7Q5c+EuxlwRaYt4irAXkjQUM4nOzW8hNbKmK5TJbJV8vy1icr5O8dfrUGcgZXAwzep3+B6ETkOkHrQRfj7y9+WBboTfd28kbXNwpcZSzmwk4nPDNpLx8EPKhPwW+EJjzFLXnr7GdhJ2QHUcGp2oWixE7S8T2LipLHpyOn/1XWktgplNU5BwEZkQmByKvNxm2EYrTEUez1/MN0jsXQ87BYjATNxqvYVN9zuXg9z3pk5q6tT6cWO5Z4ioNEnqnryVjq0ROcTpi+zlijuxiBsmd0/eS65zIvfNhxzZH1d8pYCdkYStsTE7ilxD1uL9P3f/4Tn0G6F1nqbyXLRvoAaH3Jnt2lPKRUGdALhNxcybO3885zc6MviYzcd06l3eQvHb6lEiNPOHoZdsLNZ9+6JVyo/fSuz+6PyftZKFGJWZLpmBH70xfiR9NhuVGx9YLawQj8rPTd5KLQAzyxnyD2Uy9qVN92HweeifsERy1nzH85NEL6gSfkxud2NpzJXnqupFTqUjwunO9EN/+nGXekOTixqRgHYLoXugQnhgx1+BmGiWDztlOicQdhYMFjcazOL4kFLPB3nLyUpE9Zhc8mNyJ05f+UBGSwEW/hHYK3ruEep7ukiInYqlXa26bXmvLK+LvTdQgzxOzgkp7CdSEBsH7VtLHF67Zf5LTO+9BMnmd2Y7iXxMGwffC7qmXUpxhkFznWkvO5iMP3zlKCnhHoGdLzIe/+jaSuzZoQqc3P7H45AyIIfYU7eT88UQAeVuhCMjwCvSVQewVGqSRh38JFReEvDZgHoRvO3+S3Dt9ThzlYMYsHCwI3VOr4xubWPTkksjhHtq7OGKYuVJb91xOdrUQqb0VRqGtZaEv7XUO7OTwfZ0saCc+mAyj1kEgbo58v7WCsjDMBPeyd85H/KnaWf6NMAj0KcJK2Js8ikPEdCafYtS4xmeAmQ+GIj4KcEAPaG3cM2z8tOS7wocGngnUglYCtadyLeRlbNWneJ2h+qpeXDA2+t2WyD9YcltrkB1U2kvonO7FJIUXZmIq+XhqEzOYE50dYoOKMAqbF+rNP4k5NLpTb/w1Zit5L2QhCT5vJ30QKpEzCjUGefVH9ytxVSGWmLmcQT76VNpL2JpyND42/uz0s+IXz0b8MYBG3JgMwL9Z/I25NhfUuGsoH9B0pshafhbKS0NpYkfpWRga+5RKnZXQOydLB0kYG3t2+lKxU+AgcNMWgf1Cdae96TDOlIw780mRh91uIoL6U/aJ0LMm5mqpkyEL9Rytv9VXMvBSQzx5EDZqV5/vs7OYOZbkoN7YBFBj7WyfzBVw9iz5KPTOVmfBthf+IBShXj8X+jkxyzL3IrWVD9APgmeTz7FyLuCMfyV/L8CZoSXOWSm1Tukb4WDw7kEK8UUgZhQuIXZQaw+Sd0ISRuP4zxGxt6Vz9bmWl9KpHHa4cgXXksulDRTP3PQhfxTm/iUqy8dOz9FGAWMThK0Sc9MzC0VYCaMwR3tzJHHy5miUYysMcwHOXhbEnAqpM3WuX5pI4Gyc1xO29vtkI1u2IM64FopPuqI8PlAtvlezwFkqjU72IjGfCb0zJsnF6e0ecREzWsxOvBP2wtL9tTWzclvbUTaeu9SsvlHypXQqp32mUbuNb+e6qP/SF7iVqnJgaCf8SNgIe7M9FKP3ByeaMedaOFhMEh9PxJ9ycV5y+4kgzr6UsgLLTPAoexKYmZrEcQZ6c4ZeOEdJAeNM0D/P2Kl/W0pKHM8kD/IXAb6E6jzEc+4pYi9LKCtorgb5l56d+EEoJDe0lz4Ia2EjoKcmplWJG1qj9E5YNb4svUzEtiby+DfdJfRjBXGmlrBtDf8g/s02wHTmP0z4hgkbpt7s6xn/ErOflz7MCd8vSVbMuDDuNmF+tqX5p3Lw7QzMvRL6pYWbOHKvRadmrj2YeyscLmjaKXZv8WkiL8tWJuytibPe9TNQa64lcN7cNnE6MXXuqbCNjOwBVCLH61N5j2HLalouaMy5l8QfFce/3PYX1G5D6bUTkpBNbmOW6Ox+ipjxPmlQ8VPXfGn/pDpjO+jbrWFG/53sm8a3mom9T/Og4ryUbGeaMOPefPzb0LqJQ6/+mRIvzUVSaoI4MzXwtbWrr0k5+SXQW/BBHHmwuszY1k+yAehoHJac3IrUaf3o2G9L+UwifmYfJuL6CVs1cb1S48+mT13L6mtLYmc//PRh0zjRS5twRmdXU3VI6y33YPJ4ptZKfn4qOri4ZLmYts6OmIVifBBfY5ygo2zsaAmIPUU7OfOJAHbBeT11UobGVtUsocz4lpiTgvYukPk6gV2Mzn5XcXXXAlfMZ5atwFnhx1vUTsqhzsFyfy6OzdNayr6xzanEEU/NSsjYKhGTha2wtC7xByvwfi3kOP4iwHeC7+fCbna05P4nhtg5yo0/TQTWeVuXt2/l7FwAdUZhZfbB+arI2Yh5CGIHyRpVvqQv9xFn8JRnEtnHly+InyrDnHuBfY5TAQts5Kcmjl1/doeaC9re3I9t37wkcSKGeSu93P/SF7ifKrNYdhbn4CwFqss19d5ZVoeN0P7bFrZRONoEnTioh83mq3OvLS4ZJ44fqVa9SCa2Nz+sEw5CEQahFyp1EvBB8GTyaHyKbWSsc9AfIncQjgJ+CB/yaHoxHTWZbYptZcxCrb2WTN3BgpNx7KfoIOdUDPn8mLjWpwb1oWrbSOZGzkK58UxTJ/NWSOb2/bDj977ah3Be1vCBPQZRZ6hzoBcD/iQwO9xT7VvtvZzYWjs5G2enz4jxBG0thnuXnyQPQifsTBZ7hZI0ehyEvTC+6n5FK9KW4ESJRS7moU+lLGEQ1s52W/HY1GFfvTC6gsjF6afElZx8pt+bCKJGnZm4LPj7AVt7fySzid0Q+tcEYqFr5TDXx4aPxLlfao8Xnab/2ZmZ2F7YCqPZkDdCrZMlH4XB/KnhxLG7tdlHcWrsTId1wsHp1Ns7/ZxI/uiCPpNc+1VzkpAF+hRhFOYI/xLM5WPnjNSAOE822TNmaecssiULosYgjKbDskC9JAxCEVqiJrVPETFcl5UFpYZjn7vnfd29lOwNrg73faUkAb3220ruBOwQ81SfmV6yo6RO2LmYjeTRIpL4qbNY2E1+J5RqME5f8pOzI/t5cR2EQehQRORtBWaplCScOjc5EHGVkE/ldPLThzhobRyWTM7Oht/XK9Krv/Yn/CD0CG/xjwVEMoc9Csh7k0mlKfYR5QGJeWpv2jLT2PRnVuLqfAfzYye3Enavj9IBhB1Qoxi/cYiwZ4H8cmP5I3USV8IgkDtFG/MRB/ZNLPUB+fg8dVLIGQT8p2hjTuKKyeRSu9IoAcxRloP4QaAOMjUqFScnyfQ8GIhDHwRyoSz0xl9YXtQjjhhyx+oQrzWwVVR3ktAJxVDt5GQBTj0AJYOpN3Z6EreuRvHi5I35fR1iyYGSMJg8xdrao4LAHGU5tkLtt58LvAf7H1Rz7ruhk29oerIbdtE3dtRidvg5qrvMFsjZx4mkTrZhwt6amKnWPEqmnqcsJQnV15mzF8dXaZRAHV+LnPTHkJtzZqePkn0OOpSMw4qQnT5KJmdjvuoiBtu6GiY414yfdhBLjSKMgqdam9nxHcyJ3dfG7nViAYQ9C6NQBGp52kghf2zsU2q2uBpLbhFqTXrtzEbNvXDf1KnB4Jq0enUVCdnFISaz4Rsb3yCdcwHOVM/owzjfpsnzapICIPJHYW067NDo+MEcdXKshEGgHrXQIXSQTIcVk4nZCPhH4WD2OZbkyALxxTg2UKlIyE6nJvGV8BWnI2KrRHwSmK1ScTLi2uDnqPaahw8kl1skZ6e3veZySKHuRiBmFMjFthYqTdXDBnXG9+LUgMgdnW7mYLGB+99AshZZvNx/u2fbodfkG4EPKx9e+OqBTsPLwCXEfEnIE0llxj4RetLkz749Gfm8nexrirCfuv6XXrOpHre1JZfY3bbIRB7XmXrULwKfAeSnQJ2GAEvpYIG9eDb4XGyv833tzxrylTbw9pXqRJnYwNIN8LDfCDyMytKkNzAu2X744gc8vI4PuAeuz1JivizAW6LONeYeVScJWdi3TV4jfdBZ+uY8XPtRmNtjvVaVP/Q6mJkZweGKzTlPEUarW/dwxRa3LjUoM1+QXSwWnoT2WnayDRYTLDawaANvLYqKoNjA9TbAlzBfzEkYrlc2Kl1xA71dn614+6BZ2iYp8H3hq0K3NOlEXL1vmOdwIu51cCUdIgt+9/uZgxHbOd8geZyJvS/zm3Rt/A6TlE7o77hYanBPv+739R3XFOmxgdhAbOBhNpAeps29dVmpchbgtyFy020SIyc28Iw2cI17/Bo1ntHKYtRrbSB+AnetTUad2MAfN8BPJLKQhG0sJjYQG4gNxAZiA7GB2EBs4GlvIGm8jY3Ii1yVn/bUMV1sIDYQG4gNxAZiA7GB2EBsIDYQG4gNxAZiA/e3gef8K1R+uvFtW80ozv8h+ihgP9zfyqLyhRvgevA/J+Rv0CgX5j6n8JWG5X/q2tJnMoytMfTYQGwgNhAbiA3cZQNfuEvyI+XyoPzfhb8Q/lfhHwUekluBFzr+q7ciPHXiHL9/6kNeYb6vqMbfCl8Syi3rsSvyn/K+mO/f2Bm5HwH/gsQL7CD8UvitEBQbiA3EBmIDsYFH2QAPU/BYtFfjfqZ5OeGbSXkUc1ZX8KZQr4OC21JWInjqlDQg/0NVuCf+5eJXjS3U2EBsIDYQG4gN3HoDb98ic1AOP1V4DOrUNAv9TPM5+0z4o5kfa3+PduA7Ns53zH+odOac+pUp/8LDX3MUFBuIDcQGYgOxgats4IsXVuHF4zFfPjr15ydwc1Qm5tvIxgP0YCAXnT+vBOfPzmWBvxT3E+EoJAE/D2PyoHM5xCSBP+/Fr87KRA4xzPOB0Av8tKZ94OOHRqH2NtMrbC2NmclnNvpylkr4s8BPfjgXhF7/yh3yyIH482n0qjWrTl1+LV3PM95Ef34XzIzPz0st6v/6JuNVom4W4DXH5/poan9f+J0wtS/q0Muf0+dPyeTU640M7Y3D6Flnm5vLhb8Us6Qy4ehk6yfsYYoNxAZiA7GB2MC9b4AHGg+2IuR77zbdgAd4P+36nDXJMgjMDXVCbzI2dOrB0cEocMa1AO1NR8a/FciBoyehCDUenRxsEDEbob40oWfhIFADOQkQPnLXpu/EO5OnGHFFoE6VswUO4huT4dSCsjAKvZAEfMxGPpSFUah6zVvJtjc/ceg1t7P4ekb8g5ARRNQqQv9CvckdTIZtnc+Zb0Ris3AQiENOQqVBAvUhOPopol4RiEWmbhI2piMPJotd/H9ZHy2HXKjORP2g2EBsIDYQG4gNPMoG6kOoqHt2E6BvH2iiS17gBs3UPjj3zezUS272Innn9F4y8ORfVLBnYXQBveTidMQ2B39uYnrpvvdK+rGJadU2B38WShN4kJ7MRg/fZ5S+Nh89O5Ox+bk30vfmqww/dohcqBd8/WrDDmXhI5Mr2zZ6qxbL8/ZeSptH39bmc5iVWpWI750+SCbG015KbmxTapKx7oN45iB3PRUctthAbCA2EBuIDdxlA28vTM6KO8zEdrIPM75rmz9TwdWZotX/vuJKE8sZNo1tbHQeup7OPYCLgt8RzsU1ZT+nZll4AXjPwK9w+TXfOTo2AZyPXzfWOnDqJosbxOsOsHHeznxZfDD5IP6WkARqwFfm84w4qM5BbWrOEfFfF/iV54+ELOwEdvjfOvzXkk8RfWrvGkdf7P+l4Gv92xOFjs73vuTSxNKDmucoK4D7kxmKsDN8PJG4pN5EWphiA7GB2EBsIDbwYgNfXLCIlWKSUGZixxn7fZgHFf32mcKd/Dw85yjNOWbsnH8JLY1ra5F3FOCfGGoML1CXEnV4kaBWJV6YKh1MWItnoRewbWuAcep8JFBnJzDLpolBHRtbmojxpqMUelMrC4PQC/y5u/9OqPR/Sfh/nF7FJGEUmG+O/hs5vuOcB8n/UdgLnIV7iH5J6IVzRNw5ygooE0FTc/aKY5ag2EBsIDYQG4gN3GoDS34CxwMoCb0BuTOIPSjx8P2qkGe6bmSvD0ZeYtZNXJJeZnJva6Zm/UP/UzXwn6JszoP41Lyncqd8U3WISy6YPXamH8XJ2Tg/Yi+wQ2KhlXFYO6dz3ew3eUMjZ/PvxbcCeif8g/A/OPwfkqeIWIiZ2znQR+H/FHyt/+0m48VcnTjnOgob4zdO0V3umaz8YnUqw0bNSklC7YkvKDYQG4gNxAZiA7fawJIXuFGVewf0wSB24+Oh9BDEQzcLg9A+vJNsK2EUoE7YmgzDl4Sd2dBvS7439ehztGL7pmieaEJs7Z/MT43O2TFvzDfHag3v31mNtTN2kulZaS8BGxyCk1d1bKsXrpf/rPWwe18bt5V/43Lx56YWMZWOEg6Nv1WJqX1Gc0716eTr22SnUwc/P4ED9VfDNaST4GejZxJ2NWCGE8OvgEvjxz6abWOcmnuhmA7rBF5gg2IDsYHYQGwgNrBoA19YFPXHIB5C9f+Gz8MQrIXRcGG5W4Xzf7MfhK3wLeFd4RsC/yf8nwmVRgm/EYhJAnP2AvOjY+d/cYHMOTgbfCUQw98gAOr/PX+UDPXC3wvUo0YRfF/iqfGukARm4G+NyMJBoMcvhE6gPjI2eg7O/g3J1MU+RVlG8iH60afSTyQwGzMm4ZdCPQcx9PuSsEcRjaYX02HI2eychVnoBzgHvv9kOvUABGcW+jMX5yB+g1M0CsTgZwb8zPt7YY6Y/zsC8b+wWGoUoTM79bbCb4U5elcO5v8X4S2BXy3/UKAWM45CnT1JXgu9cGq2LD/4v4WVQC0AjQLnY25qcA5m/JnQznnqp7gKD4oNxAZiA7GB2EBs4C4b4D8ICHqeGygzY48z9vsw86II8XIYFBuIDcQGYgOxgVttYMmvUG9V+DVNSq/pud6kY7XXkBep+lL1EHugF7++XT1Es+gRG4gNxAZiA6/nBvg1UtCyDSSF8esvHvhFGIRRCHo+G+ClqRNGAfkoJGEwWSwoNhAbiA3EBmIDsYHYQGwgNvC8NxA/KXze1y+mjw3EBl7TDVz6HzG8pmuIY8UGYgMTG+CnzTuB/wjjMOEPU2wgNhAbiA080gbiz8A90uKjbWzgiW+An7xlYWNzdk983hgvNhAbiA3EBmIDsYHYwBu/gfjV6Rt/C8QCYgOxgdjA/W0gq3T7P2ONB8/pfcd+Tu8nvJ/fAJ+zu1KnAvxKNujpbCDdwyjXqsn3FN/t1/y+otY1693D+qLkA25go16Xfidd6/6+yzFf3sfP9VeoLL4IHIT/kpaL0AvYt0LQ5zeQZSpC7Ofzu7nEslMw99ubQr0OCu5C7IvP6uEuRSyXWj+6Qp03uURn1wJ+LcoqtBeGKxTkGvM5439uXa5QjxK9MApv0mf3Sqt7WSZJeszPHvcEM1yLuF87YbWgIHGjxS8Iv7eQ3uZ4tvcxB2DxLXERRqFvHU9QZ/n5EeZiNw+9n4fud821Tl0j7r3NNZvM1HqIHjOtX5qTpINQzgWe8U99Xs+kzLrZyzXrzTZ6jRz9xFmwTdknQhebsiLL4uj5QK4vtVbCej7sYk+xuhcn3jLhKXyGbzn6zYtNS1yLInBd7pPokycaFNmueT/QgrMME72mTL2M4LGpaIDMEEt/Akcwf1fjB8JHwpbkR6Csnsww1f84Y3+EMc+2vPZNeLbhIwU853Nyr03RRsb9lOOKNr5UnsLuOOsg/OkdzsY5rrkvajFX0LINpGVhTyqK+x86CocnNdllw+TLwp9UNJ/blrgWnInrcp801Zt+9L72/VDPku7zQPdVe+kLHP35cTa/rvy5sLuvgc7U3Vr/cSauzNjnbojVTPwp85KcUzH4vn+qgXxz806lUe9c/Kl5pmp627n6p2r3J4qfq+tT08QZycd+Cc3tKasI9fxZdpcUPhF7aj/0naJzvefOgT0Jp3pO9ZuyZRmLcBDm+k3ltbat1Wjt6OfmTK73udip+qdsdVenYvARt6T3kphzvZb4T/Vh1pZ2reEW+lTdW5T5XMo16iZVBfdFt73+5L1/YihmBkuIWm0s+ql7oa07F0+Neh1qvXOzt31bve09pfu+3o/93POxrTd3tjZuTi9ybOact7AzD5gizsef7YTavaEn8y1il7zADarYC/tFle8nKKvs4UTpo3w75+8lc2FYDPZsviTOOX5nNmJAEdYmZ6dLvFlszeksBo6N+lAWDgI2KAmDUP/+VOK2wpeFTuiFLFRCx08cNdZ/dE1KG1mzxQ8T8fh3Zkf29cijB7N1ApSEg4AduRNqzuBkiTez11xqZ6EISYC2ArlZ6IVOqISchZVAr9qj+ncSyO+EQUhCFiByBiEL5JGP7RRRr7M44smDNobR+PDCfDMvNTsBOQtQJxyE/oV6Y6deEZCptxUGIQmdQK9iutgNYetM7p2MCXsSstCbLnZDWdiZTJ/O5CTeC0cBeRDuSsx4uGsR5U/V4QzcO/igTqBX/0K92T0yHEImNgk198Yh6oWjsBU6IQtFqLUlvnLP9NKpQU6Hc4bo3QvwTiCnUpawF4qAvBE6YRCgLOA/9dmy0FdYJ+0g9AJyFgYn0wdbETx1UnqBWZkTHdoK2LLQC53QEjFZGISucfbS8UOdwdQblgX6wZkNLKFOQYDZqN+7JOQkdI3dhbwUiaXGWtib7OOoDbLQNX70gwE/tBGOwk5YCeTCsWOrlCTshSKsBfwAHUoCOrm9ARuErTNOLnWwVUIehCz0wk44Cp0AkTMISegE/Keo1iM+C4NQ+5GLnfrIW4H6G4vpxUESIGLmPredfFkYBPLR4dTE5glbFlbCXlibEx3f1POxk/0oZIuFkbcTyMNeZXydcDAbchYGAbmlIkNujTM6PTuLZ1bkSsyBDiduMLn6e7N/Jo5MPkT8IGSBvL3ZxG6IOJCFTiD+IiKRAp3BF+hl21xU7fbB3Dz0W0LEdU1gkZ6cjXrZ6fh3Tu8lg0qcmxx//k76xy6ml1ycjkiOJ/y5sXXSB2dLkscmplX9h2ndxKOXJoGz9c7GOY5CcratydVXz7qRfe/iEIvQOxuy11t/DaXn2pQs/qvqEKdPcfoguTuh4yNmjjjPzjnpW0yvvLp7F4cvO93H+Dhi+BeB5GK5LpyjEv2Zo1Jp9KP05Py9ZOBpJWUU4JUOEpLQC/ml9e6fR2atfeD+PnNtFollJqr97PWKA1BnMPXmjLkq4lOfp53z95JBpUFC53T27ffoXC/FXtLeOYrkjdOz5Pa6Uze5mFbfOt+c2MtxcM4s+VNh7Wz4sUPYfTy2UVjdeF/soTfZM2xTfWoMs7Z5g2zZApJ423cjWzH/HCN/aJw76Z2zFcm5iWlV/OylUi8BzOnYmTe7mE7yx05HxAZx/r3JMOTO6Vny1HVZu5j2PsXV7oiz06vSIKFz+lHyynQ4PauOeS9k80+xIiM9K9V+WYbe2anp9anZCT8I2eWRg60SvvZzQa3kYo6S16YTX5wPER17S7lB6EUAACAASURBVN7OvOiespS9M/SSi9Pxe925Pncve1+Vewnsz1ORks2wEUevROzWlCze5vbmG8Q7k2HI2KDeYOoNOwgZ4W1vPSEf5SvCYNwvaZQNPATxa9ylxOJKE4zeTdi8qc1JzskeoMqRByELCeUO1CmXC1NplPCO06fEr8tIXCUfP3V+Pzc56FzLzgqsxEeT8W2EmpMk42+ptIYFOnUPFkf9tctJTf4o3dvel15zCW39TfrnvjDJfc8FfSyZmsywa5MX6tyXo4v9RDLnqoS8cnoneW96Ev9UgJ+iTs5R8HWL9CxwpkH4QOBsRbgtJSXyZ96+JlALDvn50dOE7UXk3f/JeXbCj4RvC5wZ2ylq/dkFJ8l+b5yvPU9be5Bha0ZiuUbrJqi97syQXMwgudbAPDrfKdGfpSgwCd52dMmbxodrFNpZXcpLcapPdXYSSpM0SscOwYvJlfm5GtdLlX34vjj2QjeXMGOnxt84X5Kcnc5ngbqe2vkGOfkOTRa0ES8m4+tNXom3LyW4kuDPQn1iT1GRc2sBxLb3YpLNz+n9WT7uOe+nP/YpSjLyGd47J3IWRoE56meMkB3/OEO+dw319bG1nwu+D5Oru5HM3BB2ZryUqNFSkYHvi5VzYFtCX1sSpJj2/NTfWq6XmcFfO87bCR8Jdcad5b0vXveBaRSS+ahdTK7s5QxLX+AoXhuMkpOQrdrgfGa6N7Z3feeacGCI5Y0TQXnC5k0vl2PGdCa+upfGzZVbmSOLV3xzLtjs9OyFjXDbuXfK7axeFi8mw5gJP/VHZ7+reKruYH2JAfTGBqFXniUAaGt8ivHlkITsUPdK7Z8L3xWK0Au3ofHCJM7BzJ2AzBfeOSIOZIe95CLAO+Grwk+FjwVib0MbJX0ivOXAfG293W2KL8w5KC4LXxY+FHh5SmdyxxN+Zu3Mvxbnmp+KJ/QobA1J/DOMDY2todF934185Ux8dZ+r68twHig79JLZ4TkaTwTMPdCS5WRxdnQprZQwTiRd+hCnNzvthdzUpAe0ZAeD4rYW387GgxXfWpi6/pZ2Eatz72bqYu+sIn39vYoOZYciebixfp6SmbJ4BaZeGAXq8Tmvn7Fan5i70HgmeSU/59wIxzOxc+4055D9Nuf49Yl6p1zMz3kgZM7UC8zg7xl82PgurfvOkmsuHB1AW+O8xxSTP8e++DnLtIECFDyYm2Eeg3o1rV/m48QALKiYnUVlp2NmSdVvYXdm9QKMM5Wqf8b9ckby2/lO5XLWnZAmCmOj3hI6KOgoZIF+yNBG2ArZ9MpRkYvZl7JegSALfm56VkoS6I+/MzkbF7vx1ZedgsHI13DmG/ETMxTnqPEb2egFsB2ErYurYpZQJuy3NVFrLYxNgTRhy7IVsx9NbtJeXg/iIM7TGTfTIpYVVQT24AmdefdCEt4ReMCDul+Jk5QmrZ83rmTifFAnDMbRtwbst6GkpF7oLHmzoMigGM4LhzqB+ZIwYlhAxB2EzmLr+RakLg6h/looTcZqokIvG1hCPNCmaoyWXGb852qzA+bdu0D6nLuP2rq99d+aIxtPLhB5dPqUuJPxIMCZrdIgoQjYoWw8iY8mX8LIL0LtMzd3UkwvdAK0MQ4rAi+VcE+rRq8q54Lg/mzEr83m5+hlyyQ0hK20xlvq1NoJyfLRKyEXpyNO2bCPTRxq3UM990TIrMnvZzZowkHP0eycizq96dl4Ev+3wn8U6r7XkonfC6eeaVPvMVZ2+a9QizJGy2LgJBxM34gzzEPQqCbfFTg0c7SUZahz9ZLRPaEPje02qu+9VQH/b0m1f61Lz1OUzLkTb2M3JxKJHZ3fxybZp+oxN76WiP1A8LNn6cUFklspOfmUWJyz1s6y+T5rF4MvCdgGw9H5EXshN7ZNo3uV+Na/tYDKUelzcInIyfSVsy8RT8WvVcD/dJjY+tOH2m+UrdaofJANf42ReLOnbPD2vWxHAsz3fZNPMfpk4XAqSL5ReEvgnv/kTOwpd5ubXXCS3Dl9kFzP48yLRc7GSyd1wBLKCvK7QIfY+SW0UzD7P7fXuZrMfoqoz0w+Ljd6zR9PFGr79IrdNPHo9IMGIZtcGXOco14BuQnydc/lVz81ykRwko2zfCgQUwkb90BLowwHgXPtnTObvZqYEUrVsIB/phj6QpWvJdOvEnaQjCPP3atFPmpmoVJqdOe6+cz8WOi8UfJGoA+8UpFAfCVeHIiBKm/lGnuO+/ys4MNMAr6WUmswfS+Or63Nef05ZtJfMa8vyPH9KLIRdlaNOsVVrrFJtv9C6JzvILnO2UvOzodIXYja2eTKUhW+0DjmVBpRhAfP3wk/EH5jwci/FxjoIYg+vxU4GPN8ScjC14WfCcwC/VJYC+8KfCD+e+GHAnMnoRf+xeRRfCushK8I1EhCFiB61PP1kv+TkIRvCcSTW/sy2zeEPwh/IUDfEZLwa+EojEIvEMMHEr2iMzvnYVbqTRE++jMbc9OfDx29mRWMArP9Z4F6Wfh3wj8J9TwSb+LqftAh+jI3uZyDudfCV4RfCNSlLzq9s4Afwub7c052/Ruzb8T/RPiyQA18WfiZQF9q07sz+X8W/8R8vxSnz7sC1/+bwl5ghikaZTwK1GMO9kAN+nxLYFZm+RuBmQGE/3sCPvY1Cp1A7yTgh2+ELwnMU4ReoOZK4Lz4yUEHvxA4N73rbv9eMr1+KYyGTpze/1qoM+0lbwXqJ6HWoz79sf+5wPX6iQBl4X8Sfib8/oXpc/+k5k74S+HnwtFFYK/nox8zdsY53zliNuqNTSD6RmAH7J6Yfy9wD9MnCfT9M+GvBebgfFvhLYG67LIXiMfH+ZKQBYj8g0DtHwvfETqhF94T/lGY28lvLf534txj7K8TOD+9NsaTeBF6AZlZqDkK0G+EOr+ZZlknD+eiBmDuHwinvqeoX4StwP3Ctaf/QYDg1PDfNZ30c304JzMQx+fsu8JPBOwQsyF/T6A21xDiGrIfdjtF7HUUOqHmUWdvwTvxLwnUAPUc5n7JsG8F7p+/srhs3p+JFwGdGtzX7OWrwrcFvr9HwRM7K87AnBuB6893xiBw1lHg7J1Qr0uR3AsrgX7UGgXiyMHOGdFrbp0b+3fMvjd+6l4lhvhkoB+2OfqFHN8S1gKfJfaAjbx3hSTU6/tDycwHcX76EFdn701P4gdhI1B35YDtSwK1i9AL1CDm98JeIIZ7letBbP3+/4nkozAKvdDes0m2bwjM9huBWt8TqE2/tfADAeqEJNTexNAXHXsRKn1LAvUOzjYlUr8IWZj7TDDP/yd8U/iFgM6Z/l+BfM7LdfhbYSdwll+ajzO0z7TifO9I5l6nzr8TuI8vonRR9P0HM0++/zavdOCmCrq/DRSV5sPmiRv23Ifr/iZ6/pWzjtDu9C6nKlYvLShC735B3H2FMOMwUZyZthP2a5vY++baRaPea7mBpFMNEyfrZXuIe3Wi9Rthmtr5szj42xdOOV4Yf9/hzFPuu4mrf82H4AOO/exaHZuJuc6t7dkd6hEHXt/T/vKCMxXF0P+xiM/s1L1zuMeBkmp3Vj+Ll3vsFaVfnw08xr36+mzvdidJSpv6frhdtQfO+sID93vO7bjQ/4tQH0afPecL/4QvBA9WfuzMj5L5cTf7/q7wQ4EfNwddtoGk8N8L42VpJ6OLvN8QfnYy6o9OfkWwFch7aOKe4T7KAr+2SQK/3uDXFT8R2M21iQdxJ/DT+reE+3xZvPbsUe/xNvAY9+rjnfZpdO41BriP74GnccKYIjbwSBuoL8uP1D7aXnEDXMt8xXq3KcWLVbpN4i1z6BcUG7jNBh76Xr3NjM89Z6MDpOd+iJg/NhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHXawNZxxmE/vU61pM7Td3z/gEm4w/Scz13wvYB+s214A8ovz/nvKK9Uy32+rHQT9TFX4RBSBP+MMUGYgOxgdhAbODJb4D/IqWlXgbwphIvGrz03DdlNSj33UT1R4Ez0Q88BHUTTbAdJ+z3YepVtMwUZhfDjC/MsYHYQGwgNvCEN3Dp/8j3CR/lTqM91IvKnYZ8hGRe3tjN60L8VSS8OBXDQ5xr6gV4UOOH2uuoXl+bOWgn+3bGF+bYQGwgNhAbeMIbiBe4Fxene8LX6DFHi4f73bbPS9pD/Kr01JSjnFMvi7xYHoTjqeTwxQZiA7GB2MDT3MAXn+ZYJ6fipWIUeCglobfoLI5vL9SHEv7RbGKTRE6t2UsehaGJzNLptxYOAj0qbSQkgZ74e5Orv+XEkEM8KMIodAZq74RsutjLv5h3iyLCTw1+osRffIz9PeG7zo4MdcZhSehNpwY/meEvoc5moya1jgJ+7MjwYhD7HHWyAGLJH43zF0f/XBiETsCfhIOwF+aI3sz2oVCETtgI5AwCtBLoRU3kg4B/ipKMnTl68dGAjXzqJOP0/aYAdYbqx7YW6EO/Sp2ElSnUw58E8rD35hvER2En+D641wZkcg5CQRH1QhbIg45CJ1Af+RSN5qQ+NSttJPRODzE2EBuIDcQGYgP3toFBlbOrzkNp73QeSvyB7WS2lfi5BxyhvcHSXjLs1E9mgY8mw7LQ9h+cvxWZZxTgUBJ+ZTKsCL3TOQ82T+jYk9A7B39tT9fY0QcXg947nRxqVRokdE4vkrPTT4m9nEMTgA3aGqr7IGFdFfEsFKcjMhv2Sr0EUKlI8DWomZx/SqSmp6n9tjFFCTuX1EkenM7ZvN5LB5XaelP2JOPe5SCiZ2fjfH4OZHovoXaXnZJWSxIjJjYQG4gNxAae5gbefppjTU61ljULxXl5qCUBH3QU3hJGp/N3at6FeNDVevB3XLFe8t7pyO87vRW3MhSBOSv9tA1yuo+r5vckFGEUeheL2NrxDy4G/8bpiL7HKD01/qXqoEBqrywBGRt0EEaTYfTMTr9UJDcJ1K20l9A5fYnoz34q3seNCkwuuJc8OL1IBpdQr2B/FnJ3wtYVYQbOWAl95fRT4q/lXFtAEicXTFGNm/KFLTYQG4gNxAaeyAa++ETmWDIGD6txIpAHURYOE777NvGwYy76V/rwRFPiivOPkncn4udccw9f6nniZbN9IO/nit7RTu8ibIRBYC/YoCJ0Qi8cBHzgtsSZjkJ2BUbJ4KGJf0HgTJXKLQZIyhkm8nhZvwaxq7rvTnI/UzTL7s8yExbm2EBsIDYQG3jsDTynFzh25X/6VXfHg+maDx1eDpbWGxXLw7HUYRrZmW/EsTXcs/7Pqs9Ziuvj5UvaL9nLoIIfWD/2UmknAb03w9Z4Eh9NvoRxJl6cyiVJ9xjLbpbM0ilumJkjT9T4bCb2UjP7YsaNMJxITvKVE/5wxQZiA7GB2MAT2cBz+hUqDxZ+PZrc7laSAQ+oa1G+oNCg2DZ+eyKf+K7x+/yx8fHAvQsNSs5Nge6WBdOCvL1iviwwN3Kl70sYnF7FbsJWTfzaz5PfBdd76l44Ve9Eq5eutCSoifmp9NzY5q7baqb+Tva2d7vDmdRF5qOi+JcfeowTGVvZOvPD5+b3qUlK9oaQYwOxgdhAbODhNvCcXuDYCg+WXkgGZGw8oLDx5894UPGrp5XwbQGCo8/RXg7qJIFaEDL/pWD9NST51IdqvZ1k4rdmz+IHk6dYkbEXBoF6Waj9JN689KwbH/1rv9ofnkiw2HpOb8fXC0moeZ3kOt+GAFHdFX2ROTMytBO2QhJWL0xn/zlMxP5UNj/DXjo9jkIS6PuOUM8h8WZHGUHUCQeBuJq3kbwTkgGdulO0krH2hycLoiY+dJDN/n2z04u52Al+wIzYmAXaCsTVa9RJHm88L+gTMWbLAvPRz18vdOyjQC2I+KPQO732TLKtBWaq+7CwWXaQhxfr3UQEtiIwA3GDsLW4ZHZTX2HE9FOOsMUGYgOxgdhAbGBuAzzAcuNMTse/EqoNGZwiYrOLQ6+E7GtU3fuzxbxMOiFQay6e2vjgNa7G+jPgh/w5q26ulyxJyo0RG1TPVXVsa/PBsGennxP9nnwsNalTz1B7JGdD9uRz8GXBz0bslM3XQG5nSk2Ar4EMyPFxyF5v56izKuxzlF2ur9vOhe5ja6FkQs31vauvxk5xYtp5a1wxYSNe59mbjZxtDQweG4gNxAZiA7GB2EBsIDbwNDYwaAxe3DobZydeX/Z4ecvCRhjMHyw2EBuIDcQGYgOxgdhAbOCRN1Bf3vaaoxPQKw0S/Audc4UYG4gNxAZiA7GB2EBsIDbw2Bvgp2wtDWaA59YZemwgNhAbiA083ga+8Hito3NsIDbwRDbAr0wPwu+beY7SR8HzJzJyjBEbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbiA3EBmIDsYHYQGwgNhAbuNnAW89oD0mz1v8Z69zY/E9Ty5zzjJ3/+o4/6/MmUNIh2SVn/lt34NdhB6/z2d6EezPOGBuIDcQGYgOv6Qb4w9a75mzJbP3CMxPf0l4GcN+U7rvBwvq8rP3BxVY9L8x/ymHt2ZiVs26uOHS6Yq0oFRuIDcQGYgOxgXvdwFrV+clNutcup4sXufuZkGHG3pq71iA9GSZcVzVtr1rtbsX8CxyV8t3KPansqbPxYnct6q5VKOrEBmIDsYHYQGzg0g28fUHCzmJ/Kj5ckHffof6hvF/QLCsmTcSNsoH7JGbN99ngjrXLHfOfcjpnO15pQK7j5kq1okxsIDYQG4gNxAYu3sAXF2Z0iuPhd7D4sjDvIcKymtQXN/8CVx+wPGz/VPixgO1Hwih8IHwmDEIW3nN6ksxPGqEPBf+wpkc2XxJHP5oOw7c2Hd9oMnZ6Mw+9IWpXqj2SDEU4OJ8X8dfZeJnO5hzFi8mV0Yu68KMwNH6vdlLeEag5Ogd2iHxmGoVOIBaq8dVWd2ruz7EsC/OA0XgSp3a9BshrsyFD2WzMsRfgnvBTE6o51d9JYN5PhFKN4lsnD5KpmQXmIBadmkkowiisBa4jtqnrKPPniBziqcVsoFL1OdOD/Crf9ws5NhAbiA3EBl7TDRSdKwtTD5skOw+mh6KiRh8LPDzBpxONe9n8THsXgw+0tJGhOCP5fxA6Z+PBuxWS2XrxnfNTA38l4pPTs+Ti9CqS0zv7XnKaiKumlQRm8zmDdN97LR1bpSShOB2RGp6YNzvDTrKvyVz0hpDxV8Lu/c71Uty4nCR5NM/aOP7D/8/eG8PIllxpeuqeBjRrMSkI0KzVQWtnAQFMWjuyOmjtCILAHECARlbf8QZymPTG69uW1mNShrBe3/bGY9IQMHLEaG8ECGDSEmXxtqVZi0lrZy3q/+pFNONF35t5syqzKuu9/wB/x4kTJ06c+DJf5u2q16TE2Eml/lZ+zDkrjaPEWIx1VKyT094tKdb/MeWbB3dC1DpUa+QNUsyxtcYx+wzEUzU/5bK3rxLYF/M8NGttbrXNrgmYgAmYgAn8kcCHC2F8orxOOkrtl8yQ1xaWukoaPx35POsXExXp8dMqPkzktCHuVluZj1WQ2EoqsSSfs2qr56x3zXo7pR4Portq4dy+0ltf7dnKLz8ZIszaUK2P8g9SV8Vat9Qlzj02Ut3XSnOEEWe9GPlb6VjFWrdTIOXgmHOjxkOOsZf6e2mQqFcsZIecUVrnOXHydnnOMFR+cVMVY2+USoyaB2mTc4iHap21j/PapcMq1yr7kpyYJ6xxJiPGOchmAiZgAiZgAicJLP0VKkX20pjV5zlfNlF6SaOvYnwRBqmXiDMmiV+fPtbYv9Q4k/xOOkr0w3jK1jmPsRh7wqlNE2vs+bVEnVH6kbRr8sjZSEMTn5qSNzYLsZqn7BPDD3nM4cmhvddKWYcmc5zYyT2CtM357CsW5YzVfInL3eiFvcVGOXXdaulJbtJu7vhjidcnSKOEEU/S77K/1/iU9+pDUZsJmIAJmMC7T+CSBzi+8IqVLz++gF7aUtXAWv6Y50Ej8yjxxViv5ZS3vkxL7Cljp81oI8GIM09ZqBbTqcSFa6uFeUvS6P+c9UropIO0JH+nPPYUNtuF+zgjSozYkrNy6uTAfpSq1dqf3HQiGLQ2zqyvFefeXc6JVd5KPgx6iXgnlVy5NhMwARMwAROYJrD0V6hfTWw/TMReOtSpgVHiSxGjR74Qeyk8RN7+8i+xvPTkoVcFdMyVSv04U5n1JP1eKrkldT2zZy7M/pVUXhd+tTxVcz9XoImTN9VDXZOcTdaSuiV30J6ttGTPSnlf5HwNb1nULElTfba59Zxzp/ZMxU7VKWvhRNJOa2hscqLmnbSRjhI94QfJZgImYAImYAInCSx9gOMLKOZKK40o5TlfPNvsP8dQzq/Piprw98hCDq418mVYjPiYJ0ljWSNejLr816r1HJ94HWvn9Z5RiSEnl7HuN+V1YlEac26nsZdK7br3nDI5sK/YTs5WGnMAnzql5jqvDXm9xMtIGD/k9VHjLiuHvnnYKPOjnL0USuDMmLR+kHitimK1h/PhWffEGfUD7lpzYiHnjRp7aSsVK36oYlF+qTvK76Vdtd7JJ45F6dR7IWmdPqhH7ijNWemVdfJRyONBY5d9DQ81E47NBEzABEzABE4R+ODUYrO20TxIfCHt80hKlMYs5reyoMLdmeKj1gcpSvhB4guTL0rmxaKcIBFHzBE2SknqHmZvbNAQpZDnKY/EivVyOKuTqAknxq2Usq/hoUaURilJxYKcdZ6wb/zj0qT3B0W/K0UpSHtplGqjnyjRCz45WJC67DMMzTxpjjB6QtRIeXxYyLbReJDGKjbn9jmXWhh1O6mXiEWp2CBnzJOgMVZzzttKQxWjVpCwJLGODVKUwsPszVrKPnsQZxMrPcS8zjBIXTMfNQ/SRjpIqVpv3VXOGzXi76VOYo6w0kNZz2EPJmACJmACJmAC7xoBHuDuwbqFTUTl9RO5xIeJuEMmYAImYAImYAIzBD6ciTt83wT4SQ1WxufuNunAKPGTo3Hh4Yec3/bcKc5PpWwmYAImYAImYAILCXywMM9p90MgqJWuameQPz5zezy4lYe3dMHZPLxtpKMU8sj+UbKZgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAmYgAnclMAHN63u4iZgAibwfAQ6HXXMWmscsx817rKvwWYCJmACJmACJmACJnAPBFZqYls18ofK7++hQfdgAiZgAiZgAiZgAibwNoFNNY3yUzWv18zNBEzABEzABEzABEzgDgn06gnZTMAETMAETMAETMAEXgmBpD7jK+nVbZqACZiACZiACZiACYhA/fffDMQETMAE3kkCH76Tt/KlTMAECgH+cv9P3yMcUXf96j26r69qAibwnhLw/4zIe/rC+9rvPIGgG/KX94/SF9L78Ged+/LAinHvffY9mIAJmIAJmIAJmMCrI+BfKb66l8wNm4AJmMBpAv4V6mk+XjUBEzABEzABEzCBuyPgB7i7e0nckAmYgAmYgAmYgAmcJuAHuNN8vGoCJmACJmACJmACd0fgo7vryA2ZgAm8FgKdGv34Ss1+rTrDmVpB65+eyblk+WdK5j92sJlAS2CtwCj5/dGS8fxuCPgB7m5eCjdiAq+OAP/FZ5+75mHokv/qky9IFCUeAvmiZP+pL0zW/ibnj9nXsMiCstBG+n7ewX+Z22ffgwkUArwvO2klbaVT70lTMwETMAETMIGbEbjlf4XKQxf1fyeFR96g075R4svynEUlcB7qzyXPrAfF6fu3M+sOv78EeGjrq+t37y8K39wETMAETOClCdzyAY4vvFHijF8+4aLU4aFqifVKKg9xccmGmZxOcWT7NgFeD/QUC0/YfOr8U2tPONJbTcAETMAE3kUCa13qqV9oz82l14E7KeWR+S0squhTfypGX0HaLGww5TP5KdpTXpftzHnU/OQChZk6bZi6n0qftQt3Nh+u0E96Yg3eu1MWFUS3trnzb32u65uACZiACVyJwFp1+DAfpKc8LFypnbss06ura/xUbOnlghKP+cyfL910Yd6Yz0gaT4m8KC0x3j8bCVb3ar0aC1doLj2xBj3QS2tRAXRL26r4Pb9Gt7y7a5uACZjAO0GAL9w+3wSfD3bbNIGkMF96T/2p2HT1b0fLgxBn3uJ16fJ9wrePfivSn1mfWr7XhwPuOkw1/IhYesSedgu9hCYYNUe3Mv6cD9K9vka3urfrmoAJmIAJvKcEgu59lPjiu9VPxVq0u3weZ67bxSvMR9UYTtSJWkOX2r0+HPAgfC2O6VIoE/lRsb6JE0O3MhgEideI0WYCJmACJvCKCUT13kn827ltnsBGS3zx3eqnYlMnH/J5v9J47deny7XD1MGK9TPxc+GpBzjYRYmxNe5FvJNqv81jvpY+mRDxczaeSAha20qMSyzNJEXFL6lzaOqwH93C6tq8Rrc65xa9u6YJmIAJmEBDYKd5+fLr5adHEJr7Um2/aFePqH1vW+BVHuIKt1v2yBnHfOYXNzhoVM1hom5UDD3G6gc4XvO9VFgxDlXRIP8gkYc/5rVNHuthp8lQqc8+47ZOnPHTTLxTnBoY/m8l+jllaWKR/so++gkTOW2orROVgG5hNSM/wN2CsGteqbtXEgAAIABJREFUjcCHV6vkQibwbhLgC+coHarrFX+t2I8XXjsq74cLFBbWu+c0vgR/nRvkgWp142Z5PTgT67Ly9CpDryqfSqGpFjVPTaxMN3K6mbU2TO/UKe8rRtTnxE7jXjpKYxb1idVGbJA6if2M5BDrpZ1ULMghf93Equk3bpTXSX2OJI0fSMc8XzrEnFj2hQtq1H0uPe/SvE4bhmrTV/LjpUWcbwLPReCj5zrI55jAKySwUs88oH2v6p0vkiHPDxr3Ev8vBOdsdy5hwfr/rJz/ZkHetVK41//9yGKd9iUJXvjXuP+pVgYtbqQfST+VmF/LqNVndblo1Jiy3w7cdZC4e5RSm9DMyembGO+tEjvKD9X6Sv7Y5DPdNzlMo9SyD4r1UiexZyNhQeKs1noFhioY5acmiRqjdGji9ZT1TuKnzoPUS/V5zNGUraaCV4xRnzuEqmbtl/A65+yveLZLmcCjCPgB7lHYvOk9IcCH9dfSWN03yu/ynPVUrZ1yyf3OqYS8xk+ujjN5/5vi6DXYQU3upI00PFPDnc75nfSTG5zXqyY/TWQcpZh9DW8ZcYz7BylKicAJCzNrJT5oPUk8nBLDp/6ckVPeQ/itDQp0ObhqFts5yzxwlXzmUUo4lXXyt02snY4KrCXeE38l/UDqchLxkP2pYZwKNjFqcO8luW25TgH6qi1ogmrrNNk3MU9N4EUI+FeoL4Ldh74iAmPVa/mC4EsCi1KSiPPhz3zOVlr4YIHIexcMJp0El8Lr1vfa6YDPpeEGB1GTh/leilKSpmyr4JAXYLDk7kl55NbGnDiGHyUe7r+UOOOU9VpMOYG9de2g+Uoaq/XsnnwoLPnkRinlkVq9xBmdNGcbLeykMY/MQ07G7yVq4U/ZOBWsYlv57O/zeCb9reWo2WFiw6hYqOKc0UlR4iybCZiACZjAnRLgQzrl3oo/VL3u5cesTR6r5Zu565tVfvPF9NT6sDpIT61zyTV7JQ+XbHhEbqc9/MX23Ym9R631WaPGOQbUKVZ4MWLtnBp76bOsqb+Pl7e+9XBGjH1dWdS4lZLUS0P2NXxjh3qSfWKlt05+6R0fo79d9ushVRPOJa8Y81jN9/JDNa/dsYmzDxXD7/OEkflSC0r8+Uxyr/ivqjUYpCY3aP5bidFmAs9K4KNnPc2HmcDrInBUu520yW0zT9UVQl7nywc9l9HPYeKwoNjH0lpa5Rx6/moidy7E3pj3zuWcinPuL6XdE2qcqj+11ikIE3q/pQ0q3kv7mUOC4oecA4cuz+t0cojzk8JeGqRRihJ3CNX8KB8jRt0P8vx7Gtm/ldpeyO1zHsNQ+bj01UtJYj9jbZyzlhiLbeQgjPOOUpd9YlFKOCdsp7VOojb7k1SfETQfpdbIH9pgM+8173JsdSa3Xqb2RuInm1FK1WKX/V9o7LOixpTjZeAuX0qMNhMwARMwgTslwId0yL3VXyyHHIvP1HffnMO5O4k4Pabs7/OcvxfGT2+WWMx7l+RO5QwKbqcWbhTjdUjS6kb1LylLDzDHemmT/acM1EwTBcJMfCL1rVCvWcyR0mudwBo5lxh16DM2m9LCImvlDTm3rbFTPDR1yCl5nHuU+iz8IBHnPc94LaOXKIWsa9V1HRN4FIEPH7XLm0zg3ScQdMX61ydbzQdpzFePGlP2Dxo7ifG5baMD6Q31Ev+hBF80+KwFqf4pgqY3sz5X5vzH2uqCjUG5e4m7Hy/YV6dect65I+jhIPErTkZ6e6pRE7V9RsXSI4oP2rOW+qy2BDWD1J7X5tXzURNqXrKn3l/u2Cl4qBZKPerPGefupV4apDGLmt+TGK9l1Co/4R6vVdR1TMAETMAErkuAL49eilKXdd0THl+Nvorhly86Rv5+El9qrSUF+Ls65ywqoT+XNLHeKXaYiF8S2io5LNzAXXnAjgvzp9KosZtauLMYfcJmI0Wpy7pVm+W8p9ZPTyzQaz+9tBYVQBhjn31GGBUb5LA/5vGPK/ZMwARMwARM4AUI9DNnRsV5gJsy9syt1flRE3IvsbWSD9LUl+3SOlslpqXJObe7IL9NDQrwAEjvtm8T4LUM3w5fFKkfpi7aqGRel7n3E32V3sjpc/5OYzH2D1KUOilINhMwARMwARN4UQL9zOnE0yPW6i1RE+osNb4oR4nxUuPLl183/lLi4bJbWGBQXv1lvXDbQxp9/lTi7wWOl2x07t0SCOqsff9tFUND7pr3ms0E3ikCH71Tt/FlTOD9JsCXWJpBEBTf5zXyfiR9KY0z+UvCfCl+IfHwxUPRJRaUjIr9Xk7prwp/y+VLmYe+JPHgd4nFJvnzSzY7924JjBOd8R7vpEHaSGkixyETeNUE/AD3ql8+N28CbxGImvUTTIJiiIcf7CAN0lMfYKjHgxf6gMIX2NfKRcWSnOOZ/UHrfBl/JV16HqXZV9u+mXv67hAo7/VOV1pJ595b787NfRMTMAETMIG7JdBPdMaDzTgR58srSetqrcQm0t/6S+FT646ZgAmYgAncAQH/BO4OXgS3YAJXIMAD3NDUiZoTR/VPIIgnKUg82PFAN0g2EzABEzCBV0LAD3Cv5IVymyZwgkDQGn+nbSt9JvFAxq8YDzmm4S2Lmo0SD2/kMNpMwARMwAReEQE/wL2iF8utmsAMgY3iP5O+J/FAhsaZXMJRGiQe3PbSKL0vFnTRY9b7cmff0wRM4B0k4Ae4d/BF9ZXeOwK7C268Ui4PMOxJeV+s/AtKXZTKwyLqpFEKefdvNf5EoqdLrdcGtNS4814KEufh20zABEzgVRLwA9yrfNnctAk8mgAPUeXB5SB/W80fXbTaGOWjYoOclbSR+jzuNHI2Nkg/l36Y50uHoETustQ6JY5Syhs4f790s/NMwARM4N4I+AHu3l4R92MCtyWQVB5hPLw9xqI28b/9VuzXco5Sl8f+j0vfeOWBbaVI8VlkT11rYuu3QlGR0jv7h29lfDsQFEpVmH5tJmACJvBqCfgB7tW+dG78PSZwy58c8XB16uGGBycewtZSyeUBDgtSn/25gb21MS/7qRmkfZPTTpMCnbST6IEa7J0zcqKU5hIcNwETMIHXRsAPcK/tFXO/JvD2T7CuzeOogjzwTBkPSoNETp9V8qKc0MRYG6QRRxallP0ydHL6PKFulPZl8cTIA1vpk33pRC5L7Tp3sZmACZjAqyXgB7hX+9K5cRN4dgI8KGFh4mQeptBuYq2EOjl9tR7lU3OfY8yTFCQesKg3ZayPeWGjMUnrqcQcow75UUon8rxkAiZgAq+GgB/gXs1L5UZN4G4IlIemuiEexHjoCtI40Snx70ufSPx/mX4t8WA1SMWinCRRB5/1KeMsRE7KPuMpG7SIOP8HUncq2WsmYAImcO8EPrj3Bt2fCZjA3REI6qg8RLXNbRXgAaz8hwlfyh+ldY6Tn/jHhO0VQ9RmtJmACZiACZiACZiACdwxgaDedrk/xs0d9+rWTMAETMAETMAETMAEMoFVHoOJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmMCzEVg3J62e7WQfZAImYAImYAImYAIvQODDFzjzGkdGFdlLO4kHtm1WyKMG2wICcDtKfZW7kR8X7L23lKm73FuP9MP79ccLGuM1GKV+Qe5Lp8Cee73rNuiC/HmJV74o9VLWlUs/qtzUnyV65LPBZgImYAKPJtBp50FqvzCCYqPUS/duazWI7sF2aqKvGmGOXqNxD3SJdZckXyGX1/0ohabWVB+9ctA9W1Bzo/TS7+fumSAlnRNvcBY1qX0vRi991QyfCcMVm+P98tLvmadc5zX3/pR7e+8dEfhoYS+98r4j8cVT7Gs5QzV/DjfokC+k70l1L5w9Sr0UmNy50WPb/0u13PaxfalGXujcqHOHZzz7oLNWzXnMwzP2cM2jxjvoHXavld81X4tb1rr25wKvV/vZc8v+r107qiB/lm0m8GIElv4KlS+Y30sfZP1A40v84et17lfSKE1ZmgreYezaH4Z3eMVX0VKnLsMddOr3w5sXgc+Zx1j/mE3v0J7HcntJBK/5PU/vr5H5S77ePvsGBJb+BG7Q2Yd8Pm/cTtrfoJ9zJTdK2J1IGrU2VOv8QTtm0Td3QEHqpU8lfpoXc4wc4pyDz9hJoxSkXmLPTyTqkkN8K2FR6is/5Dl7ePgt87X8TopSytLwECtGzk4aq1jtdpr0UsoKGulnn+caHixInXSQWMeGPLZDVKDPQfxi5X7UoK8hj53Gj6UvcyxopOej1Euj1FqnQC8NeYx51PDN68B5/D2xH0pBom/GXqJ2sSiHfoiRg+p15p00SlFK0l7COqnPfjuybysdpCBRc5AuMfaXXtby+zxn/EzibknaSRtplPo8DhprY30lUecgDdXiVv4xz4NG6jFnhCH/srWRPpE4c8o6BXtpyGPMo4Zvfl1IDYza6xwn1kmchZ9yvM/+QSNW1oY8Z+ikVSVyS71jlVfcKCdIYx7XGrfSToo53udx0IhRjzOoF6VBOkhYJ7GXESMvSsRKDvGOf8hKHcba1ppEiT1bqc8+OV2eE2cfc3xymeOP2We+xDZKWud9jNSK0k5ijfcWrzlr2zxq+OYe+Kz10pFJNvausojjF4ty+jzBL0bsIAWJ/F0etxrLe481tJaIk9fneacxSilLw6SRw/5Rwh8kzsWIl7WQfc4paxs55HLuUdrntagRscY4ZL/T2EulBvuoQYxc4mjMMcZBspnAqyDAm5c3dTHm8Zk6/4PO6ReeVf6g1en8Aax7p96mSmC9ro9fz4Pm7GEs1ssZmnmq5rjsqY312MSY76tYkJ+anHbaN3u42yitcyLzJNV37jTv8zoDfj2PmicWsm01Ds18l+fU5W51ffLPWa8EVCzKSdUcl7rEi+Efmnmq5rh7qa9i9Fn3w/51tU5uqubFJS9U8XY+seWtUKfZUEU2zTxpHqv1Xj5qjRhnr/JC0DhWSZ38oZq355BLDOvzODewXudEzVOVXPuEh2YtVvNOfpJK34zHan0tn3sV28spfVbht1xyattWk14+ao0zQg5S/1dNQtJ8qGJdM+eMvlrHHaVYxereV4ofm/xe812OdRqDRN5v85iXvnnILfOpkb1jtUBdeqwtabLJgT6Pa42/rJLYN1Rz8us5Swepr3Ki/FTNWavP7jQnVow/v8SKpYl5rNZPufQScgK9wq5YqnzcLs9XGkeJEQtS2YdPzWIlt8wHOX21zpn7as5avV4t2TWB5yPw4YVHBeUfs8pW3vzlD8mF5S5O59e4S61XYmqS+UO4nYiVEHcbm/X6bmWtztkp/1MpNPsunfbakKpNo/xPFhQ5VDn0X9+x05x14sXq9So86261MlSrqZpT90upq9brs6rwo1zOKoa/kmIO9BqH7JeBu9bGXesYvVHjlMWcM1ZJ1OlObWrWtpqzpxj+5oL9dSo9I2yUPs4+Qye15/BeLDbKKXv7Kv4YN2jTZ1J5T56qN+YDytmM/B3aYrAoa8QO0rpan3O/0ALnr6RhLqmKk1PO2cufOmOs8vFDNe/lp2qOOzZz6tI/xln1PUtq6WFQYJQ66WupxMmrfeZTFhQcqwX2TN2p1OpzLnt+Ue3byw/VnLyhmuOe64f3Qqr2jPJjNcc9VPMkPzTrS6eDEks/be/Efy6V9wXrWCeNUtmH/zkLsl4qeczJQYGJbMxjGVhbNTFPTeDFCXx0YQe98lFt7bxZvuo0qdr6TMVO64PEB+lhIvfc/rHZcy6fP9xYkMbsP2YoHxCx2vzDRxSih03eFzSW/kop5rDhXocSPDHWDwyktXsGxdBO6qS9dCsbVThKSaJ/5qeMXvlVTsldnUrOa1HjUWIsluSM1fyc+30lBClWiZtzmx6xzpcWvOtzHvOeWXI0Z+ykv5F4//DXCIYlGydy2FfzwD/Hp1MO538p8Z6kj0E6ZbDZSgfpeCpxYm2lGPdME2t1iPVOGrPOpD8sU5ueLjX2hLyf+6ylYUERctm7lUYpNHt4v15i9I8xxmoj9WvjzGvYXkWoTb1jU7DTvJfK+2Inn/dmkNrzB8Uw1rCYRwbqt7WrZbsmcH8ELn2A2+gK3Qtegz+cv5RW0tQftqD4WPW3ll//IWZfPa9Sn+zW515SrPTIfegvXbJ5Ipca1MJKzTqNdWwph3M/9Uy5XsxjOTtPrzrQe+mbsdxl7pC9FtAuJ2zzGDSOzaZ1nieNn0qMj7Vfa+P4yBr0wd2W2FPOWVK/zgmabHIgahyy6pyl/pj39hrxuzxqmLWY80gIUsoaCVTWyR+kICUJnkepttVErEn5Zk/Qwtgu5jl1+Dz6K6l9zU7tYzvrlxr32EmdVPy0oMhGOVsp5lxGYhh3OPdnPKd+M3B2sdQuPmK+1p7DzL6gOGeQU59LOr0T32YFjfscIzdKUzYqiNLU4oUxzj9cuMfpJnAVAh9eUIU3Kv9G2hrxVRu80Typ7s+kL2bqbxQnByMvZr8MzPkDfk3rVOwracxF2z/M9HTKYl7caYRlbef2ktuy7xSjFjZIMftloCZslhq1YpPc9kXOZ9JhadEmL8zsW1dx/JWUcmyvMTb7QjPnJ1T01lrpP1UL1D5KB+kDKTRrXZ7TR9lfpbzlDhM5Zf/cnhKP5xKq9aecc+6Y0CR01TzJH88VOLEe81qvcZBgfs7ILTbKGSRerznrtDBWObFK5DVcYlOfIbxPilHna6n0X6+deo8M2hP+WObBq/c2S29N2beTBimdSqzWuia3vj8+9WJT61w/sKnrsP3UnZvyb03bs+tFao7SMQfrXM6vzyRvn/MGjUGq7xFzbKexk2pjHprY3HSscuv6bT9z+x03gasQ+JMLqvy5cjvp82bPv9ecN/E/XlDrKan/oM3/WtpK/1H6f6Uo8Qd5kP5ZwugnSvT2X0k/kQYpSUHigeMvJIwPYerxhR+k/5BHfhLDnH9DPUhYn+ecw/6/zirn/lOe/1bjdyWsk+iDXo9Zf6uR/jl7lH4j/ZlEzf9c+oFELmtzFrVAPnl/Kv2d9PfSPm8gTl3uxllr6b+VyKPfTuJXjPRGHyHPea3LnQ/y/1IiVvoa5XPPYpzxP0n/roqdckct0hO16R/9W4k7fCVhvQQv+mS9k9gzStg/SsRDnpc6/0bz/yRR+3sSvx7iLtyBWCftpVE6Sn8t8VrB+x8kzkvS30l/kP5Mgtm/lzDyv5A+z/Opgd7WWTD7ocTZMOulH0nc9dcSPYw5/v9o/BcSuVGq33/kte9Z+uUccum1PYf9GO9Dap6yUYvbnPcXGlH9mvSaw4Y+6J/X6TcS8U+kIJU/N/TJfCV9LbV908vfSz+VeolzOe//kqg/ZX+rIOdjvMZwSnnOHl6X/08qdx3ldzlG7p9J/1LiX0LpM0rco/QY5PNn4c9zjPvxOnIubKnxlxJ9/iuJGtyDdf5sfSCtc5z+uEuUeA04O0jUxOj3nyT2wjBmtX8GyG3t7xT4QuqzOJM/J9QjVt5b9WsON/jQL/0XXnIf7pgk7sZ7lT87MYvPTV6/IHEP4n8qFTbMWYMpf35GiXU4Rgm29LbJMeZwog96KOw4Y5SmjHt1Eq8tr8GfS9ztY4l99M2Z1IsSrP8+z0eNvcTetcRrBm9qjlI5P8rnM+IfpU6i/9IrY/u+GBXjdeDPK/dmHwZjXpvP89yDCdwNAd7I/CG4J4tqZnOmr2v3zYc5BgtqzxnrIS9GjW0ua2U9p30zkL/EeiWhJXc81++5806dwdrmXIGJ9ahYyP3j11Y4s45OGXfDglT8kk8slonG9nVgqV6vUr/prY7hX3LXtp+2Vpmf4ju3p44vPedcraiEINEPfjHmJTbFsEo96+6U0fYbFUsndoa8Rt7U+cTammwhP+A8weranNHWI1afPdXf3PHsIx/FuaQc3zXnEGbPeGZfWSb3VG+slXswFv9U+XrPqbyptaAgWmL0MpVb7hO1Pne3U/c4tXauL85sbdMGPDcBE7gfAuXB4h466tUEegkLOrR8WDHOfXg+trd74tzeYdsGPL+IwGEmey4+k/7ehdPMjefiM+kO35CAPxtuCNel3ybwoYEsJsADymc5u/zqZfHmGySuVZM+vi/hP7cFHbiRGLFjHp86wJl7YeVXGU+tec39sB6vWfA9rNXrzoMU8t1hupO2ee5hmgB84AQvLEjEhjdT//OFCfiz4YVfAB9vAnMEVloIeZGR+Uta+RCnh9p/zp4499pn3xvn5+T5vp0VdeGi9+3uT7kvf+bM7SkEvdcETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAEbkDggxvUdEkTMIH3m8Ba148ZQdJ4eKU4tur7KK2k3Su9A213uffhFd/BrZuACZiACZiACdyQAA9v+6p+7d/w2KuX5oEt5qp95V/9oBsX7FS/iNfGZgImYAImYAImYALfIpAUqR8U/vCtjPsPRLXIPYr1ctBTbfvUAo/YH6s9tf+IUt5iAiZwTwQ+vKdm3IsJmMCrJhBz94dXfYs3D2v8BO7atrp2wQX1UpVT+wu2OsUETOCeCXx0z825NxMwgVdFgJ8w7auOo/zfv6obvPnpIT9BbO9Rz5/rSjzwddIoBWmQjtIlVtdgb5CGSwo41wRM4D4J+AHuPl8Xd2UCr5HAj9T076TPcvNRY3rGiwSd9enC8z6fydsoXt+BtO9L/Uz+rcI8RO4k+uHBi/mQ58RSjmuYtZVW9lKpwdhLw+wOL5iACZiACZiACbxXBKJuy4NGbUmTbQ7wMDFI67cylk/Y/9i9y09582BUemYfZ9b32mi+u6RgldtfsG9Ubmjyy98npIdip/oZlFTfBZ8Yxr320irPPZiACbwyAv4J3Ct7wdyuCdwpgaC+DlVvPBh8IvGAgR0lYoxz1mlhmFhcK1bq1GdMpF4lVJ8RVXFfVR3lc49z1k8kRMXa+KDY2OR2msOpjX+t2Faq+yNvyuiRn0b21eJafspzagRpbn+1za4JmMA9EvAD3D2+Ku7JBF4fgaCW6weLjeZfSvUDAjnjiauxPmXU5YEkTi1WsSD/qb9CpcZY1dzKj9UcP1XzObefWCA2FW9TgwKpDWoOSzjUa3Emd604D3yjVCzK6fOEdbjaTMAEXikBP8C90hfObZvAnRFI6idWPbUPPuWBgTHkXHKuaaOKzf3dtqXnUANhnTRI48PsjUUN9N1JQdpL134QOqrmlK0UTM1C1HwndRJsB4l+xiwNDxbzOFbzlONBI7WpgzHHxjx6MAETuEMCfoC7wxfFLZnAKySQ1HOX+y4PFPWDSNRame/ljzmXePm7XR/L59euxdqfIFVLN3MHVd5Io7SWts1JxBB3WEnHZv0a06GpHzUP0iEX3+R1psRLP0l+6WeUj1ijz15KUrEoh3poL5FTbJDDa/G9KmbXBEzgzgj4fwfuzl4Qt2MCr5jAVr1vpF7iwaC2mGOdRh4W6vUPNJ/SWwWeaTLonGPukfvUttaEvoPEPUeJ3GsbNakf8zhqHKROClKSsCjRD+NUPx1JsjEr5TlDyLGtRs4bqzVq/aya2zUBEzABEzABE3hPCfCggfVSlHhIaI21OYtaOLU+t++acR52ulwwaQzS+oIDLsldUrZXUuFIP9QPeWN71rFZG3Ie+7D29Yg57sEETOBOCfgncHf6wrgtE3iHCKx0lyHfZ68xSOMF9+uUi9jXX7DvFqkpF+UePCQdLjjkktylZet+gjaNErxLnDqDtJVGJtnoHxsk1tKb6cM/Q+XbNQETMAETMAETMIGTBLqTq168hAAso7SRwiUblcsDoM0ETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETOBCAn9yYb7TXweBoDZ/LH1POryOls92GZXx6cSdVor989ndTlhKICiR906Uvlq66cZ5ner/SPpAGp/prK91znHirKDYrflsdMZfSEv+7D5HPxMYHDIBEzCB+yWwVmt/kHhAuHcLEw2WmKVOAAAgAElEQVTyJZAm4q85tG3uxB15ja5pr+H1nrtvmFu4MA6Da3OdaoHX73cTC1OvwV55/UTuLUIHFY1V4SR/V83X8m/Jh7v+qjrvnHvrfs6d73UTMIEXIPDhC5z5Wo7kQ/yvpOMraLib6PE19D3R9skQr0ltSRNeo2tad81iz1zrWr0/13sniU/7+gXFeCBprX3t2/Vrztv79yq+qw64ZS8rncP5MAgLL3XLfha24DQTMIHnJuAHuNPE96eX72KVD3w+7N9H44vumq9RVD14vlbrXlnjvH6p6fke70CP4zOx3eicrcSvcPFtJmACJjBJ4KPJ6H0HeVgJucX6CyAq9olU/t4OOUkapdaiAuRifFAOUpD4O1bYl1In8WsS/JGgjC/3mEfmA/84Y6f29NpbziAPcb9dUzPkeAnvsxM1/lRi32c59nlJymN9/kExVFunyVEKUjmX2McSdye+lgaJvDnr8vp3NMI0SGMeqfWVRC9oL1ELP0oY/pD9ethoMndurzX41XemTtkT5O9ysU5jfSfC62qdPbAcJVhyh0E6ZezBOHMvHavkmOMldJAz5gnnhuyzJ2WfodRkfZBYj9InEj1xDvOVlKRRoh49cz9G7EtpzP7UEBWkBjpkTeWVGHmbPCE/ZA0aj1JZx8f2eQwaP5VK79Qgh/Veql+/reb8/TLuwX15zySptpgna42DVM7r5ZdanFGMc2KetHuqtFmXvfRy6v3Qa52zsc/zWHoImg9S6TMvnxzIpW+47WYyuUuQyD1M5LBWNMpPVU4v/xasqiPsmoAJmMDbBDpNt1WIOSo2yEF8uDHWuSWnjKz9tgmyJ+bYSuNYzQkPEnGM/bvsnxoGLc7tIX6QeinkItSkdrGNnPqcODFPVX5xySPe5QBnjVLphXCSAo4sSin7DPi9RD7+Wpoz+uvyIr0zZ1/Zs5ef8hx/k3NLHtNO+nmOl2GQU2oQIz9VOZxRvjgJM2edEeMc9hQ7yOmlkAP4qFg7r5a+5W4VWedo0Ejt+lzOLtbLiXnSaRzKgsZeKnXoteyj1iiVmlv5h2o95HUNDxalmkWJT41Bwb5aSPJjk9hyHbVeeqGPIK1zDH9frRMfpGJRTpI6CZ9cLEhtz0mx+LD6tvWaso/aWCcN2WcIErU20irHk8atNLcnp31rYF+sotQgVlvd904LvVTOZc4ejNihWsvhyYHcLq+sNXJGmMikdqnPcpdzSyp1+JeRYns5m2oe5F+LVVXWrgmYgAlME+BDaeovPB8UD3lLr5H5UhuVGKrk+kORcJJitU7tdZ4Hjcdqbc49tydpY19t3sgnVmyUs6rmuEkqfcQ8b1Imv9DZRz7WSfvsl6HutVdw16zPTWsOQUntlzK1honNSbFyD5brfd3EnqgYe2qr9/RaaHuue2NvvR41J1asl4OWGHW2VWKSH/O81/hZtcYd0UriPcxYbJATpSDVvbJen0HOSLCy+u6st9yb9G+m9JKqxV7+0CS3r0Wbz55iezmxmuPSe1fFqFffuyy1PSctxGpfcXs5QxUnJ1VzXGoRL9bLGap5lJ+q+ZxLDrnF8FM1xy19b+TDs1io1kqsfh2r1G+5W0VqRgfN+yYraD42sah5zZEc9hajx1TNca/FqinrqQmYwHMR+Oi5DrrCOXxI/nqizlEx1sa8Vn9wTaS/FRo022ZFjfszGziHvE7iXH5deM6W7Kl7pm4x9n4s1bGytpFT76u2LXJjrstYrK03de5UcfLolf1B+moiaZyIRcXY10ntWRvF2n4mSrwVot4oMRbbyym9EWvPqVIvcrfKDlInjdkvBXZyOJcc7vCzPI8aV1LdQ6c5xjhK8WH2xsirc8dq7SkuPUWpy/XXGulrzsivLWhSx36k+a7Joe+NNFRxYk+x8RGbH7Nn6THcj3uvqw1RPp9TjMW495K7U2db7Rvlfyr1VYwziZ8y1qnVSZxb93du36l1r5mACdwRgdf0AHcK26paHE8lNmuD5geJD80gpWa9nnIGv+L7UhpO5D1mDx+y17KgQuPCYuSlKrf2F5Z4SOslGMKH/6oQf4kNSoL/bknywpxReanKrf2FJR7SgjSe2NBpLUrcldevzWVtnVXuR94pYz1VCbV/at/UWmxq1Tn0RU/0fpCYkz9n5NBbL/1WGiX239pCPuvW5zy2Plz2Ui/Bshg/3UrVvPar8Ftu0IxaqBgxeK8lzsJWeTw1kFP6SvKPUjy1wWsmYAKvj8CHr6jlUb3yQdZ+gBFLj7wHNdm7XbB/o5zfS0PO5dxitV+FH34Ccemeev9Bk6+lUAfzfN/EmK4mcifSHkJJis3iJfvrrUGTTvpFHun7nMEGbrucyNnFiCcpVDHcOqdZepjuJ3LCRGxqbxtj3yn7Qotb6dgk0Ttxej1IQ54TZ877IUi1sZakT5o4U9aW2NgkndrXK3eQ6Kc29kztC4oPUi/xLzCMtfG6hybGfN/ELp22NS/df+v8UQdspZjFeUmaYjgVI79YlNPyGhXjp3ndN1lvckI1n3J3CqYs1uv8c31M1XPMBEzgDgm8tgc4Piz7imOZj1UsXMh5UD51GFtbVYGj/FDN8fkyJqeOVykPX+712tyeNqf+1Sy9oWKdnL10yIFRY+kzymc+Z+SV3CHnbqrkLscIBankVimTLnn8penPsn7c7K3PLQXGJidqzpcVuUHa5XGtESNOr1O/ti59sidK9R7mR2nK2FfXS5rHnBimNlSx+kGsnMeesm9b5XJOkuiD+K5aizk+avxJs7bJa6S3vTIvccZRqt+P6c3y5D/pI+SVUocpPmtTtXvFy+vLyK9Ni23ldFLZt87+kBNCHuuzCLXnENtLm2Zfya3340+9F/LWh4GcS/eU/e2+tm497zXhp89BGqW/kXZSMe5zrOatGxT4VGJsLSkA69LPQf5e6qpE6mMlZ6z8EmctPGRNG+tlPxn4Nd+gOT9ZXE9vd9QETOC5CXzw3Ade4Tw+QEKuc9SYsh81omJ95Z9zN0rYN0n1/kFroxSlIJVz6QXtpDmLWgjS1J6+2pSyT36xPjtBY5SoUeqUHEbWgjRKKfudxmK9HFQsyUFYJ43SSkoS9YkFqVhf+a3LPtaTxF7mUdpI6zwGjcX6yo/yg8S+g8Re9u1yTMPDnHXWyOkIyvqsPH14AB/zpMs+e/ZVLGQ/5THmkaHPPudho8R5cxa0QG7JYdxKnLeSRmktHSUs5ZEhSFEaKxHHosR+jJqjFLMegrJB6rJf5qOcIEUJP0mnrNPimBOSRub0ypn4xQY53CNIrGH4G2mUtg+RP752xII0PETf+F32GVJW0FjHB81HCaMmfpLoqZOChI1ZMc8Z+qwSGuREKeTAqBERK9ZXfu12moQcSHmMeRw1EuvynKHP85Bjg8ZRitIqx+BGbMqCgl1eGDUOVRLxUM2TfITFPHLGKPF6YH0eO42ww/YS6+QOUk8wG/MohTwfNSJixfrsdBqpVer+McOeCZiACZjAqyPQq+Py5VE3Pxd/dRd0w7MPH6PZmIAJmIAJvAyBD1/mWJ/6DhFIukts7sO/6RM7vEP3fJ+vwk9d1g0A5sRtJmACJmACL0Dggxc400e+ewSCrhQlHtywo5Sk8c3U/3wHCGx0B15fxOuL/AD3DrywvoIJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJPCeB1/QfMfCXp79/Bg7/Q7D85eqXsqCDP5b4H1Q9PEMTUWdwJn+Z/CXvfeur1q/9V7c+zPVN4B0hsNY9+B/j/Voa35E7+RomYAKZwGv7nxH5rvr+UuJ/sZ6HT0Tsh9IvJD6wsCjxv4z+EkYvu2c4uNcZ3JeHm+c4b+5KUQu/nFu8Yvx7qpWuWO+1l+p0gS9e+yXc/80J8HnJe8VmAiZgAi9OIKmDfqILHmZKPMhvH2p40IkT+64d4ox07aIT9Y5VjLs9h/UThwTFWtYTaVcJ8X/lY3tDIGrYGoYJnCGQtN6fyfGyCZjAKyTw0cKeeUDYSKOEj4aFe58r7aCD1vmwUWP75Vb6f65+bn1O/f9TeLz1Ybl+4VsfN8X6mdp5r49Juj2ymYAJmIAJvIcE/mThnf+d8tAo/UbqpH9YuPfaaZw9SikXjnmep//ZP8khxt9F4+9+YDx4/C8Sa38q8avXuYcecslbSX+W80Ieif1zXvs3GomXM+Q+GLEoDQ+zP/bC34tjL/X/ldT2QO25mg+FspW8TvMk8Stk7sTfD8Qvvbe1yrn0QT7rJV/uWxY1g1/d41bzH0tJClK5T5Rfs9b0wYL0ifQvJHoqFuWUvydY+mhZVOlvub1mn0tB4r5fV6vEjnmNEePOnB2kEnuz8uafxGtuMefxOmGw/rcS74P6LNaCxF56x/5SYl+QyutLHM5tn3Bpa7KvxKlZ+qUHanCX/yDBrHBjT9sXeeT/J6mtUfIZ6b28hnLPWsx76KHwCTlW3keczd3bntriZV+5Z1QC9yzvk1Jnqj/2Tr2vOKPsq/nV8fq93+aQN2V1TXosTMllDutSK+R1xnN/HtmPRan8eajfd/WfX3rgNa/PZh9n1K+zpg8WpPL6/rX8UUosnLByl6Ccwr3EyusbtUYP5f1Xxynd9td+hlD73J+3woB6NhMwgRMElv4duKAaPz5R5yWXuurwQ/bpN1VxPoj4kC2qliZd6gzNfj6corSWNtJXEh90e4n6c8ZakthX7HM5XTWP8nvp1xI9UvOUkYOVEf9vpF2OM3xPStUc92dSJ5X+e/nbKodeOfsocT96JjdIBwnjzPpc1tKbpW/+2cuL0i9yZKjWOYN4JwWJOw9SlJZYl5PYt5PoEWOk7y7PGfBL31X4Lbdw2yoapT6vFh8O3DflOMNGIp8eGHuJc6KEfS71Enk/kcjBuizu/11peIi+2Rc1Ei88iAeJvfTA2k5asyD7gZSyX4YhO+RGqa/Wvyc/5XjQyDnM8U8Z5+0l7kfdXorVhsKvU2yUuFeq1udcGHG3TqJ2lLgf86P0+xxfaSxW8sv9yC/WyWEfrHqJeTFqfJljUSM57N18kzHtcF6pSY36PNaYfy2tc616/SdNfvt6UY8aB4nXYp/raHiw8np18oOUHqJvPmvIHaVyD84va4OckNc22X+zOv9P9u8k+uA+g7TK6X+V1zqNIa+x9MMqzv6BoIx9+KNEf1HqpGKl3javDXmBXn8nlXP/uMOeCZjAkwjwB/Qo/VLqJObF9nL4w/hclnQQ6qWdRF9T9ocmmDSPU4kTsY1iv6riK/ldnvcaU7XG/csa4disE0s5jo/1WfjUbj+4uFfH4glr7xeVm5r8NqfXOrWLRTmpmg/yu2pObp3f1iupdRx29R5yqNmXZI1JqnNYq9er1LdczoFXMfxjFRvk939cfji3zq+WvnGjPGpg5IY8/jaPb1be9NvlyagxZj9oLPtz6KGHQ7Ve6o4lIY9J41rqpVitbbJPfFvFya3zau7kdlUu7k4qtZiTH6ucQX5dv1r6xj3IC80CsVWORY3t6zL3PqnL9JqkKkC9oxSqGOsxz9caObe2urf6bkFJ1KotadJXgSif2Ckbm8U+z6PGdm2nWKryya3nLNVcYjPfar4nqTLyN3nO/bFB6rLPEKWyr5ePaksTsSblgWupzxq9ICxKfD4FaZVH4tgxz+s4vWxKQh4PGtfZjxrL+6Xeh9/ua8p4agImUBP4aCEO/gD2UpR+KvFvl8SwnTRm/7mGpIP6fNjxBofuVZN7rSXuGSViWC+tJNa+I/FhFKTHGnUwfrVQjLqhml/TPZ4o9qnWhmp9eyJ3bqnTAsxqGzX5sdRXwcLz7czzs7p/fM6KEvV2eeyrMnV+FX7LLf2Si6K0kpa8JnP16Qcb87jVyBfhJ3nO8FtpLSWJ/F9mn58wYYN0kNjD+GUeWWutUwDVRm/ESi+sUafYKGdVzVuX3mBAXm3U3UhDDvKTFmJPMfbzvh9ninBey495yHu+m/fBijtRa6n9l0r8L6rk38vnV8V7ideI8SD1OafTOGS/DEnOuomdmpJPz0H6WKJf+m6NszHOx/gz+iupvI/+UPlb+Zs3aYv/Sc+8xpxf1yw/KaPQB9KYKx7zyFC/XiX+I8XbHrgDsUPe+2uNJb8eybOZgAksJLD0AW5Qva20k/iDyDxJYx41vJgNjzh5pT3lg2NuO3W5cyeFKh8fDoPEB84PpadY0OZR4kuwWO1X4bty5xgSn7L1VPBKsZDrHDSO0kYq/mOPoM7cazJorctn9Bq30hL7WklzNeFD37yfPpeYj1KQOilK/IsTOUlqjYeAKVs1weNU0kys3VunhZk9twzzxV/zg1UxuESpl+qcP2bMe/+9lv6Havn/kP+/SltpL1Gb1yRKfyMFaZRqu4Rr2bfLdRg/kOr7NOXfmrYcysPrJQ+tpWB5jWtm+D+rTjycaqZaW5/IK+eQ8hhWJ0p7yQTeTwIfLrw2f/jKH7q9/E7aLNx767TxEQec+qAp5YZ8x6gxVWcQR3DAygdTyPNLh4M2TPVT6l5a7yn5X1f3KXXm+ogzB3Gf1qhRf0G060+ZUztVBQb5nRSbeJVy1h2Vceo1Yb3POTuNg3TO4DLFMijeSaNErY3U5xjjsYp38qM0ZfCdqs+5j7Wyt63LPD226CP3jdq3nthLL1HaSp00SnW/QfNz9oUS/rtKPLxh1Ex5DBp/8BB98y8HU0zy8uQQmmiv+VEq49KeeXhrLeTA1Fqb286TAr+XSo2y3s7bfVPzw0ytteLj1AbHTMAEHk9g6QNcewIfNvsc5EMutgk3nrcfnkuO48OFDxJsyf5ReUnifuwtRo1jNY/ZL2O1NOtuqhVq82+7nFOMM2I1X+LS06pKpAYWqli9XoW/cemh7oOFTZXIB32pMVdr1+xheyf1VZ0pd65encv5sQrgj9Khig3yiR+r2KXuqA2nXpO11vm1E7++4idfzM/ZPifEKrH4QbG6xqh5ynnbKp94udeqiuP2UtfENjnehL+ZtjXaPM76vKlLn8RTm3zj+aD69Mv5xbocCxrpqVis/Dq/Ci9yt03WIc97jRuJfop1lY+bmnnbR9D6VM/UZG3O6KlvFmOeE++qtXO1SupuYl+pWZV7y63vXi/0E7WCYpxxyuDz81MJXjMBE3ibwJ8sBPIb5f2txJfWfyMxP+S9f6GR//x/zPNbDUGF+fBi5H+CgREb81iGIKeX+M/r+VCg16P0j1InYaNU/icLcmhyIIf9qBh1osQH2J9LfDBtpCSF7Jf+iGGw+svs09Mowe1P89o/5Dn7qIkRm7KgYC+V+/2z/FGiV+rVNfiJAfPfSPTImczJo/8uj0FjksgbpU7CgrTPPkOpU+Kc3UsfSNyr7P8H+VsJ49xBghvWSyHrqDFK7C19HeTPGedxT/LhhMo59R7u9/cS+acsarGTVlKQRomeMO7wFxJnBImcPQsyeuXPwb+UYEwP/6P0C+mvpSiRw94kFWM/69Rijbv8Js+DRlT20T9zcshFa2mX43+nkfdAkJI0StSiPjUYyWF/kHqp/jOx0fwvpD+V6OcgTVlSkDxqMtLH3+bEqLGT2B+kUdpKH0j0+hvpKLUWFUAY55PTS3V/1AlZrI/S30udxHn0cpA4gzFUYv5raSP9Qx5ZL2fhdxJ18JM0Zayxhzsjav2T9M/SXvq77K81UuvPpEHCRilIxNnL/F9L5NIz+/9SYk+QBok8/H+Ueqn9M67QQx3E3eiNevRFT9Q9SlHi3L+QsCDR9/hm+q1/JkVKPrno7/PYaSw9jvKpz/rfSVP90XupxfjX0t9K9BelLq8HjaNEPYy78Gdqn+ceTMAETMAE3kMC2xveuVNt1NpuJt7mef5uEuh1rfRuXs23MgETuEcCH95jU+7JBB5BIOU9UeMt/y1+VP0w099hJu6wCZiACZiACVyVwJ9ctZqLmcDLEVjraH7Vw69kbvkgNar+SuLXU1HifzGfX43x66sk2d4/AlFXRhi/Crzl+y8f48EETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAETMAEHkPgg8ds8h4TMAETMIEXIbDSqd+fOflrxceZNYdNwATeMQIfvmP38XVMwARM4F0m0Oty35V4WPuh9FfSVznWabSZgAmYgAmYgAmYgAncGYGu6ifJX1fzeGe9uh0TMAETMAETMAETMAER4FeoxY4NkXrNsEzABEzABEzABEzABO6MQFQ/+zvrye2YgAk8IwH/HbhnhO2jTMAETOBKBKLqpCvVchkTMAETMAETMAETuDmBzc1PuP8Dklqs//7b/XfsDlsCQYFPJEabCVxM4KOLd3iDCZiACTw/gagjeWBBQXpff30IB7700Y8k/h7cKNleFwHexyvpIA1Z7+t7+nW9cu7WBEzABEzgUQSidqVH7fQmE7gfAjys8V7GGFP2PZjAYgL+O3CLUTnRBEzABN4bApv35qbXvSg/VVvCbqu8lI8OGsfrtuFq7wMBP8C9D6+y72gCJmACywnslDouT3dmReAoP0rdGSpjXi8PfDzQ2UzABEzABEzgnSUQdbP0zt7u5S/Gw9uSnyC9fKf33cFe7YUzLfLwNkiMyGYCFxHwf8RwES4nm4AJvGcE+GKd+/8efQwK/m+vzllUwh/OJS1cv+T/H5UHtyDx8GF7mwDvg+MFULbK3UkwnTLqdVnrnNdPJTpmAnME/AA3R8ZxEzABE3jzQJMyiF9rvOThJiqfL+r6AZD//9JSL5f91sAXf9nzM/nHb2VMB4LCiAeC7+SUX2ice4hoq3BubIOeP7yGsOF12C7kMSrvkPPZ2xqvSRFrQ5vguQmYgAmYgAm8SwSiLpOe+UJ8AfMTMbR+xNlBe3qJB4BhwX7yyeW8ny/In0rh4SDlGmEqoYl1mg8L8t63lFXmwshrz3thqQUljkuTnWcCJmACJmAC7zKB8mDy3Hc86EAeqH4r8WX+GGMfdcKCzdyzPDR2C/LnUtg7zC1WcfpaL8hzymUEYM9raTMBEzABE1hA4LFfsAtKO+WFCASd+5n0S4mHKPzuGXvh/KP0lJ+K0S7vzX5h30M+73can/Jwtc3nzh1L7XFu8YrxS+4+dWw/FbzzWKf+9nfeo9szARMwgbshwIem7XoEokrxEPC+WycA1/ipGA8yS4y8Qz7zV0s2zORQ59SZvLbDzN5O8SQdpV5q67DO2iCtpVO202K7/1R+u8bevg3ecM5Zo5QkfMQdUJCWGHm8Z2wmYAImYAILCHQLcpyynMBGqXxp2d48qPCF/NSfii1lyUNReWi81WuQdMb2REOs8VPPKVud2Vv28B7qpgpcGKOXeOGep6SP2kzvtTGf4zF1FjXi1IJjJmACJmACfyTAF0r7gWs+JnAtAry/DhIPVU/5qdgl/fDQUh7ibvHePqp+PNEQa3M/RepP7KuXhoV559Lgvz+XdKX1oDrcmzNrizPxJu2baZLHa2gzARMwARM4QaDTWjix/hqW6H8902j5Mmm/VEivY+w/V4M9nFXb3J42j7PKeXN7mtLvzJT73vqnYi2sfT6Tn/y1r0Wbe+mcu5yqydpUTlR8yWvP+4T+T1l5L53KKWtJziX5S2pO5XQKHiYW+pn4ROpDaCcNc4uOm8BjCfj/Suux5LzvngjwYd5LGylIa4kPzef4kNcxV7U+903veynk6ozciS9w7tllPy8/fEFEaSux7yh1Um1BE9ZQlKizlpIUpE5aSYc81/AQH6T6V0a95qPE/k5q9yj0Tht8fpJv+GONcLi1dTrgawnWX1zxMF5/bDxRs6yFKoc+mMPinHHGXB5rO4lxI8VzxbSecv6C1Cel0Atn1dZpQq+xiZ+aHrUYTiV4zQRMwATeRwIrXXov8aGKdXkMGofsP9fQ66AlorcpiwrygFZsK2eo5rj8JIQvOizmkbxd9hmSVHhU4Qc3SjyM1esHzalRrJeDikU5nFtb0qSvAvj1vM59V/29LgaXW/xUbIpZzOdxZj+V8IhYqXlu66iErkrCX53blNf7Zm+9jfdeMd6D9fuwzqv9XhN0axvzOVFjJ9Hr5hGH9tqTHrHPW0zgJIGPTq560QTun8BOLfJFyofrSjrmlkeN5cOW+CCRxzhl9d56fa3J96UPpGFqYxXrz6yfW05K+KucRD9Tv7Zimbx6nOo9KOGQ89qhXTueyG331vN0arFai/L/64W5z53Gw+z//shDO+0bJfjzU7EfPrLO0m1JiZ9Ln2UxR89how4J+SD+TDDnfdMaLFhPzcLYJuY5+T+V9tKuyQmaj02MKTHW5myrhe/MLeb4uf9XDep/LNFTuWeSf5DoubagCTklr1n21ARuQ8APcLfh6qrPR+BTHdXl46LGlP21xvKBWn8A5+VvDeSXve0iv7qKbfBGc87ZSHuJL5nyQFcf135RDFpMUi+tciLze7FRjfzHe2mm6aP+ieelLfI68Fr9UuK1eg7rdUiUeL8enuPAfMaYz2TKnfuZs6PivHeXWqn1pTbwMF0/BFOHe7a2agPNfAmX8UyNqHVe02OVh8+DIWupivfyd9KSc6ttdk3gaQT8APc0ft79sgSCjq+/OPlgLx+4W/l8qBYjd5TWEnmpLJwZ+VCOZ3LK8tKanTaMEzXpOUisYzGPDEEaq3ntcj2/9fYAABhXSURBVJ8ui/hGOs7kvkR41KHoXTTeTzx88No9h/FaBylKz/kaj/nMTuNOmrKoIOv85PgXTQKcUhODGbU2OT7mMWjkp1/U+UT6KsfLAINDE6un6cTa0qWoxLYOd8BKnD746Tw9fiExP5JgM4HnIOAHuOeg7DNuRWDMhdsPzi7Hd3nkg5cP/PIBTLz4OeUqQ3xiFfbTZ7FN5Qf5YzWv3ZUmfImkHDzO5L10mD7vtbfHsOm0CcXHbH7EHvj9UtpK9fvkEaW+2bL09Ri1g4cVrN1DX720kwaJn1jj0ydGr+vslyHK2UjkFUvZGTWSz0Ng+/BGyko65txbDVGFS//lDGLF6CFI3JV/iZzq84/Zj++XHg5SqovZNwEI+AHO74PXTmCjC5QPOR5iyod7V10syl/ntZR9ljsp4OQxZp8hZVWhm7u9TthJP5K+m33m3O//lDqJLwrGJI0Sxgc8eT/Mcxjwq8Gf5LUcfmADB2pspH0ejxqjxEhOkDByxzxesueQ97cD58U2eKU5vW6k/kr1zpXhvK0UpeO55Cut71TnZxIcr2WHXChqTCeKjlrjQWWYyCEGCzggaoYqL+X1KvRw1iBtcjA0OVHzuXuum9xc4ioD5wZpzGPxKT5I3JPzo7STOilJp2ylRZg8xqI2cV56zGbvMQETMIHXQGCjJvmgnLK9gkHiAzdKU3nE54y1fm7xDuJpogd4cO97sqk+t7nPo8ZRIodfnfEA+nMpSkuMvL5KZD7mGPFBoi7vAeZolDrpUuP9w971pRufkE/fwxP2n9p60OLmVILWuDOaspSDpUbQfGgS95rP7Z+qSU9Yuycotpva8Ewx+llXZw15Tl9zlrTQzS0uiBeuC1KdYgImYAKvj0B3ouXyZbBVTpSmconPGfn8HZd7tXK/ur+gyf7OGk5VP3wR0h99YqPUZ5+BLy3uxUMX/jmLSuirJHzOKEaNYzXHJRab2LkpNX/1iH3n6p5a77SYTiU8cW2v/f0Tauy0dy3Bk5E5nGoj3jexU1N6okbXJFE7nNr4zGtJ59Hn5sS5R61x/8faqdqPrel9JmACJnA3BE59yNUfnlEd84HbGvEpYy9rRVM5Lx2jx17iV6/8GvlTaSdN3fMxvcbHbJrYk6rYIL9+zXhQa8+h/1H6bbVvzmVvXy12TSI89k2sPr9Zmp1So609mzyz0M/Ep8JRwYP0lNdyq/1hqniOdRrTifUlS2sl9RLjnJ3ro94XJmpRu5sr/kJx+ownzmb9eGL93NJGCU957c/V97oJmIAJvHoC78qH5C3u0V3p1U0zdYLiPMAxttbntTbezqMC5GJBajkcFNvm9TLEZn5uOigBPcXYHxcWWCtvlMLC/Kk0agxTC1UMVvB/ilEjLijQLciZS2lfv7m8e4rT83BPDbkXEzABEzCB94NA0DXjla6aZupsFD8+Yq3eQo/9TI3ygMLDzGOt08b9YzdrH3fkV6/jwhr0fJAe23PQ3s8kHsy6BWdyN3p8rIXHbnzH9yXdL77jd/T1XojARy90ro81ARO4bwI8QESJL/WUfcZbWFRRHiCmjAeYeo35KB2nkmdiUfHfS4eZ9XNhGPxU6iR+RX3O6HGVk/BRyPPPz23Oe3+pkQe+70jnzuQszsDwYzXn3sPDymnbabmX9qfTZlfH2ZX3dyHo6ii9vwh8cxMwARMwgeckwEMAX+SMXT6YL3jmT7E0s/mgeDezNioe8hojuWWeww9DlPo6UPn0zn0eY0Gb+C9i+UnWNUS9czZc6Sz6pdZSg1Fcmuy8swTM8ywiJ5iACZiACVyTAA88m1ywy+OgMTzxkDSxn5o8aKwm1raKdU38MJFHKEr9zBp7qGU7TSBoOZ1O8epCAlF5PMDZTOBmBD68WWUXNgETeM0EeOgJ0iiVhyv8a9tGBflf3D82hfscG6r4Wj59YVFifs7o/ftSOpfo9YfXepB6s3gSAd5zMOyeVMWbTcAETMAETOBCAnwBbaXPpE32iT3V0kQBHsg4g/r8Xa8fSzspTOTSUydFaS0NVQ6xvpoXl9rHibhD8wQK5/kMr8wR4H2cJN6fNhMwARMwARN4EQLdlU9NTb1ec2JoL/HgEKQ5IydJU1+OUfG+2rjJc/Yg1rpq3e5pAuH0sldnCKwUN7sZOA5fl8BH1y3naiZgAiawmEC/OPNNYtDAw9tB4gGNL0v8KSsPblNrt4zR36cSI3aUkvRl9t9El/+TOtQYF24hv+xZyR8W7mvTlp7X7nvf57xWyGYCNyfgvwN3c8Q+wAReLYHhjjrnoeSQ+xnzyAPKS1nUwfy6l18zd7mJXR75O31fSz+U/kb6rjRKj+l3e+E+ehikvRSlINlMwARMwARMwARM4NEE0qN3vnn4CXk/D0LrplbUvH9C/aVbOZuHpM3EhpBjnUYevGpLmsQmdmrKOTwc/jaPp3LLGkxSldjLb/tYUsc5JmACr4CAf4X6Cl4kt2gC7wiBwxPucdRehDG2tUbF0DWMB6FeCrkY56Uc44GItdJLTnkYxjxh/75eqNaa8OyU+vza9XvS5zmr0xhmdsBjqqfVTL7DJmACr5yAH+Be+Qvo9k3gFRHg4edWNqrwcMXiG9XqpCSNVV0eiPifJamt/X95WGuxb3KC5kliPyM55yzm3JI3nNlAvs0ETOA9IeAHuPfkhfY1TcAEFhM45MyocZjY9UETO1bzIH+U6thO823OIc76EotK6iUe9uipkz6WpuzXCqaJhXKXiSWHTMAEXjMBP8C95lfPvZuACdySwGqi+KgYD2FzD0Ybre0lHrp42OI/YNhJo4QRZy95UarXHhIqW8k/SlFiz1AvzvjkccaYx34mz2ETMIFXTqD9N8lXfh23bwImYAJXIRBUhQeh/US1rWKs/zyvlV+h8sDFGjZKKY9vIm/+yXqUBmmqdp1LvSAd6uACf6Mc9lL/uCDfKSZgAiZgAiZgAibw3hCIuilaX3Dj8tCWtCdIPGjZTMAETMAETMAETMAE7phAzL3x0HfJg98dX8mtmYAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmIAJmMBrJBCu1HSnOl9IjFiUPpN2eX7LYaXiyPY0Ar22/1xaV2WC/HtgG9XHT6XneD9V138Rt9Op/FmqX4dbNbLkjKjDn+vP8q3u6bomYAIm8GIEwo1O/oPqbq5Umy/XvqoV5acr1T5VhjOf45xTPbwrawddJFaXges1H5rCE0DxPqWfe7WVGgtXao57xivVOlVm6s//1Ln3zv7UHb1mAibwDhH48BXe5VoPWe3Vf6DAvg0+cn585L6nbuMBo3tqEe9/INC+hnDtr8gmPqFW29sTSt1k61pVw00q367o1J//qc+ae2d/O0KubAImcFcEXtsD3Er04o0IHm5U9znL8uUyPueB79FZcL3ml3f3DrObevC59+u2f/7XahjZTMAETOAuCXx0l11NNxUV5u/+8BDH30PBPpd6iV9/YF9Ko9RJH0tf5/knGj+QksT+IO3zmoa38gcC2fgiIh8dpJTjzGO1Nsg/5rVzQ1DCp1USd8B6iXuUO+TwN0MnjzthU/dMilP399LuTdrDP6MUspJGdMqCFqPEfbgn817CmG+yz7CXyr17+d+RfpJzyMUGKUohi97KHrkPX5III07NUxa0GHPCqJE5GiTuj08PnUS9QcKiFCT6OkhJqi1oEnOA9dqIl/dQXy1ssh80lpr45XUY5Edpnde5G+u8j6n3mYR9KY3Znxu2WjhmTeVEBUNeII+zsCitsk8cv6wF+ZtqjTg5WJRC9omxxrxmvNY8Soe8vtLInX4s8V7gjl9JSZqzoAV64Az27+YSq3hX+aP8ILF/n+P1elJszPFeI33BO0hR2kqd9LH0tTTkeXltyvi54rUFTdZZnHvIi53GUmuUHyTuNeRxao+WbCZgAibwbhOIul6auCIxPoiLlQ/MMu/kjBIfnhjrfODGPGcYpL6a7+TXNVkPeb32o2LUqq3XBBWLclI15/w/SIzFOG9TzafcQcG+WmD/vppH+amar+X31Zy1WM2n3LpeXb/4jFg7J8adNnmdgVo7aZ1jXY6VlNjMya33l7wyBjmpCh7kE0MYPdHDVgpSIijj/D77DEmK1Rx/qObkjxLxYtRMzbyv5qzFPA8amdd3GTUnXow+l1pSIj0VG+QQK8Y5u2o+yCe2kvoqXs+D4oecQwr1Sy579wSzUZsYRo3COIce6rC/WJITq/kpl1xqYp00ZL8MSU6sYvRVeunk9xL7Q85JlU9okOre6JUYhh+yT90++wz4qZoXN8r5ldTlAGcfm7yd5kkq55JLfUZsak9e8mACJmACywh8uCzt7rP4wPxR1eX/397ZXzlSXFH8LMcByBFQRIAyoDKwMqAzsDKg+ct/Wo6AJgM5AjcRWI6AIgLkCOz7m30Fbxt9tBb2Y2bvO+fyvl9VXe20ajUzS5W9T36TDU4R44FLT//bNWHyXYoMPkWgpgsP3S70l3Bm6S9/Ta2y6P9eGFI1ezve6WY/uafIH2/0sGdquswyavIvmdtUwz5ZE0GzP2IIGr/nI/zGm9lJwY2ARlr44T69keZ+5u178oLeLWLUD0KL+Dk0cWI1/I10CRs1CzX5k+wx+SfZLfmYxLKMcqYUwO77a7K/EthHF2Il+WvNvQrpzetPi+aD/LwWefo2wiAUATkLc9j0TBEjVCKOIge6MJt5CDOQU+geq8l/xGReiQbW+fpOM1/n1CFN4OuUGdjskX1hd5lk/D351M7hb6Vb2PStlS9UOEUx8/hUr6Tmvp8+synHvqeoIQ9q+FZmwAyYgYcZ+NPDHR9nw1HbOghbgYfmRuABeUuo++5KQVX8h0Vul/y9bGpy7Mqoq+FJGXAQinBvvwxiz9RVYQ5N/zWhhp5BoK8ITbglo5JHgU9ZZoFvRyJbYQq7qybj3uWVmmvyuRI7oaaCdq1Ycc5CfRf2dLhQ3xaxWT69g3AWitAEBJt9tNfuqv9WVfGmPaTqzarOx4tYi71fE9Zl/9QBhBg9TZiEH8OepfvrWWVn7o7yQZ+3k00N0ueF+6Tm7ETNIrTKZR1QVlW//jbnVrWnWDN/ndaI51HUEc/SFv6jLjOXUhRoKZhtwsvnyTKfWm2aATNgBu4z8NwvcEVHbHHMSXovjMKlB2yU/aI2y0Dyz7J5g74mkxKzgH5boR+pQhGmJ+++HFQyCJyx3SnfKk/9TuBM+PfkqIIpaumbhXKj6dWN3JrUpCLOskZmFXGOMTR9xO4J556EKqzl4dZMZiDjraIHckW17YH6XNr3Ml2YsVHsIIxCFfbCJOyE3ifzDcnzTsvkW/hFPe1KH/v7l8Clcr5SswxznkH4WeDPHnaXS2fapvzbmiUa29sOcJ8ZMANm4I9m4LM/euB7nlfSepNs3pjAmjceev+Z+rNJ/xcCbzBZqhzAm8L0Rua1M16I3QpRP9wquJA7KtbPOV/I59BBDjgv6qp8cEnGCMIBdgt/lmbdLPjE31b4VKIsmuuNYeQmYRQ4F3qN9NpLPBQN+EnYLgZtbgyGm0s99UbPMsUvm3Qpy2Tyj7Lv7Q0elzU1YruYNUt3m9DSJ9ZnPPq6xBIXVb0YfR1kP/AwR01JtTXZ2dzI2QvfCqNwTslZ9pKHoti1r/PU+huzpQgzWPddyrue/y737tlmwAx8AAae2wVuFkc8oHnYVaElzrBnIT/QU/rpotAf7uhB2KeCPpdQi9wh5ak/RY71ATIIvOFVoRGQFIFva3Xpfu/p8aOMnYBeK+eor9LYWYoc1u3rNNnbKOixGvlTxJeK/eSeXjcqXgL0UIM/4kj6fGJdsG/xMCi/F6hD+rrh/kaxl1HgUxvAzzZ9napK2Ms5Z8V7jn3i95pZ9k7YRy8Kn7qSYlV2/lSWmjHqKMPv0meXCDBr+XNSs2J9HfLXZFLiLPSZ1NLHvN43yAZFQHot9iD0uiK7EZTsBep6Lbq8Tv3ytdH9PG8ZYzZ7ya/zKc2NkRdVU5R5fX872T+lWJ9dUzex/vp/J/ubqKdkEs5CxZFQy8x98tnrl+FnVaO+x+bkF9mnSGBn7vGRrrt96889+2IPueco/0CzxQyYATOwhoFXa4o+spqi/fBQ5oE6L/Y2yOdBeF7Eq/wxUCKX67aKbSLepAFShCo0gfX6XOK7RXyQPwnk+izqAbEuc7IxB2FaxO65JQpaKmTNbfLnsAfpvu+jbPwmzMIlYUbff5E9LYp28plHDfO61GSfZBehz6EeEOsyh0HNTiDfBHqvyRAJ6rrsZeCPwvbX8FMMdBlk9H3MsvGbgI0UgX72Mws1Ykdp+oqAYJ/CpnaXYk02sW3kUdRmnxpA3SCchUm4J30d+pg5CPQeopF4ryHfhBK5mvwpYih6yPWZ9HW5NI/YNtVQXwTiSAtgD0+R+2djXhWaMAt93YNscnm23KfXecIIoWYvoM8pVmTTe0zxGnkUtafw6aUWaQHsKhSBOoANuswyavJ7TZ/FGqCkmmVPUw6wB2qxLWbADJiBT4KB/rDksPsrJ66Kz1dyHyJcYlF0/RAbeKZrzlf2zRun5eUzMOqI9cIxr8UvlDpkBsyAGXgZDHz2Ao7Bmzd/ewXHK+cpiudvf10pey/hmva5kz2/l1VfxiK81vCXZZBz7XVflNp95gzwOvM1s0nnKLIBfzYsZsAMmIFPhoFXL+CkW52BB/pZuPQQr4qDLmOyP5TJmxAyC+zbsp6BqtL+Bo7mNb/0uq+f6MrnxEDRZqvA1w3AP4YtZTEDZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANm4GNm4CX8EkPndyvjK4EfbP92JemD6vgHN/lh6O9Dr2x1mRkwA2bADJgBM2AGzMDvZaBqQBP+t3IQv7nIb6/RR89+ZZ/LzIAZMANmwAyYATPwrBjgU67yEe941t7WXOBq1KH5xI6LHGezmAEzYAbMgBkwA2bgo2dg7T/ky+WGSw4yBMJ99uqsE+yE07M/iQ9gBsyAGTADZsAMfBIM/GnlKQ+qGwUuOaAJXOi4/LyNDGrq/7PnH2TPMaRK83NsCD/Hln1+Rq29Tv3y362sv4THnDVSVfR1FKJZr//M3EY280rk896I9b6+Fr25JtqszIAZMANmwAyYATPw4Rng25I1bWNe+Gt3yAXp38IUDUW6CfvkczFkvUGoEZ+lfwy7q0kGl8kSAWZwoaT3llBPLXXoGsVb6Z+FIfyNNPO/S/5ONn1HgV70vfWi3coMmAEzYAbMgBkwA++XgbOWq2nJWfYYfrZTyUVzUpQLD5ejLlyE8iVoDH9INT22jRg5enINqTni2LekRh26y0kGyLKTs1wHv9eRZ/8WM2AGzIAZMANmwAy8NwYe+RZqiV2huUgdwx+kz2HfUzUKvkyF/fJGbk7xluxubsKgFmmhf69iLntafhu2n6sqN6VFTmF3Dn7v+u43A2bADJgBM2AGzMBqBtZe4EZN3AWaNBeYfonBf1TyRQn7Hw8O6Be5B9uult+bt8y3q5OcMANmwAyYATNgBszAO2Zg7W+hcoHh0ybAxa2EfnR7czTQn6XKWV6SFiVvuHN4j/TcmteU/K/w+aKoz+/r3ZrhnBkwA2bADJgBM2AG3gsDay9wk3YzxI7Q+OfwR+ld2PfUXgU/Cd8I/XK0lc2MPu/eDPIH4T/CX1Nxlf1V+H12St81OcOfhSHNYD7rsJ7FDJgBM2AGzIAZMAMfBQOvVu6CS1YRuBhx0TqmvkH2KZDCV01m0MNMhN5D2FUadJnCL4tYC38vzRxmEKvhn6V3wiWhBnRpMqZwivQgMLMJzJ0EpAhD2F2NC9+uGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAZeBgP8FurfXsZRfAozYAbMgBkwA2bADHwaDPwfheLh5AgR0ZQAAAAASUVORK5CYII="
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "training loop of vanilla policy gradient<br>\n",
    "reward-to-go function is inherit from Rtg agent<br>\n",
    "\n",
    "\n",
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class VPG_Agent(RTG_Agent):\n",
    "    def __init__(self, env, model=MLPActorCritic, optimizer=Adam, batch_size=5000, render=False, \n",
    "                 steps_per_epoch=4000, epochs=50, gamma=0.99, pi_lr=3e-4, vf_lr=1e-3, train_v_iters=80, lam=0.97, \n",
    "                 max_ep_len=1000, logger_kwargs=dict(), save_freq=10):\n",
    "        \n",
    "        self.epochs=epochs\n",
    "        self.train_v_iters=train_v_iters\n",
    "        self.max_ep_len = max_ep_len\n",
    "        self.save_freq=save_freq\n",
    "        self.render=render\n",
    "        \n",
    "        # Special function to avoid certain slowdowns from PyTorch + MPI combo.\n",
    "        setup_pytorch_for_mpi()\n",
    "\n",
    "        self.env = env\n",
    "        obs_dim = self.env.observation_space.shape\n",
    "        act_dim = self.env.action_space.shape\n",
    "\n",
    "        # Sync params across processes\n",
    "        self.ac = model(env.observation_space, env.action_space)\n",
    "        sync_params(self.ac)\n",
    "\n",
    "        # Set up experience buffer\n",
    "        self.steps_per_epoch=steps_per_epoch\n",
    "        self.local_steps_per_epoch = int(self.steps_per_epoch / num_procs())\n",
    "        self.buf = Buffer(obs_dim, act_dim, self.local_steps_per_epoch, gamma, lam)\n",
    "\n",
    "        # Set up optimizers for policy and value function\n",
    "        self.pi_optimizer = optimizer(self.ac.pi.parameters(), lr=pi_lr)\n",
    "        self.vf_optimizer = optimizer(self.ac.v.parameters(), lr=vf_lr)\n",
    "\n",
    "        # Set up logger to record the experiment\n",
    "        self.logger = EpochLogger(**logger_kwargs)\n",
    "        self.logger.save_config(locals())\n",
    "\n",
    "        # Count variables\n",
    "        var_counts = tuple(core.count_vars(module) for module in [self.ac.pi, self.ac.v])\n",
    "        self.logger.log('\\nNumber of parameters: \\t pi: %d, \\t v: %d\\n'%var_counts)\n",
    "\n",
    "    # Set up function for computing VPG policy loss\n",
    "    def compute_loss_pi(self, data):\n",
    "        \"\"\"\n",
    "        compute policy loss (i.e. -log pi A), \n",
    "        \"\"\"\n",
    "        obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data['logp']\n",
    "\n",
    "        # Policy loss\n",
    "        ######### Insert Code Here #########\n",
    "        pi, logp = self.ac.pi(obs, act)\n",
    "        loss_pi = -(logp * adv).mean()\n",
    "        ####################################\n",
    "\n",
    "        # Useful extra info\n",
    "        approx_kl = (logp_old - logp).mean().item()\n",
    "        ent = pi.entropy().mean().item()\n",
    "        pi_info = dict(kl=approx_kl, ent=ent)\n",
    "\n",
    "        return loss_pi, pi_info\n",
    "\n",
    "    # Set up function for computing value loss\n",
    "    def compute_loss_v(self, data):\n",
    "        \"\"\"\n",
    "        compute value loss\n",
    "        e.g. loss_v is the average of (v - r)^2\n",
    "        \"\"\"\n",
    "        obs, ret = data['obs'], data['ret']\n",
    "        \n",
    "        ######### Insert Code Here #########\n",
    "        loss_v = ((self.ac.v(obs) - ret)**2).mean()\n",
    "        ####################################\n",
    "        \n",
    "        return loss_v\n",
    "    \n",
    "    def update_actor(self, data):\n",
    "        \"\"\"\n",
    "        compute policy loss and then perform a gradient descent \n",
    "        \"\"\"\n",
    "        ######### Insert Code Here #########\n",
    "        self.pi_optimizer.zero_grad()\n",
    "        loss_pi, pi_info = self.compute_loss_pi(data)\n",
    "        loss_pi.backward()\n",
    "        mpi_avg_grads(self.ac.pi)    # average grads across MPI processes\n",
    "        self.pi_optimizer.step()\n",
    "        ####################################\n",
    "        \n",
    "        return loss_pi, pi_info\n",
    "    \n",
    "    def update_critic(self, data):\n",
    "        \"\"\"\n",
    "        compute critic loss and then perform a gradient descent\n",
    "        \"\"\"\n",
    "        \n",
    "        ######### Insert Code Here #########\n",
    "        self.vf_optimizer.zero_grad()\n",
    "        loss_v = self.compute_loss_v(data)\n",
    "        loss_v.backward()\n",
    "        mpi_avg_grads(self.ac.v)    # average grads across MPI processes\n",
    "        self.vf_optimizer.step()\n",
    "        ####################################\n",
    "        \n",
    "        return loss_v\n",
    "        \n",
    "\n",
    "    def update(self):\n",
    "        \"\"\"\n",
    "        compute policy loss and value loss then update actor and critic \n",
    "        \"\"\"\n",
    "        data = self.buf.get()\n",
    "\n",
    "        # Get loss and info values before update\n",
    "        pi_l_old, pi_info_old = self.compute_loss_pi(data)\n",
    "        pi_l_old = pi_l_old.item()\n",
    "        v_l_old = self.compute_loss_v(data).item()\n",
    "\n",
    "#         # Train policy with a single step of gradient descent\n",
    "        loss_pi, pi_info=self.update_actor(data) \n",
    "\n",
    "        # Value function learning\n",
    "        for i in range(self.train_v_iters):\n",
    "            loss_v=self.update_critic(data)\n",
    "\n",
    "        # Log changes from update\n",
    "        kl, ent = pi_info['kl'], pi_info_old['ent']\n",
    "        self.logger.store(LossPi=pi_l_old, LossV=v_l_old,\n",
    "                     KL=kl, Entropy=ent,\n",
    "                     DeltaLossPi=(loss_pi.item() - pi_l_old),\n",
    "                     DeltaLossV=(loss_v.item() - v_l_old))\n",
    "\n",
    "    def train(self):\n",
    "        # Prepare for interaction with environment\n",
    "        start_time = time.time()\n",
    "        o, ep_ret, ep_len = self.env.reset(), 0, 0\n",
    "\n",
    "        # Main loop: collect experience in env and update/log each epoch\n",
    "        for epoch in range(self.epochs):\n",
    "            for t in range(self.local_steps_per_epoch):\n",
    "                a, v, logp = self.ac.step(torch.as_tensor(o, dtype=torch.float32))\n",
    "\n",
    "                next_o, r, d, _ = self.env.step(a)\n",
    "                ep_ret += r\n",
    "                ep_len += 1\n",
    "\n",
    "                # save and log\n",
    "                self.buf.store(o, a, r, v, logp)\n",
    "                self.logger.store(VVals=v)\n",
    "\n",
    "                # Update obs (critical!)\n",
    "                o = next_o\n",
    "\n",
    "                timeout = ep_len == self.max_ep_len\n",
    "                terminal = d or timeout\n",
    "                epoch_ended = t==self.local_steps_per_epoch-1\n",
    "\n",
    "                if terminal or epoch_ended:\n",
    "                    if epoch_ended and not(terminal):\n",
    "                        print('Warning: trajectory cut off by epoch at %d steps.'%ep_len, flush=True)\n",
    "                    # if trajectory didn't reach terminal state, bootstrap value target\n",
    "                    if timeout or epoch_ended:\n",
    "                        _, v, _ = self.ac.step(torch.as_tensor(o, dtype=torch.float32))\n",
    "                    else:\n",
    "                        v = 0\n",
    "                    self.buf.finish_path(v)\n",
    "                    if terminal:\n",
    "                        # only save EpRet / EpLen if trajectory finished\n",
    "                        self.logger.store(EpRet=ep_ret, EpLen=ep_len)\n",
    "                    o, ep_ret, ep_len = self.env.reset(), 0, 0\n",
    "                \n",
    "                if self.render:\n",
    "                    self.env.render()\n",
    "\n",
    "\n",
    "            # Save model\n",
    "            if (epoch % self.save_freq == 0) or (epoch == epochs-1):\n",
    "                self.logger.save_state({'env': self.env}, None)\n",
    "\n",
    "            # Perform VPG update!\n",
    "            self.update()\n",
    "\n",
    "            # Log info about epoch\n",
    "            self.logger.log_tabular('Epoch', epoch)\n",
    "            self.logger.log_tabular('EpRet', with_min_and_max=True)\n",
    "            self.logger.log_tabular('EpLen', average_only=True)\n",
    "            self.logger.log_tabular('VVals', with_min_and_max=True)\n",
    "            self.logger.log_tabular('TotalEnvInteracts', (epoch+1)*self.steps_per_epoch)\n",
    "            self.logger.log_tabular('LossPi', average_only=True)\n",
    "            self.logger.log_tabular('LossV', average_only=True)\n",
    "            self.logger.log_tabular('DeltaLossPi', average_only=True)\n",
    "            self.logger.log_tabular('DeltaLossV', average_only=True)\n",
    "            self.logger.log_tabular('Entropy', average_only=True)\n",
    "            self.logger.log_tabular('KL', average_only=True)\n",
    "            self.logger.log_tabular('Time', time.time()-start_time)\n",
    "            self.logger.dump_tabular()\n",
    "        \n",
    "        if self.render:\n",
    "            self.env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32;1mLogging data to /tmp/experiments/1646381919/progress.txt\u001b[0m\n",
      "\u001b[36;1mSaving config:\n",
      "\u001b[0m\n",
      "{\n",
      "    \"act_dim\":\t[],\n",
      "    \"batch_size\":\t5000,\n",
      "    \"env\":\t{\n",
      "        \"<TimeLimit<CartPoleEnv<CartPole-v0>>>\":\t{\n",
      "            \"_elapsed_steps\":\t0,\n",
      "            \"_max_episode_steps\":\t200,\n",
      "            \"action_space\":\t{\n",
      "                \"Discrete(2)\":\t{\n",
      "                    \"dtype\":\t\"int64\",\n",
      "                    \"n\":\t2,\n",
      "                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                    \"shape\":\t[]\n",
      "                }\n",
      "            },\n",
      "            \"env\":\t{\n",
      "                \"<CartPoleEnv<CartPole-v0>>\":\t{\n",
      "                    \"_pole_geom\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.FilledPolygon object at 0x7f5c493e7208>\":\t{\n",
      "                            \"_color\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7358>\":\t{\n",
      "                                    \"vec4\":\t[\n",
      "                                        0.8,\n",
      "                                        0.6,\n",
      "                                        0.4,\n",
      "                                        1\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"attrs\":\t[\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7358>\":\t{\n",
      "                                        \"vec4\":\t[\n",
      "                                            0.8,\n",
      "                                            0.6,\n",
      "                                            0.4,\n",
      "                                            1\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                        \"rotation\":\t-0.047655701937088506,\n",
      "                                        \"scale\":\t[\n",
      "                                            1.0,\n",
      "                                            1.0\n",
      "                                        ],\n",
      "                                        \"translation\":\t[\n",
      "                                            0.0,\n",
      "                                            7.5\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                        \"rotation\":\t0.0,\n",
      "                                        \"scale\":\t[\n",
      "                                            1.0,\n",
      "                                            1.0\n",
      "                                        ],\n",
      "                                        \"translation\":\t[\n",
      "                                            414.0354050993562,\n",
      "                                            100.0\n",
      "                                        ]\n",
      "                                    }\n",
      "                                }\n",
      "                            ],\n",
      "                            \"v\":\t[\n",
      "                                [\n",
      "                                    -5.0,\n",
      "                                    -5.0\n",
      "                                ],\n",
      "                                [\n",
      "                                    -5.0,\n",
      "                                    120.0\n",
      "                                ],\n",
      "                                [\n",
      "                                    5.0,\n",
      "                                    120.0\n",
      "                                ],\n",
      "                                [\n",
      "                                    5.0,\n",
      "                                    -5.0\n",
      "                                ]\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"action_space\":\t{\n",
      "                        \"Discrete(2)\":\t{\n",
      "                            \"dtype\":\t\"int64\",\n",
      "                            \"n\":\t2,\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"shape\":\t[]\n",
      "                        }\n",
      "                    },\n",
      "                    \"axle\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.FilledPolygon object at 0x7f5c493e70b8>\":\t{\n",
      "                            \"_color\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7048>\":\t{\n",
      "                                    \"vec4\":\t[\n",
      "                                        0.5,\n",
      "                                        0.5,\n",
      "                                        0.8,\n",
      "                                        1\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"attrs\":\t[\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7048>\":\t{\n",
      "                                        \"vec4\":\t[\n",
      "                                            0.5,\n",
      "                                            0.5,\n",
      "                                            0.8,\n",
      "                                            1\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                        \"rotation\":\t-0.047655701937088506,\n",
      "                                        \"scale\":\t[\n",
      "                                            1.0,\n",
      "                                            1.0\n",
      "                                        ],\n",
      "                                        \"translation\":\t[\n",
      "                                            0.0,\n",
      "                                            7.5\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                        \"rotation\":\t0.0,\n",
      "                                        \"scale\":\t[\n",
      "                                            1.0,\n",
      "                                            1.0\n",
      "                                        ],\n",
      "                                        \"translation\":\t[\n",
      "                                            414.0354050993562,\n",
      "                                            100.0\n",
      "                                        ]\n",
      "                                    }\n",
      "                                }\n",
      "                            ],\n",
      "                            \"v\":\t[\n",
      "                                [\n",
      "                                    5.0,\n",
      "                                    0.0\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.8907380036690284,\n",
      "                                    1.0395584540887965\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.567727288213004,\n",
      "                                    2.0336832153790008\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.045084971874737,\n",
      "                                    2.938926261462366\n",
      "                                ],\n",
      "                                [\n",
      "                                    3.345653031794291,\n",
      "                                    3.7157241273869706\n",
      "                                ],\n",
      "                                [\n",
      "                                    2.5000000000000004,\n",
      "                                    4.330127018922193\n",
      "                                ],\n",
      "                                [\n",
      "                                    1.5450849718747373,\n",
      "                                    4.755282581475767\n",
      "                                ],\n",
      "                                [\n",
      "                                    0.5226423163382673,\n",
      "                                    4.972609476841367\n",
      "                                ],\n",
      "                                [\n",
      "                                    -0.5226423163382666,\n",
      "                                    4.972609476841367\n",
      "                                ],\n",
      "                                [\n",
      "                                    -1.5450849718747368,\n",
      "                                    4.755282581475768\n",
      "                                ],\n",
      "                                [\n",
      "                                    -2.499999999999999,\n",
      "                                    4.330127018922194\n",
      "                                ],\n",
      "                                [\n",
      "                                    -3.3456530317942894,\n",
      "                                    3.7157241273869723\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.045084971874736,\n",
      "                                    2.9389262614623664\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.567727288213005,\n",
      "                                    2.0336832153790003\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.8907380036690284,\n",
      "                                    1.0395584540887965\n",
      "                                ],\n",
      "                                [\n",
      "                                    -5.0,\n",
      "                                    2.8327694488239896e-15\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.8907380036690284,\n",
      "                                    -1.0395584540887954\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.567727288213004,\n",
      "                                    -2.033683215379001\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.045084971874738,\n",
      "                                    -2.938926261462365\n",
      "                                ],\n",
      "                                [\n",
      "                                    -3.345653031794292,\n",
      "                                    -3.71572412738697\n",
      "                                ],\n",
      "                                [\n",
      "                                    -2.500000000000002,\n",
      "                                    -4.330127018922192\n",
      "                                ],\n",
      "                                [\n",
      "                                    -1.5450849718747377,\n",
      "                                    -4.755282581475767\n",
      "                                ],\n",
      "                                [\n",
      "                                    -0.5226423163382712,\n",
      "                                    -4.972609476841367\n",
      "                                ],\n",
      "                                [\n",
      "                                    0.522642316338265,\n",
      "                                    -4.972609476841367\n",
      "                                ],\n",
      "                                [\n",
      "                                    1.5450849718747361,\n",
      "                                    -4.755282581475768\n",
      "                                ],\n",
      "                                [\n",
      "                                    2.5000000000000004,\n",
      "                                    -4.330127018922193\n",
      "                                ],\n",
      "                                [\n",
      "                                    3.345653031794292,\n",
      "                                    -3.71572412738697\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.045084971874736,\n",
      "                                    -2.938926261462367\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.567727288213005,\n",
      "                                    -2.0336832153790008\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.8907380036690284,\n",
      "                                    -1.039558454088795\n",
      "                                ]\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"carttrans\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                            \"rotation\":\t0.0,\n",
      "                            \"scale\":\t[\n",
      "                                1.0,\n",
      "                                1.0\n",
      "                            ],\n",
      "                            \"translation\":\t[\n",
      "                                414.0354050993562,\n",
      "                                100.0\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"force_mag\":\t10.0,\n",
      "                    \"gravity\":\t9.8,\n",
      "                    \"kinematics_integrator\":\t\"euler\",\n",
      "                    \"length\":\t0.5,\n",
      "                    \"masscart\":\t1.0,\n",
      "                    \"masspole\":\t0.1,\n",
      "                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                    \"observation_space\":\t{\n",
      "                        \"Box(4,)\":\t{\n",
      "                            \"bounded_above\":\t\"[ True  True  True  True]\",\n",
      "                            \"bounded_below\":\t\"[ True  True  True  True]\",\n",
      "                            \"dtype\":\t\"float32\",\n",
      "                            \"high\":\t\"[4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\",\n",
      "                            \"low\":\t\"[-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\",\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"shape\":\t[\n",
      "                                4\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"polemass_length\":\t0.05,\n",
      "                    \"poletrans\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                            \"rotation\":\t-0.047655701937088506,\n",
      "                            \"scale\":\t[\n",
      "                                1.0,\n",
      "                                1.0\n",
      "                            ],\n",
      "                            \"translation\":\t[\n",
      "                                0.0,\n",
      "                                7.5\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"spec\":\t{\n",
      "                        \"EnvSpec(CartPole-v0)\":\t{\n",
      "                            \"_env_name\":\t\"CartPole\",\n",
      "                            \"_kwargs\":\t{},\n",
      "                            \"entry_point\":\t\"gym.envs.classic_control:CartPoleEnv\",\n",
      "                            \"id\":\t\"CartPole-v0\",\n",
      "                            \"max_episode_steps\":\t200,\n",
      "                            \"nondeterministic\":\tfalse,\n",
      "                            \"reward_threshold\":\t195.0,\n",
      "                            \"tags\":\t{\n",
      "                                \"wrapper_config.TimeLimit.max_episode_steps\":\t200\n",
      "                            }\n",
      "                        }\n",
      "                    },\n",
      "                    \"state\":\t\"[ 0.03613242 -0.04375426  0.02195785 -0.02908766]\",\n",
      "                    \"steps_beyond_done\":\tnull,\n",
      "                    \"tau\":\t0.02,\n",
      "                    \"theta_threshold_radians\":\t0.20943951023931953,\n",
      "                    \"total_mass\":\t1.1,\n",
      "                    \"track\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.Line object at 0x7f5c493e7080>\":\t{\n",
      "                            \"_color\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Color object at 0x7f5c49406ba8>\":\t{\n",
      "                                    \"vec4\":\t[\n",
      "                                        0,\n",
      "                                        0,\n",
      "                                        0,\n",
      "                                        1\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"attrs\":\t[\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Color object at 0x7f5c49406ba8>\":\t{\n",
      "                                        \"vec4\":\t[\n",
      "                                            0,\n",
      "                                            0,\n",
      "                                            0,\n",
      "                                            1\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.LineWidth object at 0x7f5c49406fd0>\":\t{\n",
      "                                        \"stroke\":\t1\n",
      "                                    }\n",
      "                                }\n",
      "                            ],\n",
      "                            \"end\":\t[\n",
      "                                600,\n",
      "                                100\n",
      "                            ],\n",
      "                            \"linewidth\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.LineWidth object at 0x7f5c49406fd0>\":\t{\n",
      "                                    \"stroke\":\t1\n",
      "                                }\n",
      "                            },\n",
      "                            \"start\":\t[\n",
      "                                0,\n",
      "                                100\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"viewer\":\tnull,\n",
      "                    \"x_threshold\":\t2.4\n",
      "                }\n",
      "            },\n",
      "            \"metadata\":\t{\n",
      "                \"render.modes\":\t[\n",
      "                    \"human\",\n",
      "                    \"rgb_array\"\n",
      "                ],\n",
      "                \"video.frames_per_second\":\t50\n",
      "            },\n",
      "            \"observation_space\":\t{\n",
      "                \"Box(4,)\":\t{\n",
      "                    \"bounded_above\":\t\"[ True  True  True  True]\",\n",
      "                    \"bounded_below\":\t\"[ True  True  True  True]\",\n",
      "                    \"dtype\":\t\"float32\",\n",
      "                    \"high\":\t\"[4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\",\n",
      "                    \"low\":\t\"[-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\",\n",
      "                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                    \"shape\":\t[\n",
      "                        4\n",
      "                    ]\n",
      "                }\n",
      "            },\n",
      "            \"reward_range\":\t[\n",
      "                -Infinity,\n",
      "                Infinity\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"epochs\":\t50,\n",
      "    \"gamma\":\t0.99,\n",
      "    \"lam\":\t0.97,\n",
      "    \"logger_kwargs\":\t{},\n",
      "    \"max_ep_len\":\t1000,\n",
      "    \"model\":\t\"MLPActorCritic\",\n",
      "    \"obs_dim\":\t[\n",
      "        4\n",
      "    ],\n",
      "    \"optimizer\":\t\"Adam\",\n",
      "    \"pi_lr\":\t0.0003,\n",
      "    \"render\":\tfalse,\n",
      "    \"save_freq\":\t10,\n",
      "    \"self\":\t{\n",
      "        \"<__main__.VPG_Agent object at 0x7f5bbacaa438>\":\t{\n",
      "            \"ac\":\t{\n",
      "                \"MLPActorCritic(\\n  (pi): MLPCategoricalActor(\\n    (logits_net): Sequential(\\n      (0): Linear(in_features=4, out_features=64, bias=True)\\n      (1): Tanh()\\n      (2): Linear(in_features=64, out_features=64, bias=True)\\n      (3): Tanh()\\n      (4): Linear(in_features=64, out_features=2, bias=True)\\n      (5): Identity()\\n    )\\n  )\\n  (v): MLPCritic(\\n    (v_net): Sequential(\\n      (0): Linear(in_features=4, out_features=64, bias=True)\\n      (1): Tanh()\\n      (2): Linear(in_features=64, out_features=64, bias=True)\\n      (3): Tanh()\\n      (4): Linear(in_features=64, out_features=1, bias=True)\\n      (5): Identity()\\n    )\\n  )\\n)\":\t{\n",
      "                    \"_backward_hooks\":\t{},\n",
      "                    \"_buffers\":\t{},\n",
      "                    \"_forward_hooks\":\t{},\n",
      "                    \"_forward_pre_hooks\":\t{},\n",
      "                    \"_load_state_dict_pre_hooks\":\t{},\n",
      "                    \"_modules\":\t{\n",
      "                        \"pi\":\t{\n",
      "                            \"MLPCategoricalActor(\\n  (logits_net): Sequential(\\n    (0): Linear(in_features=4, out_features=64, bias=True)\\n    (1): Tanh()\\n    (2): Linear(in_features=64, out_features=64, bias=True)\\n    (3): Tanh()\\n    (4): Linear(in_features=64, out_features=2, bias=True)\\n    (5): Identity()\\n  )\\n)\":\t{\n",
      "                                \"_backward_hooks\":\t{},\n",
      "                                \"_buffers\":\t{},\n",
      "                                \"_forward_hooks\":\t{},\n",
      "                                \"_forward_pre_hooks\":\t{},\n",
      "                                \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                \"_modules\":\t{\n",
      "                                    \"logits_net\":\t{\n",
      "                                        \"Sequential(\\n  (0): Linear(in_features=4, out_features=64, bias=True)\\n  (1): Tanh()\\n  (2): Linear(in_features=64, out_features=64, bias=True)\\n  (3): Tanh()\\n  (4): Linear(in_features=64, out_features=2, bias=True)\\n  (5): Identity()\\n)\":\t{\n",
      "                                            \"_backward_hooks\":\t{},\n",
      "                                            \"_buffers\":\t{},\n",
      "                                            \"_forward_hooks\":\t{},\n",
      "                                            \"_forward_pre_hooks\":\t{},\n",
      "                                            \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                            \"_modules\":\t{\n",
      "                                                \"0\":\t{\n",
      "                                                    \"Linear(in_features=4, out_features=64, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([ 0.1462,  0.0184, -0.3483,  0.0928,  0.4426, -0.0786, -0.1266,  0.0643,\\n         0.3026,  0.4176, -0.2833,  0.1837,  0.3147, -0.0257,  0.1503, -0.1987,\\n         0.1459, -0.3707,  0.2170, -0.0347,  0.2715,  0.4065, -0.0771, -0.4615,\\n         0.0582,  0.4531, -0.2310, -0.1571, -0.2624,  0.3416,  0.0954,  0.0164,\\n        -0.0971,  0.3121, -0.2008, -0.4443, -0.3475, -0.2054,  0.3458,  0.2278,\\n        -0.2764,  0.1556,  0.0531, -0.1441, -0.1364,  0.1680, -0.3923,  0.0759,\\n        -0.4472,  0.1701,  0.0505,  0.0485,  0.0678, -0.0381, -0.1631,  0.4840,\\n         0.0926, -0.1851, -0.1995, -0.2301, -0.4040,  0.3142, -0.4590,  0.0906],\\n       requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-4.6998e-04,  3.1012e-01, -1.4570e-01,  2.8674e-01],\\n        [ 4.2093e-01, -3.6876e-01, -6.9338e-02, -4.1966e-01],\\n        [ 8.2645e-02,  3.4448e-01,  3.3763e-01, -1.8627e-01],\\n        [ 4.7157e-01,  2.0330e-01,  4.5841e-01,  3.4306e-01],\\n        [-1.2877e-01,  6.5271e-02, -2.6600e-01,  3.4634e-01],\\n        [-2.4939e-01, -2.9880e-01, -4.1686e-01,  2.0920e-02],\\n        [ 3.6360e-01,  4.7940e-01,  4.7672e-02,  6.1234e-02],\\n        [-4.7131e-01, -1.7374e-01, -4.0066e-01,  4.6390e-01],\\n        [ 2.2907e-01, -3.9995e-01,  3.5327e-01,  2.8582e-01],\\n        [-2.9927e-01,  4.0659e-01,  3.0348e-01,  4.9957e-01],\\n        [-9.4785e-02,  3.9156e-01,  1.4390e-01,  1.0512e-01],\\n        [ 4.1378e-01,  4.6641e-01,  2.7238e-01, -3.6649e-01],\\n        [-3.5826e-01, -1.1789e-01,  4.5742e-01,  2.5171e-01],\\n        [ 1.0414e-01, -2.7562e-01,  6.0921e-02, -4.6472e-01],\\n        [-4.0377e-02, -3.0631e-01, -2.7675e-01,  3.9694e-01],\\n        [-4.8642e-01, -9.8470e-02, -1.7332e-01,  1.0863e-01],\\n        [ 2.4798e-01, -2.6653e-01, -2.6106e-01, -2.1263e-01],\\n        [-4.2883e-01,  4.6771e-01,  2.8758e-02, -2.7034e-01],\\n        [ 4.7803e-01,  2.4318e-01, -7.4578e-02,  1.0445e-01],\\n        [-2.1208e-01, -7.2541e-02,  3.5239e-01,  2.8421e-01],\\n        [-4.3103e-01, -3.7395e-01,  3.2648e-01, -3.0860e-01],\\n        [ 1.7797e-01, -1.8415e-01, -9.3208e-02, -6.5320e-02],\\n        [-7.7839e-03, -6.8073e-02,  1.1856e-01, -3.8009e-01],\\n        [-3.7605e-01, -1.4724e-01,  2.3689e-01,  4.8312e-01],\\n        [ 1.3574e-01, -1.5994e-01,  1.3779e-01,  4.0742e-01],\\n        [-3.9660e-01,  3.6442e-01, -2.3204e-01, -2.8644e-01],\\n        [ 2.2742e-01, -4.9066e-01, -1.6183e-01, -9.5361e-03],\\n        [ 2.1898e-01, -4.4094e-01,  2.6005e-02, -7.7995e-02],\\n        [ 3.3459e-01, -7.6835e-02,  4.7678e-01, -4.1064e-01],\\n        [ 1.6595e-01, -3.9144e-01, -2.5612e-01, -1.4156e-01],\\n        [-9.1048e-03,  1.4494e-02, -6.7702e-02, -3.4875e-01],\\n        [-3.5716e-01,  4.5371e-01, -7.4247e-02, -3.1795e-01],\\n        [ 8.8154e-02,  3.8169e-01,  2.8900e-01, -1.1277e-01],\\n        [-3.9515e-01, -4.6918e-01, -2.4198e-01, -4.4776e-01],\\n        [ 1.5933e-01,  4.4806e-01, -4.0713e-02, -3.3457e-01],\\n        [-3.7421e-02,  7.0395e-02, -3.6699e-01, -3.6049e-01],\\n        [-2.6902e-02, -4.5728e-01, -4.8374e-01,  4.3788e-01],\\n        [ 3.1358e-01,  4.1480e-01,  2.3821e-01, -4.1362e-01],\\n        [-3.5219e-01,  9.7852e-02, -7.9996e-02,  3.1569e-01],\\n        [ 2.3227e-01,  3.8977e-01, -3.8492e-01,  3.2563e-01],\\n        [-4.6015e-04,  2.6833e-01,  4.8895e-01, -2.1502e-01],\\n        [ 4.7819e-01, -4.5734e-01,  3.3636e-01,  2.8056e-01],\\n        [ 3.2979e-01,  2.3456e-02,  4.3381e-01, -2.1581e-01],\\n        [ 3.9741e-01,  2.5901e-01,  1.1246e-01,  2.8115e-02],\\n        [-4.2787e-01, -4.3186e-01,  2.0767e-01, -4.5633e-01],\\n        [ 2.7431e-01,  1.8012e-01,  2.4830e-01, -1.5683e-01],\\n        [-7.1979e-04, -2.8050e-01, -2.1932e-01, -4.6554e-01],\\n        [ 6.7315e-02,  4.5077e-01,  1.8522e-01,  4.8217e-01],\\n        [-4.3260e-01,  2.4437e-01, -3.6091e-01, -2.2247e-01],\\n        [-9.7008e-02,  1.5392e-01, -2.5330e-01,  4.0166e-01],\\n        [ 6.2135e-02,  3.3242e-01,  4.5172e-02,  1.0294e-01],\\n        [-2.6054e-01,  2.7516e-01,  2.7257e-02,  3.5170e-01],\\n        [-4.2285e-01,  2.8435e-02, -4.5089e-01, -1.9109e-01],\\n        [ 7.5672e-02, -2.9286e-01, -4.3585e-01, -2.4467e-01],\\n        [-2.6499e-01, -1.2603e-01,  4.0904e-01,  3.6527e-01],\\n        [-2.7239e-01, -3.3601e-02,  4.4752e-01,  5.2846e-02],\\n        [-1.8869e-01, -3.5218e-01,  2.3297e-01,  8.5523e-02],\\n        [ 2.5517e-01, -7.8907e-02,  2.3577e-01,  9.0129e-02],\\n        [ 1.4868e-01,  3.5723e-01,  2.9645e-02, -6.8709e-02],\\n        [-7.0738e-02,  2.3230e-01, -4.2466e-01, -2.8123e-01],\\n        [ 4.3012e-01, -9.0976e-02, -9.0328e-03, -6.4178e-02],\\n        [ 2.3488e-01,  3.9369e-01,  9.2438e-02,  2.0622e-01],\\n        [ 2.3284e-01, -4.7891e-01,  1.9283e-01,  1.0700e-01],\\n        [-1.5803e-01, -1.1409e-01, -2.0284e-01, -2.7710e-01]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t4,\n",
      "                                                        \"out_features\":\t64,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"1\":\t{\n",
      "                                                    \"Tanh()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"2\":\t{\n",
      "                                                    \"Linear(in_features=64, out_features=64, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([ 0.0736, -0.0340, -0.0072,  0.0407, -0.0355,  0.0574, -0.0992,  0.0839,\\n        -0.0581, -0.0114,  0.1241, -0.0270,  0.0387,  0.0374, -0.0430, -0.0397,\\n        -0.0808, -0.0397,  0.0318, -0.0053,  0.0988, -0.1231,  0.0076, -0.0762,\\n        -0.1041, -0.0566, -0.0041,  0.0315,  0.0909,  0.0608, -0.0003, -0.0827,\\n         0.0010, -0.1001,  0.0019,  0.0349,  0.0511,  0.0923,  0.0003,  0.0758,\\n        -0.1240, -0.0387,  0.0353,  0.0947,  0.0326, -0.0828,  0.0990,  0.0089,\\n         0.1018, -0.0238, -0.0076,  0.1164, -0.1114,  0.0005, -0.0468, -0.0739,\\n        -0.0840,  0.0790,  0.0323, -0.0751,  0.0776,  0.0331, -0.0205, -0.0731],\\n       requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-0.0332,  0.0027, -0.0417,  ..., -0.1138, -0.0774, -0.0015],\\n        [ 0.0276,  0.0043,  0.0643,  ...,  0.0343,  0.0575,  0.0684],\\n        [-0.0500, -0.0768, -0.0700,  ...,  0.0673,  0.1070,  0.0332],\\n        ...,\\n        [-0.0545,  0.0982,  0.0246,  ..., -0.1054, -0.1021, -0.0455],\\n        [ 0.0352, -0.0706,  0.0998,  ...,  0.1018, -0.1170, -0.0401],\\n        [-0.0707, -0.0441, -0.1089,  ...,  0.1174,  0.0029, -0.0171]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t64,\n",
      "                                                        \"out_features\":\t64,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"3\":\t{\n",
      "                                                    \"Tanh()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"4\":\t{\n",
      "                                                    \"Linear(in_features=64, out_features=2, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([-0.1006,  0.0018], requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-0.0620, -0.0428, -0.0498,  0.0188, -0.0537, -0.0878,  0.0565, -0.0173,\\n          0.1118,  0.0007, -0.0865, -0.1171, -0.0243, -0.0620, -0.0841,  0.1130,\\n         -0.0118, -0.0339,  0.0307, -0.0539, -0.0722, -0.0185, -0.1141, -0.0025,\\n          0.0309,  0.1212,  0.0064,  0.0119, -0.1218,  0.0280, -0.0437, -0.1028,\\n          0.0625,  0.0343, -0.1195, -0.0058,  0.0511,  0.0555, -0.0166,  0.0041,\\n          0.1017, -0.1248, -0.1126, -0.1041,  0.0566,  0.1014,  0.1163, -0.0021,\\n         -0.0185,  0.1152,  0.0270, -0.0879,  0.0336,  0.0630, -0.0057, -0.1041,\\n         -0.0635, -0.0373,  0.1250,  0.1133, -0.1244, -0.0387,  0.1191, -0.0647],\\n        [-0.0264, -0.0279,  0.0495,  0.0054, -0.0423,  0.1037,  0.0009, -0.0943,\\n         -0.0530, -0.0553, -0.0548,  0.0953, -0.0748,  0.0945, -0.1021, -0.1128,\\n          0.1094, -0.0578, -0.0753,  0.0750,  0.0987, -0.0601,  0.0060, -0.0258,\\n         -0.0564,  0.0213, -0.0854, -0.0630,  0.0817, -0.0831,  0.0004, -0.0195,\\n          0.1204,  0.0973,  0.1137,  0.0737,  0.0750,  0.0703, -0.0299,  0.1097,\\n          0.1062,  0.0020, -0.0152, -0.0687,  0.1199,  0.0834, -0.0574, -0.0650,\\n         -0.1028,  0.1040, -0.0778, -0.0238,  0.0125,  0.0821,  0.1051,  0.0841,\\n          0.0272, -0.0335, -0.0848,  0.0163, -0.0905, -0.0267, -0.0201,  0.0737]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t64,\n",
      "                                                        \"out_features\":\t2,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"5\":\t{\n",
      "                                                    \"Identity()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                }\n",
      "                                            },\n",
      "                                            \"_parameters\":\t{},\n",
      "                                            \"_state_dict_hooks\":\t{},\n",
      "                                            \"training\":\ttrue\n",
      "                                        }\n",
      "                                    }\n",
      "                                },\n",
      "                                \"_parameters\":\t{},\n",
      "                                \"_state_dict_hooks\":\t{},\n",
      "                                \"training\":\ttrue\n",
      "                            }\n",
      "                        },\n",
      "                        \"v\":\t{\n",
      "                            \"MLPCritic(\\n  (v_net): Sequential(\\n    (0): Linear(in_features=4, out_features=64, bias=True)\\n    (1): Tanh()\\n    (2): Linear(in_features=64, out_features=64, bias=True)\\n    (3): Tanh()\\n    (4): Linear(in_features=64, out_features=1, bias=True)\\n    (5): Identity()\\n  )\\n)\":\t{\n",
      "                                \"_backward_hooks\":\t{},\n",
      "                                \"_buffers\":\t{},\n",
      "                                \"_forward_hooks\":\t{},\n",
      "                                \"_forward_pre_hooks\":\t{},\n",
      "                                \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                \"_modules\":\t{\n",
      "                                    \"v_net\":\t{\n",
      "                                        \"Sequential(\\n  (0): Linear(in_features=4, out_features=64, bias=True)\\n  (1): Tanh()\\n  (2): Linear(in_features=64, out_features=64, bias=True)\\n  (3): Tanh()\\n  (4): Linear(in_features=64, out_features=1, bias=True)\\n  (5): Identity()\\n)\":\t{\n",
      "                                            \"_backward_hooks\":\t{},\n",
      "                                            \"_buffers\":\t{},\n",
      "                                            \"_forward_hooks\":\t{},\n",
      "                                            \"_forward_pre_hooks\":\t{},\n",
      "                                            \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                            \"_modules\":\t{\n",
      "                                                \"0\":\t{\n",
      "                                                    \"Linear(in_features=4, out_features=64, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([-0.0768,  0.4632, -0.2333, -0.2904,  0.0132,  0.2225,  0.4939, -0.4091,\\n         0.4234,  0.1856,  0.0149, -0.1018, -0.2026, -0.3553,  0.3341,  0.0343,\\n         0.2526,  0.0417, -0.1432,  0.0950,  0.3570, -0.4947, -0.1786,  0.1072,\\n         0.1430,  0.1486,  0.1937,  0.1190, -0.0286,  0.3577,  0.1485,  0.2513,\\n         0.0349, -0.3535, -0.2056, -0.0304, -0.0794, -0.0041, -0.0258,  0.2603,\\n        -0.1816, -0.0454, -0.3824,  0.2506,  0.2699, -0.3127, -0.0323, -0.2255,\\n        -0.0359, -0.1792,  0.0718,  0.0685, -0.1484,  0.2785,  0.4688,  0.2025,\\n         0.3250, -0.0613,  0.1821,  0.1761,  0.1739,  0.3699,  0.1431,  0.1635],\\n       requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-1.4779e-01, -1.6513e-01,  4.5608e-01,  1.7705e-02],\\n        [ 1.6460e-01, -3.0660e-01, -4.7980e-01, -1.5740e-01],\\n        [ 1.8660e-01,  5.0112e-02,  2.3510e-01, -1.9435e-01],\\n        [-1.8158e-02,  5.9127e-03,  2.0455e-01, -9.0426e-02],\\n        [ 4.2157e-01, -2.0276e-01, -2.2775e-01, -4.2095e-01],\\n        [-1.0172e-01,  4.9479e-01,  6.5881e-02,  3.1854e-01],\\n        [ 4.1180e-01,  2.7547e-01, -4.1047e-01, -7.3725e-02],\\n        [-2.4809e-01,  4.9596e-01,  3.1264e-01,  4.6013e-01],\\n        [ 4.4187e-01, -8.7037e-02,  1.9642e-02, -2.4936e-01],\\n        [ 3.3459e-01,  3.6383e-01, -1.8599e-01, -2.8310e-01],\\n        [ 4.6833e-01, -3.8741e-01,  3.9094e-01,  1.2016e-01],\\n        [-7.2815e-02, -3.8083e-02,  4.0931e-01,  3.2436e-01],\\n        [-2.6925e-01, -4.3280e-01, -2.1349e-01, -3.6287e-01],\\n        [ 1.5209e-01,  4.5413e-01,  2.5063e-01, -2.7198e-01],\\n        [ 2.6245e-01, -3.3182e-04,  4.8804e-02, -4.5525e-01],\\n        [ 3.6659e-01, -3.0841e-01, -1.8592e-01, -5.2418e-03],\\n        [-5.5922e-02,  4.2980e-02, -9.9560e-02, -4.8397e-01],\\n        [-3.0852e-01, -1.3424e-01, -2.8263e-03,  7.6566e-02],\\n        [ 4.6970e-01,  4.8015e-01, -1.4308e-01,  1.5733e-01],\\n        [ 3.4572e-01, -2.1658e-01, -1.0936e-01, -4.5696e-01],\\n        [ 1.1666e-01,  4.5208e-01, -4.0053e-01,  1.1736e-01],\\n        [ 3.2337e-01,  1.4597e-01, -2.2066e-01,  1.6236e-01],\\n        [ 3.8775e-01, -2.8750e-01,  2.5579e-01, -7.6185e-03],\\n        [-2.0268e-01,  3.6229e-01,  4.6378e-01, -2.9361e-01],\\n        [-8.3161e-02, -3.7062e-01, -4.6836e-01, -4.1633e-01],\\n        [ 4.2235e-01,  2.7080e-01, -1.0271e-02,  1.7595e-01],\\n        [-3.1201e-01, -7.6494e-02, -1.7564e-02,  6.1643e-02],\\n        [-4.6074e-01,  2.2365e-01, -3.1403e-01, -3.1841e-01],\\n        [-3.0979e-01, -1.2001e-01, -2.1103e-02,  2.0690e-01],\\n        [-4.4124e-02,  2.2344e-01, -2.3629e-01, -5.7336e-02],\\n        [-2.0842e-01, -4.5344e-01, -4.7104e-01, -1.4143e-01],\\n        [ 3.8904e-01,  2.3472e-01, -3.1882e-01,  1.7419e-01],\\n        [ 1.3640e-01,  1.8832e-01,  3.6614e-01, -2.7004e-01],\\n        [ 2.8583e-01, -3.3196e-01,  2.7505e-01,  9.9288e-03],\\n        [-2.2092e-01, -4.9461e-01, -4.8184e-01,  3.3664e-02],\\n        [ 1.9053e-01,  3.2309e-01,  2.8774e-01,  3.1700e-02],\\n        [ 3.4206e-01, -1.0255e-01,  4.7264e-02, -1.7248e-01],\\n        [-1.0749e-01, -4.7309e-01,  7.2995e-02, -2.7362e-01],\\n        [ 6.8609e-03,  7.3871e-02,  4.1710e-01,  1.2711e-01],\\n        [ 8.0324e-02,  3.7171e-01,  1.2520e-01, -1.1053e-01],\\n        [ 1.6694e-01, -4.8238e-01,  4.9763e-01,  3.5502e-01],\\n        [-1.8128e-01,  6.1007e-02, -2.2929e-02,  5.5617e-02],\\n        [ 3.5402e-02, -3.7458e-02,  3.1370e-02,  9.9977e-02],\\n        [-2.5939e-02, -3.2974e-01, -4.1837e-01, -9.0422e-02],\\n        [-3.0835e-01,  4.4383e-01, -2.0930e-01,  1.6944e-01],\\n        [-3.6328e-01, -7.2924e-02,  4.5334e-01,  1.2406e-01],\\n        [ 8.1340e-02,  2.6713e-01, -2.5024e-01, -2.1234e-01],\\n        [-3.2834e-01, -4.4792e-01, -1.5891e-01,  4.3340e-01],\\n        [-3.7805e-01, -6.5416e-02,  3.8897e-01, -5.0966e-02],\\n        [ 3.1323e-01,  2.3007e-01,  3.5751e-01,  1.9481e-01],\\n        [-3.6823e-01,  2.8482e-01,  1.1424e-01,  4.4458e-01],\\n        [-1.3802e-01, -3.4793e-01,  7.6160e-02, -1.1812e-01],\\n        [-8.1325e-02,  1.4059e-01, -3.5535e-01,  2.6395e-01],\\n        [-4.8063e-01,  4.4365e-01, -4.2407e-01, -4.9603e-01],\\n        [ 3.9746e-01,  3.3746e-01,  3.1435e-01,  3.9335e-01],\\n        [ 2.6849e-01,  4.7836e-01,  4.5042e-01,  2.0844e-01],\\n        [ 1.3067e-01,  2.7304e-01, -3.3973e-01,  3.3136e-01],\\n        [-4.7853e-01,  4.6302e-02,  4.7180e-01,  2.7340e-01],\\n        [-4.5080e-01,  8.5281e-03,  4.2976e-01,  3.6146e-01],\\n        [ 5.3002e-02,  8.1845e-02,  1.7077e-01, -1.9394e-01],\\n        [ 4.6716e-01, -1.3930e-01, -1.3618e-01, -4.9070e-01],\\n        [ 1.2360e-02,  1.7640e-01,  1.6197e-01, -2.3401e-01],\\n        [ 6.8806e-02,  4.3656e-01, -3.6437e-01,  4.5467e-01],\\n        [-2.6077e-02, -5.2636e-02,  2.1000e-01,  3.6400e-01]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t4,\n",
      "                                                        \"out_features\":\t64,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"1\":\t{\n",
      "                                                    \"Tanh()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"2\":\t{\n",
      "                                                    \"Linear(in_features=64, out_features=64, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([ 0.0165,  0.0638,  0.0902,  0.0887,  0.0588,  0.0455,  0.0981, -0.1062,\\n         0.0621,  0.0158,  0.0826,  0.1044,  0.0893, -0.1074,  0.0187,  0.0989,\\n         0.0277,  0.0139,  0.0315,  0.0909,  0.1041,  0.0326,  0.1152,  0.0651,\\n        -0.0394,  0.0345, -0.0092, -0.0180,  0.0502,  0.0350,  0.1038, -0.1070,\\n         0.0312,  0.0032,  0.0442, -0.0209,  0.0738,  0.0105, -0.1052,  0.0069,\\n         0.0511,  0.0161, -0.0315, -0.0044, -0.0986,  0.0101, -0.1125,  0.0076,\\n        -0.0612,  0.0102,  0.0062,  0.0374, -0.0753,  0.0420, -0.1098,  0.1210,\\n         0.0841, -0.0561,  0.1224,  0.0989, -0.0718, -0.1233, -0.0584,  0.0934],\\n       requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-0.0058, -0.0687,  0.0026,  ...,  0.0159,  0.0443,  0.0803],\\n        [-0.0143, -0.0973, -0.0717,  ...,  0.1158, -0.0156,  0.0377],\\n        [ 0.0002,  0.0028, -0.0267,  ...,  0.1245, -0.1107, -0.0250],\\n        ...,\\n        [ 0.1183, -0.0308, -0.0336,  ..., -0.0560, -0.0184,  0.0255],\\n        [-0.1013,  0.0334, -0.0829,  ..., -0.0439,  0.0499,  0.0882],\\n        [ 0.0140, -0.0729, -0.1180,  ...,  0.0094, -0.0186, -0.0606]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t64,\n",
      "                                                        \"out_features\":\t64,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"3\":\t{\n",
      "                                                    \"Tanh()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"4\":\t{\n",
      "                                                    \"Linear(in_features=64, out_features=1, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([-0.0813], requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-0.0655,  0.0324,  0.0444,  0.0208, -0.0952,  0.0866, -0.0996, -0.0474,\\n          0.0116,  0.0831, -0.1167, -0.1215, -0.0396, -0.0006,  0.0247, -0.0174,\\n         -0.0548,  0.0890, -0.0712,  0.0921,  0.0142, -0.0914,  0.0640, -0.0318,\\n          0.0379,  0.0714,  0.0656, -0.0088, -0.0066,  0.0105, -0.0930, -0.0730,\\n         -0.0711, -0.1168, -0.0178, -0.0392, -0.0931,  0.0290,  0.0886,  0.1189,\\n          0.0631, -0.0114, -0.0801, -0.0705, -0.0045,  0.1113, -0.0708, -0.0737,\\n          0.0699, -0.0374,  0.0489,  0.1097, -0.0880, -0.0667,  0.0874,  0.0746,\\n         -0.1231,  0.0641,  0.1023,  0.1056,  0.0544,  0.0792, -0.1057, -0.0464]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t64,\n",
      "                                                        \"out_features\":\t1,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"5\":\t{\n",
      "                                                    \"Identity()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                }\n",
      "                                            },\n",
      "                                            \"_parameters\":\t{},\n",
      "                                            \"_state_dict_hooks\":\t{},\n",
      "                                            \"training\":\ttrue\n",
      "                                        }\n",
      "                                    }\n",
      "                                },\n",
      "                                \"_parameters\":\t{},\n",
      "                                \"_state_dict_hooks\":\t{},\n",
      "                                \"training\":\ttrue\n",
      "                            }\n",
      "                        }\n",
      "                    },\n",
      "                    \"_parameters\":\t{},\n",
      "                    \"_state_dict_hooks\":\t{},\n",
      "                    \"training\":\ttrue\n",
      "                }\n",
      "            },\n",
      "            \"buf\":\t{\n",
      "                \"<__main__.Buffer object at 0x7f5b29498668>\":\t{\n",
      "                    \"act_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"adv_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"gamma\":\t0.99,\n",
      "                    \"lam\":\t0.97,\n",
      "                    \"logp_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"max_size\":\t4000,\n",
      "                    \"obs_buf\":\t\"[[0. 0. 0. 0.]\\n [0. 0. 0. 0.]\\n [0. 0. 0. 0.]\\n ...\\n [0. 0. 0. 0.]\\n [0. 0. 0. 0.]\\n [0. 0. 0. 0.]]\",\n",
      "                    \"path_start_idx\":\t0,\n",
      "                    \"ptr\":\t0,\n",
      "                    \"ret_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"rew_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"val_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\"\n",
      "                }\n",
      "            },\n",
      "            \"env\":\t{\n",
      "                \"<TimeLimit<CartPoleEnv<CartPole-v0>>>\":\t{\n",
      "                    \"_elapsed_steps\":\t0,\n",
      "                    \"_max_episode_steps\":\t200,\n",
      "                    \"action_space\":\t{\n",
      "                        \"Discrete(2)\":\t{\n",
      "                            \"dtype\":\t\"int64\",\n",
      "                            \"n\":\t2,\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"shape\":\t[]\n",
      "                        }\n",
      "                    },\n",
      "                    \"env\":\t{\n",
      "                        \"<CartPoleEnv<CartPole-v0>>\":\t{\n",
      "                            \"_pole_geom\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.FilledPolygon object at 0x7f5c493e7208>\":\t{\n",
      "                                    \"_color\":\t{\n",
      "                                        \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7358>\":\t{\n",
      "                                            \"vec4\":\t[\n",
      "                                                0.8,\n",
      "                                                0.6,\n",
      "                                                0.4,\n",
      "                                                1\n",
      "                                            ]\n",
      "                                        }\n",
      "                                    },\n",
      "                                    \"attrs\":\t[\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7358>\":\t{\n",
      "                                                \"vec4\":\t[\n",
      "                                                    0.8,\n",
      "                                                    0.6,\n",
      "                                                    0.4,\n",
      "                                                    1\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                                \"rotation\":\t-0.047655701937088506,\n",
      "                                                \"scale\":\t[\n",
      "                                                    1.0,\n",
      "                                                    1.0\n",
      "                                                ],\n",
      "                                                \"translation\":\t[\n",
      "                                                    0.0,\n",
      "                                                    7.5\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                                \"rotation\":\t0.0,\n",
      "                                                \"scale\":\t[\n",
      "                                                    1.0,\n",
      "                                                    1.0\n",
      "                                                ],\n",
      "                                                \"translation\":\t[\n",
      "                                                    414.0354050993562,\n",
      "                                                    100.0\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        }\n",
      "                                    ],\n",
      "                                    \"v\":\t[\n",
      "                                        [\n",
      "                                            -5.0,\n",
      "                                            -5.0\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -5.0,\n",
      "                                            120.0\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            5.0,\n",
      "                                            120.0\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            5.0,\n",
      "                                            -5.0\n",
      "                                        ]\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"action_space\":\t{\n",
      "                                \"Discrete(2)\":\t{\n",
      "                                    \"dtype\":\t\"int64\",\n",
      "                                    \"n\":\t2,\n",
      "                                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                                    \"shape\":\t[]\n",
      "                                }\n",
      "                            },\n",
      "                            \"axle\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.FilledPolygon object at 0x7f5c493e70b8>\":\t{\n",
      "                                    \"_color\":\t{\n",
      "                                        \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7048>\":\t{\n",
      "                                            \"vec4\":\t[\n",
      "                                                0.5,\n",
      "                                                0.5,\n",
      "                                                0.8,\n",
      "                                                1\n",
      "                                            ]\n",
      "                                        }\n",
      "                                    },\n",
      "                                    \"attrs\":\t[\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7048>\":\t{\n",
      "                                                \"vec4\":\t[\n",
      "                                                    0.5,\n",
      "                                                    0.5,\n",
      "                                                    0.8,\n",
      "                                                    1\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                                \"rotation\":\t-0.047655701937088506,\n",
      "                                                \"scale\":\t[\n",
      "                                                    1.0,\n",
      "                                                    1.0\n",
      "                                                ],\n",
      "                                                \"translation\":\t[\n",
      "                                                    0.0,\n",
      "                                                    7.5\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                                \"rotation\":\t0.0,\n",
      "                                                \"scale\":\t[\n",
      "                                                    1.0,\n",
      "                                                    1.0\n",
      "                                                ],\n",
      "                                                \"translation\":\t[\n",
      "                                                    414.0354050993562,\n",
      "                                                    100.0\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        }\n",
      "                                    ],\n",
      "                                    \"v\":\t[\n",
      "                                        [\n",
      "                                            5.0,\n",
      "                                            0.0\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.8907380036690284,\n",
      "                                            1.0395584540887965\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.567727288213004,\n",
      "                                            2.0336832153790008\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.045084971874737,\n",
      "                                            2.938926261462366\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            3.345653031794291,\n",
      "                                            3.7157241273869706\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            2.5000000000000004,\n",
      "                                            4.330127018922193\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            1.5450849718747373,\n",
      "                                            4.755282581475767\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            0.5226423163382673,\n",
      "                                            4.972609476841367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -0.5226423163382666,\n",
      "                                            4.972609476841367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -1.5450849718747368,\n",
      "                                            4.755282581475768\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -2.499999999999999,\n",
      "                                            4.330127018922194\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -3.3456530317942894,\n",
      "                                            3.7157241273869723\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.045084971874736,\n",
      "                                            2.9389262614623664\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.567727288213005,\n",
      "                                            2.0336832153790003\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.8907380036690284,\n",
      "                                            1.0395584540887965\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -5.0,\n",
      "                                            2.8327694488239896e-15\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.8907380036690284,\n",
      "                                            -1.0395584540887954\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.567727288213004,\n",
      "                                            -2.033683215379001\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.045084971874738,\n",
      "                                            -2.938926261462365\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -3.345653031794292,\n",
      "                                            -3.71572412738697\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -2.500000000000002,\n",
      "                                            -4.330127018922192\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -1.5450849718747377,\n",
      "                                            -4.755282581475767\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -0.5226423163382712,\n",
      "                                            -4.972609476841367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            0.522642316338265,\n",
      "                                            -4.972609476841367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            1.5450849718747361,\n",
      "                                            -4.755282581475768\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            2.5000000000000004,\n",
      "                                            -4.330127018922193\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            3.345653031794292,\n",
      "                                            -3.71572412738697\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.045084971874736,\n",
      "                                            -2.938926261462367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.567727288213005,\n",
      "                                            -2.0336832153790008\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.8907380036690284,\n",
      "                                            -1.039558454088795\n",
      "                                        ]\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"carttrans\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                    \"rotation\":\t0.0,\n",
      "                                    \"scale\":\t[\n",
      "                                        1.0,\n",
      "                                        1.0\n",
      "                                    ],\n",
      "                                    \"translation\":\t[\n",
      "                                        414.0354050993562,\n",
      "                                        100.0\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"force_mag\":\t10.0,\n",
      "                            \"gravity\":\t9.8,\n",
      "                            \"kinematics_integrator\":\t\"euler\",\n",
      "                            \"length\":\t0.5,\n",
      "                            \"masscart\":\t1.0,\n",
      "                            \"masspole\":\t0.1,\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"observation_space\":\t{\n",
      "                                \"Box(4,)\":\t{\n",
      "                                    \"bounded_above\":\t\"[ True  True  True  True]\",\n",
      "                                    \"bounded_below\":\t\"[ True  True  True  True]\",\n",
      "                                    \"dtype\":\t\"float32\",\n",
      "                                    \"high\":\t\"[4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\",\n",
      "                                    \"low\":\t\"[-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\",\n",
      "                                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                                    \"shape\":\t[\n",
      "                                        4\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"polemass_length\":\t0.05,\n",
      "                            \"poletrans\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                    \"rotation\":\t-0.047655701937088506,\n",
      "                                    \"scale\":\t[\n",
      "                                        1.0,\n",
      "                                        1.0\n",
      "                                    ],\n",
      "                                    \"translation\":\t[\n",
      "                                        0.0,\n",
      "                                        7.5\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"spec\":\t{\n",
      "                                \"EnvSpec(CartPole-v0)\":\t{\n",
      "                                    \"_env_name\":\t\"CartPole\",\n",
      "                                    \"_kwargs\":\t{},\n",
      "                                    \"entry_point\":\t\"gym.envs.classic_control:CartPoleEnv\",\n",
      "                                    \"id\":\t\"CartPole-v0\",\n",
      "                                    \"max_episode_steps\":\t200,\n",
      "                                    \"nondeterministic\":\tfalse,\n",
      "                                    \"reward_threshold\":\t195.0,\n",
      "                                    \"tags\":\t{\n",
      "                                        \"wrapper_config.TimeLimit.max_episode_steps\":\t200\n",
      "                                    }\n",
      "                                }\n",
      "                            },\n",
      "                            \"state\":\t\"[ 0.03613242 -0.04375426  0.02195785 -0.02908766]\",\n",
      "                            \"steps_beyond_done\":\tnull,\n",
      "                            \"tau\":\t0.02,\n",
      "                            \"theta_threshold_radians\":\t0.20943951023931953,\n",
      "                            \"total_mass\":\t1.1,\n",
      "                            \"track\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Line object at 0x7f5c493e7080>\":\t{\n",
      "                                    \"_color\":\t{\n",
      "                                        \"<gym.envs.classic_control.rendering.Color object at 0x7f5c49406ba8>\":\t{\n",
      "                                            \"vec4\":\t[\n",
      "                                                0,\n",
      "                                                0,\n",
      "                                                0,\n",
      "                                                1\n",
      "                                            ]\n",
      "                                        }\n",
      "                                    },\n",
      "                                    \"attrs\":\t[\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Color object at 0x7f5c49406ba8>\":\t{\n",
      "                                                \"vec4\":\t[\n",
      "                                                    0,\n",
      "                                                    0,\n",
      "                                                    0,\n",
      "                                                    1\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.LineWidth object at 0x7f5c49406fd0>\":\t{\n",
      "                                                \"stroke\":\t1\n",
      "                                            }\n",
      "                                        }\n",
      "                                    ],\n",
      "                                    \"end\":\t[\n",
      "                                        600,\n",
      "                                        100\n",
      "                                    ],\n",
      "                                    \"linewidth\":\t{\n",
      "                                        \"<gym.envs.classic_control.rendering.LineWidth object at 0x7f5c49406fd0>\":\t{\n",
      "                                            \"stroke\":\t1\n",
      "                                        }\n",
      "                                    },\n",
      "                                    \"start\":\t[\n",
      "                                        0,\n",
      "                                        100\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"viewer\":\tnull,\n",
      "                            \"x_threshold\":\t2.4\n",
      "                        }\n",
      "                    },\n",
      "                    \"metadata\":\t{\n",
      "                        \"render.modes\":\t[\n",
      "                            \"human\",\n",
      "                            \"rgb_array\"\n",
      "                        ],\n",
      "                        \"video.frames_per_second\":\t50\n",
      "                    },\n",
      "                    \"observation_space\":\t{\n",
      "                        \"Box(4,)\":\t{\n",
      "                            \"bounded_above\":\t\"[ True  True  True  True]\",\n",
      "                            \"bounded_below\":\t\"[ True  True  True  True]\",\n",
      "                            \"dtype\":\t\"float32\",\n",
      "                            \"high\":\t\"[4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\",\n",
      "                            \"low\":\t\"[-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\",\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"shape\":\t[\n",
      "                                4\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"reward_range\":\t[\n",
      "                        -Infinity,\n",
      "                        Infinity\n",
      "                    ]\n",
      "                }\n",
      "            },\n",
      "            \"epochs\":\t50,\n",
      "            \"local_steps_per_epoch\":\t4000,\n",
      "            \"logger\":\t{\n",
      "                \"<spinup.utils.logx.EpochLogger object at 0x7f5b29498710>\":\t{\n",
      "                    \"epoch_dict\":\t{},\n",
      "                    \"exp_name\":\tnull,\n",
      "                    \"first_row\":\ttrue,\n",
      "                    \"log_current_row\":\t{},\n",
      "                    \"log_headers\":\t[],\n",
      "                    \"output_dir\":\t\"/tmp/experiments/1646381919\",\n",
      "                    \"output_file\":\t{\n",
      "                        \"<_io.TextIOWrapper name='/tmp/experiments/1646381919/progress.txt' mode='w' encoding='UTF-8'>\":\t{\n",
      "                            \"mode\":\t\"w\"\n",
      "                        }\n",
      "                    }\n",
      "                }\n",
      "            },\n",
      "            \"max_ep_len\":\t1000,\n",
      "            \"pi_optimizer\":\t{\n",
      "                \"Adam (\\nParameter Group 0\\n    amsgrad: False\\n    betas: (0.9, 0.999)\\n    eps: 1e-08\\n    lr: 0.0003\\n    weight_decay: 0\\n)\":\t{\n",
      "                    \"defaults\":\t{\n",
      "                        \"amsgrad\":\tfalse,\n",
      "                        \"betas\":\t[\n",
      "                            0.9,\n",
      "                            0.999\n",
      "                        ],\n",
      "                        \"eps\":\t1e-08,\n",
      "                        \"lr\":\t0.0003,\n",
      "                        \"weight_decay\":\t0\n",
      "                    },\n",
      "                    \"param_groups\":\t[\n",
      "                        {\n",
      "                            \"amsgrad\":\tfalse,\n",
      "                            \"betas\":\t[\n",
      "                                0.9,\n",
      "                                0.999\n",
      "                            ],\n",
      "                            \"eps\":\t1e-08,\n",
      "                            \"lr\":\t0.0003,\n",
      "                            \"params\":\t[\n",
      "                                \"Parameter containing:\\ntensor([[-4.6998e-04,  3.1012e-01, -1.4570e-01,  2.8674e-01],\\n        [ 4.2093e-01, -3.6876e-01, -6.9338e-02, -4.1966e-01],\\n        [ 8.2645e-02,  3.4448e-01,  3.3763e-01, -1.8627e-01],\\n        [ 4.7157e-01,  2.0330e-01,  4.5841e-01,  3.4306e-01],\\n        [-1.2877e-01,  6.5271e-02, -2.6600e-01,  3.4634e-01],\\n        [-2.4939e-01, -2.9880e-01, -4.1686e-01,  2.0920e-02],\\n        [ 3.6360e-01,  4.7940e-01,  4.7672e-02,  6.1234e-02],\\n        [-4.7131e-01, -1.7374e-01, -4.0066e-01,  4.6390e-01],\\n        [ 2.2907e-01, -3.9995e-01,  3.5327e-01,  2.8582e-01],\\n        [-2.9927e-01,  4.0659e-01,  3.0348e-01,  4.9957e-01],\\n        [-9.4785e-02,  3.9156e-01,  1.4390e-01,  1.0512e-01],\\n        [ 4.1378e-01,  4.6641e-01,  2.7238e-01, -3.6649e-01],\\n        [-3.5826e-01, -1.1789e-01,  4.5742e-01,  2.5171e-01],\\n        [ 1.0414e-01, -2.7562e-01,  6.0921e-02, -4.6472e-01],\\n        [-4.0377e-02, -3.0631e-01, -2.7675e-01,  3.9694e-01],\\n        [-4.8642e-01, -9.8470e-02, -1.7332e-01,  1.0863e-01],\\n        [ 2.4798e-01, -2.6653e-01, -2.6106e-01, -2.1263e-01],\\n        [-4.2883e-01,  4.6771e-01,  2.8758e-02, -2.7034e-01],\\n        [ 4.7803e-01,  2.4318e-01, -7.4578e-02,  1.0445e-01],\\n        [-2.1208e-01, -7.2541e-02,  3.5239e-01,  2.8421e-01],\\n        [-4.3103e-01, -3.7395e-01,  3.2648e-01, -3.0860e-01],\\n        [ 1.7797e-01, -1.8415e-01, -9.3208e-02, -6.5320e-02],\\n        [-7.7839e-03, -6.8073e-02,  1.1856e-01, -3.8009e-01],\\n        [-3.7605e-01, -1.4724e-01,  2.3689e-01,  4.8312e-01],\\n        [ 1.3574e-01, -1.5994e-01,  1.3779e-01,  4.0742e-01],\\n        [-3.9660e-01,  3.6442e-01, -2.3204e-01, -2.8644e-01],\\n        [ 2.2742e-01, -4.9066e-01, -1.6183e-01, -9.5361e-03],\\n        [ 2.1898e-01, -4.4094e-01,  2.6005e-02, -7.7995e-02],\\n        [ 3.3459e-01, -7.6835e-02,  4.7678e-01, -4.1064e-01],\\n        [ 1.6595e-01, -3.9144e-01, -2.5612e-01, -1.4156e-01],\\n        [-9.1048e-03,  1.4494e-02, -6.7702e-02, -3.4875e-01],\\n        [-3.5716e-01,  4.5371e-01, -7.4247e-02, -3.1795e-01],\\n        [ 8.8154e-02,  3.8169e-01,  2.8900e-01, -1.1277e-01],\\n        [-3.9515e-01, -4.6918e-01, -2.4198e-01, -4.4776e-01],\\n        [ 1.5933e-01,  4.4806e-01, -4.0713e-02, -3.3457e-01],\\n        [-3.7421e-02,  7.0395e-02, -3.6699e-01, -3.6049e-01],\\n        [-2.6902e-02, -4.5728e-01, -4.8374e-01,  4.3788e-01],\\n        [ 3.1358e-01,  4.1480e-01,  2.3821e-01, -4.1362e-01],\\n        [-3.5219e-01,  9.7852e-02, -7.9996e-02,  3.1569e-01],\\n        [ 2.3227e-01,  3.8977e-01, -3.8492e-01,  3.2563e-01],\\n        [-4.6015e-04,  2.6833e-01,  4.8895e-01, -2.1502e-01],\\n        [ 4.7819e-01, -4.5734e-01,  3.3636e-01,  2.8056e-01],\\n        [ 3.2979e-01,  2.3456e-02,  4.3381e-01, -2.1581e-01],\\n        [ 3.9741e-01,  2.5901e-01,  1.1246e-01,  2.8115e-02],\\n        [-4.2787e-01, -4.3186e-01,  2.0767e-01, -4.5633e-01],\\n        [ 2.7431e-01,  1.8012e-01,  2.4830e-01, -1.5683e-01],\\n        [-7.1979e-04, -2.8050e-01, -2.1932e-01, -4.6554e-01],\\n        [ 6.7315e-02,  4.5077e-01,  1.8522e-01,  4.8217e-01],\\n        [-4.3260e-01,  2.4437e-01, -3.6091e-01, -2.2247e-01],\\n        [-9.7008e-02,  1.5392e-01, -2.5330e-01,  4.0166e-01],\\n        [ 6.2135e-02,  3.3242e-01,  4.5172e-02,  1.0294e-01],\\n        [-2.6054e-01,  2.7516e-01,  2.7257e-02,  3.5170e-01],\\n        [-4.2285e-01,  2.8435e-02, -4.5089e-01, -1.9109e-01],\\n        [ 7.5672e-02, -2.9286e-01, -4.3585e-01, -2.4467e-01],\\n        [-2.6499e-01, -1.2603e-01,  4.0904e-01,  3.6527e-01],\\n        [-2.7239e-01, -3.3601e-02,  4.4752e-01,  5.2846e-02],\\n        [-1.8869e-01, -3.5218e-01,  2.3297e-01,  8.5523e-02],\\n        [ 2.5517e-01, -7.8907e-02,  2.3577e-01,  9.0129e-02],\\n        [ 1.4868e-01,  3.5723e-01,  2.9645e-02, -6.8709e-02],\\n        [-7.0738e-02,  2.3230e-01, -4.2466e-01, -2.8123e-01],\\n        [ 4.3012e-01, -9.0976e-02, -9.0328e-03, -6.4178e-02],\\n        [ 2.3488e-01,  3.9369e-01,  9.2438e-02,  2.0622e-01],\\n        [ 2.3284e-01, -4.7891e-01,  1.9283e-01,  1.0700e-01],\\n        [-1.5803e-01, -1.1409e-01, -2.0284e-01, -2.7710e-01]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([ 0.1462,  0.0184, -0.3483,  0.0928,  0.4426, -0.0786, -0.1266,  0.0643,\\n         0.3026,  0.4176, -0.2833,  0.1837,  0.3147, -0.0257,  0.1503, -0.1987,\\n         0.1459, -0.3707,  0.2170, -0.0347,  0.2715,  0.4065, -0.0771, -0.4615,\\n         0.0582,  0.4531, -0.2310, -0.1571, -0.2624,  0.3416,  0.0954,  0.0164,\\n        -0.0971,  0.3121, -0.2008, -0.4443, -0.3475, -0.2054,  0.3458,  0.2278,\\n        -0.2764,  0.1556,  0.0531, -0.1441, -0.1364,  0.1680, -0.3923,  0.0759,\\n        -0.4472,  0.1701,  0.0505,  0.0485,  0.0678, -0.0381, -0.1631,  0.4840,\\n         0.0926, -0.1851, -0.1995, -0.2301, -0.4040,  0.3142, -0.4590,  0.0906],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([[-0.0332,  0.0027, -0.0417,  ..., -0.1138, -0.0774, -0.0015],\\n        [ 0.0276,  0.0043,  0.0643,  ...,  0.0343,  0.0575,  0.0684],\\n        [-0.0500, -0.0768, -0.0700,  ...,  0.0673,  0.1070,  0.0332],\\n        ...,\\n        [-0.0545,  0.0982,  0.0246,  ..., -0.1054, -0.1021, -0.0455],\\n        [ 0.0352, -0.0706,  0.0998,  ...,  0.1018, -0.1170, -0.0401],\\n        [-0.0707, -0.0441, -0.1089,  ...,  0.1174,  0.0029, -0.0171]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([ 0.0736, -0.0340, -0.0072,  0.0407, -0.0355,  0.0574, -0.0992,  0.0839,\\n        -0.0581, -0.0114,  0.1241, -0.0270,  0.0387,  0.0374, -0.0430, -0.0397,\\n        -0.0808, -0.0397,  0.0318, -0.0053,  0.0988, -0.1231,  0.0076, -0.0762,\\n        -0.1041, -0.0566, -0.0041,  0.0315,  0.0909,  0.0608, -0.0003, -0.0827,\\n         0.0010, -0.1001,  0.0019,  0.0349,  0.0511,  0.0923,  0.0003,  0.0758,\\n        -0.1240, -0.0387,  0.0353,  0.0947,  0.0326, -0.0828,  0.0990,  0.0089,\\n         0.1018, -0.0238, -0.0076,  0.1164, -0.1114,  0.0005, -0.0468, -0.0739,\\n        -0.0840,  0.0790,  0.0323, -0.0751,  0.0776,  0.0331, -0.0205, -0.0731],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([[-0.0620, -0.0428, -0.0498,  0.0188, -0.0537, -0.0878,  0.0565, -0.0173,\\n          0.1118,  0.0007, -0.0865, -0.1171, -0.0243, -0.0620, -0.0841,  0.1130,\\n         -0.0118, -0.0339,  0.0307, -0.0539, -0.0722, -0.0185, -0.1141, -0.0025,\\n          0.0309,  0.1212,  0.0064,  0.0119, -0.1218,  0.0280, -0.0437, -0.1028,\\n          0.0625,  0.0343, -0.1195, -0.0058,  0.0511,  0.0555, -0.0166,  0.0041,\\n          0.1017, -0.1248, -0.1126, -0.1041,  0.0566,  0.1014,  0.1163, -0.0021,\\n         -0.0185,  0.1152,  0.0270, -0.0879,  0.0336,  0.0630, -0.0057, -0.1041,\\n         -0.0635, -0.0373,  0.1250,  0.1133, -0.1244, -0.0387,  0.1191, -0.0647],\\n        [-0.0264, -0.0279,  0.0495,  0.0054, -0.0423,  0.1037,  0.0009, -0.0943,\\n         -0.0530, -0.0553, -0.0548,  0.0953, -0.0748,  0.0945, -0.1021, -0.1128,\\n          0.1094, -0.0578, -0.0753,  0.0750,  0.0987, -0.0601,  0.0060, -0.0258,\\n         -0.0564,  0.0213, -0.0854, -0.0630,  0.0817, -0.0831,  0.0004, -0.0195,\\n          0.1204,  0.0973,  0.1137,  0.0737,  0.0750,  0.0703, -0.0299,  0.1097,\\n          0.1062,  0.0020, -0.0152, -0.0687,  0.1199,  0.0834, -0.0574, -0.0650,\\n         -0.1028,  0.1040, -0.0778, -0.0238,  0.0125,  0.0821,  0.1051,  0.0841,\\n          0.0272, -0.0335, -0.0848,  0.0163, -0.0905, -0.0267, -0.0201,  0.0737]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([-0.1006,  0.0018], requires_grad=True)\"\n",
      "                            ],\n",
      "                            \"weight_decay\":\t0\n",
      "                        }\n",
      "                    ],\n",
      "                    \"state\":\t{}\n",
      "                }\n",
      "            },\n",
      "            \"render\":\tfalse,\n",
      "            \"save_freq\":\t10,\n",
      "            \"steps_per_epoch\":\t4000,\n",
      "            \"train_v_iters\":\t80,\n",
      "            \"vf_optimizer\":\t{\n",
      "                \"Adam (\\nParameter Group 0\\n    amsgrad: False\\n    betas: (0.9, 0.999)\\n    eps: 1e-08\\n    lr: 0.001\\n    weight_decay: 0\\n)\":\t{\n",
      "                    \"defaults\":\t{\n",
      "                        \"amsgrad\":\tfalse,\n",
      "                        \"betas\":\t[\n",
      "                            0.9,\n",
      "                            0.999\n",
      "                        ],\n",
      "                        \"eps\":\t1e-08,\n",
      "                        \"lr\":\t0.001,\n",
      "                        \"weight_decay\":\t0\n",
      "                    },\n",
      "                    \"param_groups\":\t[\n",
      "                        {\n",
      "                            \"amsgrad\":\tfalse,\n",
      "                            \"betas\":\t[\n",
      "                                0.9,\n",
      "                                0.999\n",
      "                            ],\n",
      "                            \"eps\":\t1e-08,\n",
      "                            \"lr\":\t0.001,\n",
      "                            \"params\":\t[\n",
      "                                \"Parameter containing:\\ntensor([[-1.4779e-01, -1.6513e-01,  4.5608e-01,  1.7705e-02],\\n        [ 1.6460e-01, -3.0660e-01, -4.7980e-01, -1.5740e-01],\\n        [ 1.8660e-01,  5.0112e-02,  2.3510e-01, -1.9435e-01],\\n        [-1.8158e-02,  5.9127e-03,  2.0455e-01, -9.0426e-02],\\n        [ 4.2157e-01, -2.0276e-01, -2.2775e-01, -4.2095e-01],\\n        [-1.0172e-01,  4.9479e-01,  6.5881e-02,  3.1854e-01],\\n        [ 4.1180e-01,  2.7547e-01, -4.1047e-01, -7.3725e-02],\\n        [-2.4809e-01,  4.9596e-01,  3.1264e-01,  4.6013e-01],\\n        [ 4.4187e-01, -8.7037e-02,  1.9642e-02, -2.4936e-01],\\n        [ 3.3459e-01,  3.6383e-01, -1.8599e-01, -2.8310e-01],\\n        [ 4.6833e-01, -3.8741e-01,  3.9094e-01,  1.2016e-01],\\n        [-7.2815e-02, -3.8083e-02,  4.0931e-01,  3.2436e-01],\\n        [-2.6925e-01, -4.3280e-01, -2.1349e-01, -3.6287e-01],\\n        [ 1.5209e-01,  4.5413e-01,  2.5063e-01, -2.7198e-01],\\n        [ 2.6245e-01, -3.3182e-04,  4.8804e-02, -4.5525e-01],\\n        [ 3.6659e-01, -3.0841e-01, -1.8592e-01, -5.2418e-03],\\n        [-5.5922e-02,  4.2980e-02, -9.9560e-02, -4.8397e-01],\\n        [-3.0852e-01, -1.3424e-01, -2.8263e-03,  7.6566e-02],\\n        [ 4.6970e-01,  4.8015e-01, -1.4308e-01,  1.5733e-01],\\n        [ 3.4572e-01, -2.1658e-01, -1.0936e-01, -4.5696e-01],\\n        [ 1.1666e-01,  4.5208e-01, -4.0053e-01,  1.1736e-01],\\n        [ 3.2337e-01,  1.4597e-01, -2.2066e-01,  1.6236e-01],\\n        [ 3.8775e-01, -2.8750e-01,  2.5579e-01, -7.6185e-03],\\n        [-2.0268e-01,  3.6229e-01,  4.6378e-01, -2.9361e-01],\\n        [-8.3161e-02, -3.7062e-01, -4.6836e-01, -4.1633e-01],\\n        [ 4.2235e-01,  2.7080e-01, -1.0271e-02,  1.7595e-01],\\n        [-3.1201e-01, -7.6494e-02, -1.7564e-02,  6.1643e-02],\\n        [-4.6074e-01,  2.2365e-01, -3.1403e-01, -3.1841e-01],\\n        [-3.0979e-01, -1.2001e-01, -2.1103e-02,  2.0690e-01],\\n        [-4.4124e-02,  2.2344e-01, -2.3629e-01, -5.7336e-02],\\n        [-2.0842e-01, -4.5344e-01, -4.7104e-01, -1.4143e-01],\\n        [ 3.8904e-01,  2.3472e-01, -3.1882e-01,  1.7419e-01],\\n        [ 1.3640e-01,  1.8832e-01,  3.6614e-01, -2.7004e-01],\\n        [ 2.8583e-01, -3.3196e-01,  2.7505e-01,  9.9288e-03],\\n        [-2.2092e-01, -4.9461e-01, -4.8184e-01,  3.3664e-02],\\n        [ 1.9053e-01,  3.2309e-01,  2.8774e-01,  3.1700e-02],\\n        [ 3.4206e-01, -1.0255e-01,  4.7264e-02, -1.7248e-01],\\n        [-1.0749e-01, -4.7309e-01,  7.2995e-02, -2.7362e-01],\\n        [ 6.8609e-03,  7.3871e-02,  4.1710e-01,  1.2711e-01],\\n        [ 8.0324e-02,  3.7171e-01,  1.2520e-01, -1.1053e-01],\\n        [ 1.6694e-01, -4.8238e-01,  4.9763e-01,  3.5502e-01],\\n        [-1.8128e-01,  6.1007e-02, -2.2929e-02,  5.5617e-02],\\n        [ 3.5402e-02, -3.7458e-02,  3.1370e-02,  9.9977e-02],\\n        [-2.5939e-02, -3.2974e-01, -4.1837e-01, -9.0422e-02],\\n        [-3.0835e-01,  4.4383e-01, -2.0930e-01,  1.6944e-01],\\n        [-3.6328e-01, -7.2924e-02,  4.5334e-01,  1.2406e-01],\\n        [ 8.1340e-02,  2.6713e-01, -2.5024e-01, -2.1234e-01],\\n        [-3.2834e-01, -4.4792e-01, -1.5891e-01,  4.3340e-01],\\n        [-3.7805e-01, -6.5416e-02,  3.8897e-01, -5.0966e-02],\\n        [ 3.1323e-01,  2.3007e-01,  3.5751e-01,  1.9481e-01],\\n        [-3.6823e-01,  2.8482e-01,  1.1424e-01,  4.4458e-01],\\n        [-1.3802e-01, -3.4793e-01,  7.6160e-02, -1.1812e-01],\\n        [-8.1325e-02,  1.4059e-01, -3.5535e-01,  2.6395e-01],\\n        [-4.8063e-01,  4.4365e-01, -4.2407e-01, -4.9603e-01],\\n        [ 3.9746e-01,  3.3746e-01,  3.1435e-01,  3.9335e-01],\\n        [ 2.6849e-01,  4.7836e-01,  4.5042e-01,  2.0844e-01],\\n        [ 1.3067e-01,  2.7304e-01, -3.3973e-01,  3.3136e-01],\\n        [-4.7853e-01,  4.6302e-02,  4.7180e-01,  2.7340e-01],\\n        [-4.5080e-01,  8.5281e-03,  4.2976e-01,  3.6146e-01],\\n        [ 5.3002e-02,  8.1845e-02,  1.7077e-01, -1.9394e-01],\\n        [ 4.6716e-01, -1.3930e-01, -1.3618e-01, -4.9070e-01],\\n        [ 1.2360e-02,  1.7640e-01,  1.6197e-01, -2.3401e-01],\\n        [ 6.8806e-02,  4.3656e-01, -3.6437e-01,  4.5467e-01],\\n        [-2.6077e-02, -5.2636e-02,  2.1000e-01,  3.6400e-01]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([-0.0768,  0.4632, -0.2333, -0.2904,  0.0132,  0.2225,  0.4939, -0.4091,\\n         0.4234,  0.1856,  0.0149, -0.1018, -0.2026, -0.3553,  0.3341,  0.0343,\\n         0.2526,  0.0417, -0.1432,  0.0950,  0.3570, -0.4947, -0.1786,  0.1072,\\n         0.1430,  0.1486,  0.1937,  0.1190, -0.0286,  0.3577,  0.1485,  0.2513,\\n         0.0349, -0.3535, -0.2056, -0.0304, -0.0794, -0.0041, -0.0258,  0.2603,\\n        -0.1816, -0.0454, -0.3824,  0.2506,  0.2699, -0.3127, -0.0323, -0.2255,\\n        -0.0359, -0.1792,  0.0718,  0.0685, -0.1484,  0.2785,  0.4688,  0.2025,\\n         0.3250, -0.0613,  0.1821,  0.1761,  0.1739,  0.3699,  0.1431,  0.1635],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([[-0.0058, -0.0687,  0.0026,  ...,  0.0159,  0.0443,  0.0803],\\n        [-0.0143, -0.0973, -0.0717,  ...,  0.1158, -0.0156,  0.0377],\\n        [ 0.0002,  0.0028, -0.0267,  ...,  0.1245, -0.1107, -0.0250],\\n        ...,\\n        [ 0.1183, -0.0308, -0.0336,  ..., -0.0560, -0.0184,  0.0255],\\n        [-0.1013,  0.0334, -0.0829,  ..., -0.0439,  0.0499,  0.0882],\\n        [ 0.0140, -0.0729, -0.1180,  ...,  0.0094, -0.0186, -0.0606]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([ 0.0165,  0.0638,  0.0902,  0.0887,  0.0588,  0.0455,  0.0981, -0.1062,\\n         0.0621,  0.0158,  0.0826,  0.1044,  0.0893, -0.1074,  0.0187,  0.0989,\\n         0.0277,  0.0139,  0.0315,  0.0909,  0.1041,  0.0326,  0.1152,  0.0651,\\n        -0.0394,  0.0345, -0.0092, -0.0180,  0.0502,  0.0350,  0.1038, -0.1070,\\n         0.0312,  0.0032,  0.0442, -0.0209,  0.0738,  0.0105, -0.1052,  0.0069,\\n         0.0511,  0.0161, -0.0315, -0.0044, -0.0986,  0.0101, -0.1125,  0.0076,\\n        -0.0612,  0.0102,  0.0062,  0.0374, -0.0753,  0.0420, -0.1098,  0.1210,\\n         0.0841, -0.0561,  0.1224,  0.0989, -0.0718, -0.1233, -0.0584,  0.0934],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([[-0.0655,  0.0324,  0.0444,  0.0208, -0.0952,  0.0866, -0.0996, -0.0474,\\n          0.0116,  0.0831, -0.1167, -0.1215, -0.0396, -0.0006,  0.0247, -0.0174,\\n         -0.0548,  0.0890, -0.0712,  0.0921,  0.0142, -0.0914,  0.0640, -0.0318,\\n          0.0379,  0.0714,  0.0656, -0.0088, -0.0066,  0.0105, -0.0930, -0.0730,\\n         -0.0711, -0.1168, -0.0178, -0.0392, -0.0931,  0.0290,  0.0886,  0.1189,\\n          0.0631, -0.0114, -0.0801, -0.0705, -0.0045,  0.1113, -0.0708, -0.0737,\\n          0.0699, -0.0374,  0.0489,  0.1097, -0.0880, -0.0667,  0.0874,  0.0746,\\n         -0.1231,  0.0641,  0.1023,  0.1056,  0.0544,  0.0792, -0.1057, -0.0464]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([-0.0813], requires_grad=True)\"\n",
      "                            ],\n",
      "                            \"weight_decay\":\t0\n",
      "                        }\n",
      "                    ],\n",
      "                    \"state\":\t{}\n",
      "                }\n",
      "            }\n",
      "        }\n",
      "    },\n",
      "    \"steps_per_epoch\":\t4000,\n",
      "    \"train_v_iters\":\t80,\n",
      "    \"vf_lr\":\t0.001\n",
      "}\n",
      "\u001b[32;1m\n",
      "Number of parameters: \t pi: 4610, \t v: 4545\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: trajectory cut off by epoch at 8 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               0 |\n",
      "|      AverageEpRet |            19.5 |\n",
      "|          StdEpRet |            9.11 |\n",
      "|          MaxEpRet |              63 |\n",
      "|          MinEpRet |               8 |\n",
      "|             EpLen |            19.5 |\n",
      "|      AverageVVals |         -0.0858 |\n",
      "|          StdVVals |          0.0799 |\n",
      "|          MaxVVals |           0.124 |\n",
      "|          MinVVals |          -0.293 |\n",
      "| TotalEnvInteracts |           4e+03 |\n",
      "|            LossPi |         0.00878 |\n",
      "|             LossV |             194 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |            -117 |\n",
      "|           Entropy |           0.683 |\n",
      "|                KL |       -2.83e-10 |\n",
      "|              Time |            1.44 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 9 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               1 |\n",
      "|      AverageEpRet |            21.9 |\n",
      "|          StdEpRet |            11.8 |\n",
      "|          MaxEpRet |              73 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            21.9 |\n",
      "|      AverageVVals |             7.5 |\n",
      "|          StdVVals |           0.316 |\n",
      "|          MaxVVals |            7.88 |\n",
      "|          MinVVals |            5.71 |\n",
      "| TotalEnvInteracts |           8e+03 |\n",
      "|            LossPi |          0.0108 |\n",
      "|             LossV |             121 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -44.6 |\n",
      "|           Entropy |           0.684 |\n",
      "|                KL |        9.98e-10 |\n",
      "|              Time |            2.97 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 5 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               2 |\n",
      "|      AverageEpRet |            21.7 |\n",
      "|          StdEpRet |            12.5 |\n",
      "|          MaxEpRet |              71 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            21.7 |\n",
      "|      AverageVVals |            11.4 |\n",
      "|          StdVVals |            3.73 |\n",
      "|          MaxVVals |            13.8 |\n",
      "|          MinVVals |           -8.53 |\n",
      "| TotalEnvInteracts |         1.2e+04 |\n",
      "|            LossPi |          0.0131 |\n",
      "|             LossV |            84.9 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -7.34 |\n",
      "|           Entropy |           0.685 |\n",
      "|                KL |        9.24e-10 |\n",
      "|              Time |            4.38 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 2 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               3 |\n",
      "|      AverageEpRet |            21.8 |\n",
      "|          StdEpRet |              13 |\n",
      "|          MaxEpRet |             134 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            21.8 |\n",
      "|      AverageVVals |            12.8 |\n",
      "|          StdVVals |            4.22 |\n",
      "|          MaxVVals |            15.7 |\n",
      "|          MinVVals |            -5.5 |\n",
      "| TotalEnvInteracts |         1.6e+04 |\n",
      "|            LossPi |          0.0113 |\n",
      "|             LossV |             101 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -1.99 |\n",
      "|           Entropy |           0.687 |\n",
      "|                KL |        1.58e-09 |\n",
      "|              Time |            5.81 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 9 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               4 |\n",
      "|      AverageEpRet |            20.7 |\n",
      "|          StdEpRet |            10.1 |\n",
      "|          MaxEpRet |              61 |\n",
      "|          MinEpRet |               8 |\n",
      "|             EpLen |            20.7 |\n",
      "|      AverageVVals |            12.9 |\n",
      "|          StdVVals |             4.6 |\n",
      "|          MaxVVals |            16.4 |\n",
      "|          MinVVals |           -2.19 |\n",
      "| TotalEnvInteracts |           2e+04 |\n",
      "|            LossPi |         0.00993 |\n",
      "|             LossV |            52.9 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -1.74 |\n",
      "|           Entropy |           0.687 |\n",
      "|                KL |       -5.66e-10 |\n",
      "|              Time |            7.21 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 14 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               5 |\n",
      "|      AverageEpRet |            21.1 |\n",
      "|          StdEpRet |            9.39 |\n",
      "|          MaxEpRet |              66 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            21.1 |\n",
      "|      AverageVVals |            12.6 |\n",
      "|          StdVVals |            4.49 |\n",
      "|          MaxVVals |            15.9 |\n",
      "|          MinVVals |           -2.57 |\n",
      "| TotalEnvInteracts |         2.4e+04 |\n",
      "|            LossPi |         0.00507 |\n",
      "|             LossV |              51 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -1.33 |\n",
      "|           Entropy |           0.688 |\n",
      "|                KL |        6.26e-10 |\n",
      "|              Time |            8.61 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 34 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               6 |\n",
      "|      AverageEpRet |            23.6 |\n",
      "|          StdEpRet |            14.6 |\n",
      "|          MaxEpRet |             101 |\n",
      "|          MinEpRet |               8 |\n",
      "|             EpLen |            23.6 |\n",
      "|      AverageVVals |            11.9 |\n",
      "|          StdVVals |            4.02 |\n",
      "|          MaxVVals |              15 |\n",
      "|          MinVVals |            -4.8 |\n",
      "| TotalEnvInteracts |         2.8e+04 |\n",
      "|            LossPi |         0.00235 |\n",
      "|             LossV |             112 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -11.2 |\n",
      "|           Entropy |           0.689 |\n",
      "|                KL |       -6.33e-10 |\n",
      "|              Time |              10 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 4 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               7 |\n",
      "|      AverageEpRet |            23.6 |\n",
      "|          StdEpRet |            11.4 |\n",
      "|          MaxEpRet |              70 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            23.6 |\n",
      "|      AverageVVals |            14.9 |\n",
      "|          StdVVals |            5.12 |\n",
      "|          MaxVVals |            18.1 |\n",
      "|          MinVVals |            -2.7 |\n",
      "| TotalEnvInteracts |         3.2e+04 |\n",
      "|            LossPi |       -0.000345 |\n",
      "|             LossV |            69.3 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -2.74 |\n",
      "|           Entropy |           0.688 |\n",
      "|                KL |        4.54e-10 |\n",
      "|              Time |            11.5 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 17 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               8 |\n",
      "|      AverageEpRet |            23.6 |\n",
      "|          StdEpRet |            12.9 |\n",
      "|          MaxEpRet |              79 |\n",
      "|          MinEpRet |               8 |\n",
      "|             EpLen |            23.6 |\n",
      "|      AverageVVals |            13.5 |\n",
      "|          StdVVals |             4.8 |\n",
      "|          MaxVVals |            16.9 |\n",
      "|          MinVVals |           -3.94 |\n",
      "| TotalEnvInteracts |         3.6e+04 |\n",
      "|            LossPi |        -0.00249 |\n",
      "|             LossV |              79 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -6.03 |\n",
      "|           Entropy |           0.689 |\n",
      "|                KL |        4.25e-10 |\n",
      "|              Time |            12.9 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 17 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               9 |\n",
      "|      AverageEpRet |            23.9 |\n",
      "|          StdEpRet |            13.2 |\n",
      "|          MaxEpRet |              78 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            23.9 |\n",
      "|      AverageVVals |            13.8 |\n",
      "|          StdVVals |            4.75 |\n",
      "|          MaxVVals |            17.9 |\n",
      "|          MinVVals |           -2.34 |\n",
      "| TotalEnvInteracts |           4e+04 |\n",
      "|            LossPi |        -0.00663 |\n",
      "|             LossV |            78.2 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -6.27 |\n",
      "|           Entropy |           0.688 |\n",
      "|                KL |       -8.34e-10 |\n",
      "|              Time |            14.3 |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: trajectory cut off by epoch at 4 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              10 |\n",
      "|      AverageEpRet |            23.6 |\n",
      "|          StdEpRet |            12.4 |\n",
      "|          MaxEpRet |              73 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            23.6 |\n",
      "|      AverageVVals |            13.5 |\n",
      "|          StdVVals |            5.67 |\n",
      "|          MaxVVals |            19.7 |\n",
      "|          MinVVals |          -0.355 |\n",
      "| TotalEnvInteracts |         4.4e+04 |\n",
      "|            LossPi |        -0.00813 |\n",
      "|             LossV |            62.3 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -3.54 |\n",
      "|           Entropy |           0.688 |\n",
      "|                KL |       -6.11e-10 |\n",
      "|              Time |            15.7 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 2 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              11 |\n",
      "|      AverageEpRet |            23.9 |\n",
      "|          StdEpRet |            13.1 |\n",
      "|          MaxEpRet |              76 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            23.9 |\n",
      "|      AverageVVals |            14.2 |\n",
      "|          StdVVals |             5.7 |\n",
      "|          MaxVVals |            20.2 |\n",
      "|          MinVVals |          -0.166 |\n",
      "| TotalEnvInteracts |         4.8e+04 |\n",
      "|            LossPi |         -0.0127 |\n",
      "|             LossV |            68.2 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -4.09 |\n",
      "|           Entropy |           0.687 |\n",
      "|                KL |       -5.07e-10 |\n",
      "|              Time |            17.1 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 10 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              12 |\n",
      "|      AverageEpRet |            24.6 |\n",
      "|          StdEpRet |            14.1 |\n",
      "|          MaxEpRet |             110 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            24.6 |\n",
      "|      AverageVVals |            14.3 |\n",
      "|          StdVVals |            6.07 |\n",
      "|          MaxVVals |            20.4 |\n",
      "|          MinVVals |           0.277 |\n",
      "| TotalEnvInteracts |         5.2e+04 |\n",
      "|            LossPi |         -0.0154 |\n",
      "|             LossV |            77.7 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -3.23 |\n",
      "|           Entropy |           0.687 |\n",
      "|                KL |       -7.23e-10 |\n",
      "|              Time |            18.5 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 47 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              13 |\n",
      "|      AverageEpRet |            27.5 |\n",
      "|          StdEpRet |            14.3 |\n",
      "|          MaxEpRet |              80 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            27.5 |\n",
      "|      AverageVVals |              15 |\n",
      "|          StdVVals |            6.26 |\n",
      "|          MaxVVals |            21.6 |\n",
      "|          MinVVals |           0.181 |\n",
      "| TotalEnvInteracts |         5.6e+04 |\n",
      "|            LossPi |         -0.0169 |\n",
      "|             LossV |            77.1 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -2.56 |\n",
      "|           Entropy |           0.687 |\n",
      "|                KL |       -1.71e-09 |\n",
      "|              Time |              20 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 24 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              14 |\n",
      "|      AverageEpRet |              25 |\n",
      "|          StdEpRet |            13.7 |\n",
      "|          MaxEpRet |              79 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |              25 |\n",
      "|      AverageVVals |            15.2 |\n",
      "|          StdVVals |            6.61 |\n",
      "|          MaxVVals |            22.7 |\n",
      "|          MinVVals |          -0.337 |\n",
      "| TotalEnvInteracts |           6e+04 |\n",
      "|            LossPi |         -0.0213 |\n",
      "|             LossV |            71.9 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -1.08 |\n",
      "|           Entropy |           0.685 |\n",
      "|                KL |        -1.1e-09 |\n",
      "|              Time |            21.4 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              15 |\n",
      "|      AverageEpRet |            26.7 |\n",
      "|          StdEpRet |            14.7 |\n",
      "|          MaxEpRet |             110 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            26.7 |\n",
      "|      AverageVVals |            15.5 |\n",
      "|          StdVVals |            6.42 |\n",
      "|          MaxVVals |            22.4 |\n",
      "|          MinVVals |         -0.0419 |\n",
      "| TotalEnvInteracts |         6.4e+04 |\n",
      "|            LossPi |         -0.0236 |\n",
      "|             LossV |            82.5 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -2.26 |\n",
      "|           Entropy |           0.684 |\n",
      "|                KL |       -1.25e-09 |\n",
      "|              Time |            22.8 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 11 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              16 |\n",
      "|      AverageEpRet |            26.8 |\n",
      "|          StdEpRet |            15.7 |\n",
      "|          MaxEpRet |              81 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            26.8 |\n",
      "|      AverageVVals |            15.3 |\n",
      "|          StdVVals |            6.84 |\n",
      "|          MaxVVals |            22.7 |\n",
      "|          MinVVals |           0.199 |\n",
      "| TotalEnvInteracts |         6.8e+04 |\n",
      "|            LossPi |         -0.0238 |\n",
      "|             LossV |            92.9 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -11.2 |\n",
      "|           Entropy |           0.684 |\n",
      "|                KL |        1.14e-09 |\n",
      "|              Time |            24.2 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 9 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              17 |\n",
      "|      AverageEpRet |            27.9 |\n",
      "|          StdEpRet |            14.5 |\n",
      "|          MaxEpRet |              82 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            27.9 |\n",
      "|      AverageVVals |            16.1 |\n",
      "|          StdVVals |            7.04 |\n",
      "|          MaxVVals |            23.8 |\n",
      "|          MinVVals |            1.08 |\n",
      "| TotalEnvInteracts |         7.2e+04 |\n",
      "|            LossPi |         -0.0279 |\n",
      "|             LossV |            71.3 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |            -5.1 |\n",
      "|           Entropy |           0.682 |\n",
      "|                KL |        2.61e-10 |\n",
      "|              Time |            25.7 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 5 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              18 |\n",
      "|      AverageEpRet |            30.7 |\n",
      "|          StdEpRet |            18.6 |\n",
      "|          MaxEpRet |             100 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            30.7 |\n",
      "|      AverageVVals |            15.9 |\n",
      "|          StdVVals |            7.12 |\n",
      "|          MaxVVals |            23.8 |\n",
      "|          MinVVals |           0.717 |\n",
      "| TotalEnvInteracts |         7.6e+04 |\n",
      "|            LossPi |         -0.0279 |\n",
      "|             LossV |             115 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -8.62 |\n",
      "|           Entropy |           0.681 |\n",
      "|                KL |        3.43e-10 |\n",
      "|              Time |            27.2 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 38 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              19 |\n",
      "|      AverageEpRet |            28.7 |\n",
      "|          StdEpRet |            15.6 |\n",
      "|          MaxEpRet |              85 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            28.7 |\n",
      "|      AverageVVals |            18.1 |\n",
      "|          StdVVals |            7.79 |\n",
      "|          MaxVVals |              26 |\n",
      "|          MinVVals |           0.341 |\n",
      "| TotalEnvInteracts |           8e+04 |\n",
      "|            LossPi |         -0.0288 |\n",
      "|             LossV |              90 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -4.11 |\n",
      "|           Entropy |            0.68 |\n",
      "|                KL |       -1.29e-09 |\n",
      "|              Time |            28.6 |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: trajectory cut off by epoch at 18 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              20 |\n",
      "|      AverageEpRet |            31.1 |\n",
      "|          StdEpRet |            20.9 |\n",
      "|          MaxEpRet |             177 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            31.1 |\n",
      "|      AverageVVals |            16.7 |\n",
      "|          StdVVals |            7.15 |\n",
      "|          MaxVVals |            25.2 |\n",
      "|          MinVVals |            0.41 |\n",
      "| TotalEnvInteracts |         8.4e+04 |\n",
      "|            LossPi |         -0.0288 |\n",
      "|             LossV |             166 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -13.3 |\n",
      "|           Entropy |           0.679 |\n",
      "|                KL |       -2.61e-10 |\n",
      "|              Time |              30 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 10 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              21 |\n",
      "|      AverageEpRet |              30 |\n",
      "|          StdEpRet |              17 |\n",
      "|          MaxEpRet |              96 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |              30 |\n",
      "|      AverageVVals |            18.4 |\n",
      "|          StdVVals |            7.61 |\n",
      "|          MaxVVals |            27.1 |\n",
      "|          MinVVals |           0.246 |\n",
      "| TotalEnvInteracts |         8.8e+04 |\n",
      "|            LossPi |         -0.0343 |\n",
      "|             LossV |            95.2 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -5.73 |\n",
      "|           Entropy |           0.677 |\n",
      "|                KL |       -2.24e-10 |\n",
      "|              Time |            31.4 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 21 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              22 |\n",
      "|      AverageEpRet |            29.7 |\n",
      "|          StdEpRet |            16.4 |\n",
      "|          MaxEpRet |             124 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            29.7 |\n",
      "|      AverageVVals |            17.4 |\n",
      "|          StdVVals |            7.62 |\n",
      "|          MaxVVals |            25.6 |\n",
      "|          MinVVals |           0.335 |\n",
      "| TotalEnvInteracts |         9.2e+04 |\n",
      "|            LossPi |          -0.033 |\n",
      "|             LossV |            95.7 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -3.68 |\n",
      "|           Entropy |           0.677 |\n",
      "|                KL |        8.49e-10 |\n",
      "|              Time |            32.9 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 56 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              23 |\n",
      "|      AverageEpRet |            32.1 |\n",
      "|          StdEpRet |            17.4 |\n",
      "|          MaxEpRet |              87 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            32.1 |\n",
      "|      AverageVVals |            17.4 |\n",
      "|          StdVVals |            7.45 |\n",
      "|          MaxVVals |            25.7 |\n",
      "|          MinVVals |           0.551 |\n",
      "| TotalEnvInteracts |         9.6e+04 |\n",
      "|            LossPi |         -0.0333 |\n",
      "|             LossV |             103 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |            -3.9 |\n",
      "|           Entropy |           0.676 |\n",
      "|                KL |       -8.34e-10 |\n",
      "|              Time |            34.3 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 24 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              24 |\n",
      "|      AverageEpRet |            32.1 |\n",
      "|          StdEpRet |              19 |\n",
      "|          MaxEpRet |             118 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            32.1 |\n",
      "|      AverageVVals |            18.6 |\n",
      "|          StdVVals |            7.76 |\n",
      "|          MaxVVals |            26.1 |\n",
      "|          MinVVals |          -0.142 |\n",
      "| TotalEnvInteracts |           1e+05 |\n",
      "|            LossPi |         -0.0383 |\n",
      "|             LossV |             111 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -1.39 |\n",
      "|           Entropy |           0.674 |\n",
      "|                KL |        4.69e-10 |\n",
      "|              Time |            35.7 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 10 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              25 |\n",
      "|      AverageEpRet |            31.9 |\n",
      "|          StdEpRet |            19.7 |\n",
      "|          MaxEpRet |             116 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            31.9 |\n",
      "|      AverageVVals |            18.2 |\n",
      "|          StdVVals |            8.35 |\n",
      "|          MaxVVals |              27 |\n",
      "|          MinVVals |           0.053 |\n",
      "| TotalEnvInteracts |        1.04e+05 |\n",
      "|            LossPi |          -0.039 |\n",
      "|             LossV |             130 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -6.74 |\n",
      "|           Entropy |           0.672 |\n",
      "|                KL |       -2.31e-10 |\n",
      "|              Time |            37.1 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 5 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              26 |\n",
      "|      AverageEpRet |            34.1 |\n",
      "|          StdEpRet |            17.6 |\n",
      "|          MaxEpRet |             104 |\n",
      "|          MinEpRet |              12 |\n",
      "|             EpLen |            34.1 |\n",
      "|      AverageVVals |            19.2 |\n",
      "|          StdVVals |            7.94 |\n",
      "|          MaxVVals |            27.4 |\n",
      "|          MinVVals |           0.301 |\n",
      "| TotalEnvInteracts |        1.08e+05 |\n",
      "|            LossPi |         -0.0373 |\n",
      "|             LossV |             104 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -6.61 |\n",
      "|           Entropy |           0.672 |\n",
      "|                KL |         8.2e-10 |\n",
      "|              Time |            38.5 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 32 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              27 |\n",
      "|      AverageEpRet |            36.7 |\n",
      "|          StdEpRet |            28.3 |\n",
      "|          MaxEpRet |             179 |\n",
      "|          MinEpRet |              11 |\n",
      "|             EpLen |            36.7 |\n",
      "|      AverageVVals |            18.1 |\n",
      "|          StdVVals |            8.04 |\n",
      "|          MaxVVals |            27.6 |\n",
      "|          MinVVals |           0.911 |\n",
      "| TotalEnvInteracts |        1.12e+05 |\n",
      "|            LossPi |         -0.0325 |\n",
      "|             LossV |             260 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -54.9 |\n",
      "|           Entropy |           0.671 |\n",
      "|                KL |        4.62e-10 |\n",
      "|              Time |            39.9 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 26 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              28 |\n",
      "|      AverageEpRet |              39 |\n",
      "|          StdEpRet |            25.6 |\n",
      "|          MaxEpRet |             131 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |              39 |\n",
      "|      AverageVVals |            23.2 |\n",
      "|          StdVVals |            9.61 |\n",
      "|          MaxVVals |            34.2 |\n",
      "|          MinVVals |           0.518 |\n",
      "| TotalEnvInteracts |        1.16e+05 |\n",
      "|            LossPi |          -0.033 |\n",
      "|             LossV |             181 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -8.86 |\n",
      "|           Entropy |            0.67 |\n",
      "|                KL |        7.23e-10 |\n",
      "|              Time |            41.4 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 56 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              29 |\n",
      "|      AverageEpRet |            35.2 |\n",
      "|          StdEpRet |            19.6 |\n",
      "|          MaxEpRet |              91 |\n",
      "|          MinEpRet |              11 |\n",
      "|             EpLen |            35.2 |\n",
      "|      AverageVVals |              22 |\n",
      "|          StdVVals |            9.61 |\n",
      "|          MaxVVals |              33 |\n",
      "|          MinVVals |           0.214 |\n",
      "| TotalEnvInteracts |         1.2e+05 |\n",
      "|            LossPi |         -0.0398 |\n",
      "|             LossV |             126 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -14.3 |\n",
      "|           Entropy |           0.669 |\n",
      "|                KL |       -6.18e-10 |\n",
      "|              Time |            42.8 |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: trajectory cut off by epoch at 78 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              30 |\n",
      "|      AverageEpRet |              37 |\n",
      "|          StdEpRet |            21.4 |\n",
      "|          MaxEpRet |             112 |\n",
      "|          MinEpRet |              12 |\n",
      "|             EpLen |              37 |\n",
      "|      AverageVVals |              20 |\n",
      "|          StdVVals |            8.51 |\n",
      "|          MaxVVals |            30.8 |\n",
      "|          MinVVals |           0.971 |\n",
      "| TotalEnvInteracts |        1.24e+05 |\n",
      "|            LossPi |         -0.0386 |\n",
      "|             LossV |             152 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |            -6.8 |\n",
      "|           Entropy |           0.667 |\n",
      "|                KL |       -1.83e-09 |\n",
      "|              Time |            44.2 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 26 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              31 |\n",
      "|      AverageEpRet |            37.1 |\n",
      "|          StdEpRet |            18.4 |\n",
      "|          MaxEpRet |             106 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            37.1 |\n",
      "|      AverageVVals |            21.8 |\n",
      "|          StdVVals |            8.85 |\n",
      "|          MaxVVals |            30.9 |\n",
      "|          MinVVals |           0.901 |\n",
      "| TotalEnvInteracts |        1.28e+05 |\n",
      "|            LossPi |          -0.047 |\n",
      "|             LossV |             108 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -7.39 |\n",
      "|           Entropy |           0.663 |\n",
      "|                KL |       -7.15e-10 |\n",
      "|              Time |            45.6 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 8 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              32 |\n",
      "|      AverageEpRet |            39.1 |\n",
      "|          StdEpRet |            25.8 |\n",
      "|          MaxEpRet |             129 |\n",
      "|          MinEpRet |              12 |\n",
      "|             EpLen |            39.1 |\n",
      "|      AverageVVals |            19.5 |\n",
      "|          StdVVals |            8.05 |\n",
      "|          MaxVVals |            28.2 |\n",
      "|          MinVVals |           0.305 |\n",
      "| TotalEnvInteracts |        1.32e+05 |\n",
      "|            LossPi |         -0.0405 |\n",
      "|             LossV |             193 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -18.2 |\n",
      "|           Entropy |           0.664 |\n",
      "|                KL |        5.22e-11 |\n",
      "|              Time |              47 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 20 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              33 |\n",
      "|      AverageEpRet |            35.9 |\n",
      "|          StdEpRet |            18.5 |\n",
      "|          MaxEpRet |              88 |\n",
      "|          MinEpRet |              11 |\n",
      "|             EpLen |            35.9 |\n",
      "|      AverageVVals |              23 |\n",
      "|          StdVVals |            9.32 |\n",
      "|          MaxVVals |            33.1 |\n",
      "|          MinVVals |             0.6 |\n",
      "| TotalEnvInteracts |        1.36e+05 |\n",
      "|            LossPi |         -0.0475 |\n",
      "|             LossV |             113 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |             -18 |\n",
      "|           Entropy |           0.662 |\n",
      "|                KL |       -1.27e-09 |\n",
      "|              Time |            48.5 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 58 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              34 |\n",
      "|      AverageEpRet |            41.1 |\n",
      "|          StdEpRet |              26 |\n",
      "|          MaxEpRet |             139 |\n",
      "|          MinEpRet |              12 |\n",
      "|             EpLen |            41.1 |\n",
      "|      AverageVVals |            19.9 |\n",
      "|          StdVVals |            8.24 |\n",
      "|          MaxVVals |            30.3 |\n",
      "|          MinVVals |            1.58 |\n",
      "| TotalEnvInteracts |         1.4e+05 |\n",
      "|            LossPi |         -0.0405 |\n",
      "|             LossV |             181 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -20.2 |\n",
      "|           Entropy |           0.663 |\n",
      "|                KL |         4.4e-10 |\n",
      "|              Time |            49.9 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 30 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              35 |\n",
      "|      AverageEpRet |            38.2 |\n",
      "|          StdEpRet |            23.2 |\n",
      "|          MaxEpRet |             171 |\n",
      "|          MinEpRet |              12 |\n",
      "|             EpLen |            38.2 |\n",
      "|      AverageVVals |            23.7 |\n",
      "|          StdVVals |              10 |\n",
      "|          MaxVVals |              34 |\n",
      "|          MinVVals |           0.994 |\n",
      "| TotalEnvInteracts |        1.44e+05 |\n",
      "|            LossPi |         -0.0496 |\n",
      "|             LossV |             164 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -19.5 |\n",
      "|           Entropy |           0.659 |\n",
      "|                KL |       -7.08e-10 |\n",
      "|              Time |            51.3 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 16 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              36 |\n",
      "|      AverageEpRet |            43.8 |\n",
      "|          StdEpRet |            31.2 |\n",
      "|          MaxEpRet |             179 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            43.8 |\n",
      "|      AverageVVals |            20.8 |\n",
      "|          StdVVals |            9.82 |\n",
      "|          MaxVVals |            33.8 |\n",
      "|          MinVVals |           -3.94 |\n",
      "| TotalEnvInteracts |        1.48e+05 |\n",
      "|            LossPi |         -0.0378 |\n",
      "|             LossV |             294 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -58.7 |\n",
      "|           Entropy |           0.659 |\n",
      "|                KL |        -1.1e-09 |\n",
      "|              Time |            52.8 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 33 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              37 |\n",
      "|      AverageEpRet |            49.6 |\n",
      "|          StdEpRet |            31.1 |\n",
      "|          MaxEpRet |             162 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            49.6 |\n",
      "|      AverageVVals |            26.1 |\n",
      "|          StdVVals |            9.63 |\n",
      "|          MaxVVals |            35.8 |\n",
      "|          MinVVals |           0.713 |\n",
      "| TotalEnvInteracts |        1.52e+05 |\n",
      "|            LossPi |         -0.0384 |\n",
      "|             LossV |             219 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -10.9 |\n",
      "|           Entropy |            0.66 |\n",
      "|                KL |        2.09e-10 |\n",
      "|              Time |            54.2 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 72 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              38 |\n",
      "|      AverageEpRet |            42.2 |\n",
      "|          StdEpRet |            25.1 |\n",
      "|          MaxEpRet |             149 |\n",
      "|          MinEpRet |              12 |\n",
      "|             EpLen |            42.2 |\n",
      "|      AverageVVals |            26.8 |\n",
      "|          StdVVals |            10.6 |\n",
      "|          MaxVVals |            37.8 |\n",
      "|          MinVVals |            1.15 |\n",
      "| TotalEnvInteracts |        1.56e+05 |\n",
      "|            LossPi |           -0.04 |\n",
      "|             LossV |             203 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -26.6 |\n",
      "|           Entropy |           0.656 |\n",
      "|                KL |       -3.35e-10 |\n",
      "|              Time |            55.6 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 6 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              39 |\n",
      "|      AverageEpRet |              42 |\n",
      "|          StdEpRet |            23.8 |\n",
      "|          MaxEpRet |             115 |\n",
      "|          MinEpRet |              11 |\n",
      "|             EpLen |              42 |\n",
      "|      AverageVVals |            23.3 |\n",
      "|          StdVVals |             9.1 |\n",
      "|          MaxVVals |            35.1 |\n",
      "|          MinVVals |           0.284 |\n",
      "| TotalEnvInteracts |         1.6e+05 |\n",
      "|            LossPi |         -0.0431 |\n",
      "|             LossV |             160 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -7.15 |\n",
      "|           Entropy |           0.654 |\n",
      "|                KL |       -1.12e-10 |\n",
      "|              Time |            57.1 |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: trajectory cut off by epoch at 12 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              40 |\n",
      "|      AverageEpRet |            42.4 |\n",
      "|          StdEpRet |            21.2 |\n",
      "|          MaxEpRet |              97 |\n",
      "|          MinEpRet |              13 |\n",
      "|             EpLen |            42.4 |\n",
      "|      AverageVVals |              23 |\n",
      "|          StdVVals |            9.14 |\n",
      "|          MaxVVals |            33.3 |\n",
      "|          MinVVals |          -0.158 |\n",
      "| TotalEnvInteracts |        1.64e+05 |\n",
      "|            LossPi |         -0.0495 |\n",
      "|             LossV |             114 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -5.76 |\n",
      "|           Entropy |           0.654 |\n",
      "|                KL |        -1.2e-09 |\n",
      "|              Time |            58.5 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 29 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              41 |\n",
      "|      AverageEpRet |            42.7 |\n",
      "|          StdEpRet |            20.3 |\n",
      "|          MaxEpRet |             108 |\n",
      "|          MinEpRet |              13 |\n",
      "|             EpLen |            42.7 |\n",
      "|      AverageVVals |            22.7 |\n",
      "|          StdVVals |            9.76 |\n",
      "|          MaxVVals |            33.5 |\n",
      "|          MinVVals |           0.923 |\n",
      "| TotalEnvInteracts |        1.68e+05 |\n",
      "|            LossPi |         -0.0474 |\n",
      "|             LossV |             103 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -7.78 |\n",
      "|           Entropy |           0.653 |\n",
      "|                KL |        8.79e-10 |\n",
      "|              Time |            59.9 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 33 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              42 |\n",
      "|      AverageEpRet |              49 |\n",
      "|          StdEpRet |            24.2 |\n",
      "|          MaxEpRet |             133 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |              49 |\n",
      "|      AverageVVals |            22.3 |\n",
      "|          StdVVals |            9.79 |\n",
      "|          MaxVVals |            35.3 |\n",
      "|          MinVVals |           0.387 |\n",
      "| TotalEnvInteracts |        1.72e+05 |\n",
      "|            LossPi |         -0.0416 |\n",
      "|             LossV |             153 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -14.5 |\n",
      "|           Entropy |           0.653 |\n",
      "|                KL |        1.07e-09 |\n",
      "|              Time |            61.3 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 20 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              43 |\n",
      "|      AverageEpRet |            43.7 |\n",
      "|          StdEpRet |              23 |\n",
      "|          MaxEpRet |             138 |\n",
      "|          MinEpRet |              14 |\n",
      "|             EpLen |            43.7 |\n",
      "|      AverageVVals |            24.4 |\n",
      "|          StdVVals |            10.3 |\n",
      "|          MaxVVals |            35.6 |\n",
      "|          MinVVals |          -0.317 |\n",
      "| TotalEnvInteracts |        1.76e+05 |\n",
      "|            LossPi |         -0.0497 |\n",
      "|             LossV |             143 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -8.36 |\n",
      "|           Entropy |           0.649 |\n",
      "|                KL |        2.53e-10 |\n",
      "|              Time |            62.7 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 42 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              44 |\n",
      "|      AverageEpRet |              44 |\n",
      "|          StdEpRet |            21.3 |\n",
      "|          MaxEpRet |             103 |\n",
      "|          MinEpRet |              14 |\n",
      "|             EpLen |              44 |\n",
      "|      AverageVVals |            22.8 |\n",
      "|          StdVVals |            10.1 |\n",
      "|          MaxVVals |            33.9 |\n",
      "|          MinVVals |          -0.832 |\n",
      "| TotalEnvInteracts |         1.8e+05 |\n",
      "|            LossPi |         -0.0473 |\n",
      "|             LossV |             117 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -7.41 |\n",
      "|           Entropy |            0.65 |\n",
      "|                KL |       -1.21e-09 |\n",
      "|              Time |            64.1 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              45 |\n",
      "|      AverageEpRet |            43.5 |\n",
      "|          StdEpRet |            23.5 |\n",
      "|          MaxEpRet |             108 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            43.5 |\n",
      "|      AverageVVals |            22.2 |\n",
      "|          StdVVals |             9.9 |\n",
      "|          MaxVVals |            33.9 |\n",
      "|          MinVVals |          -0.413 |\n",
      "| TotalEnvInteracts |        1.84e+05 |\n",
      "|            LossPi |         -0.0482 |\n",
      "|             LossV |             125 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -5.28 |\n",
      "|           Entropy |            0.65 |\n",
      "|                KL |        9.98e-10 |\n",
      "|              Time |            65.5 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 71 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              46 |\n",
      "|      AverageEpRet |            49.1 |\n",
      "|          StdEpRet |            24.1 |\n",
      "|          MaxEpRet |             119 |\n",
      "|          MinEpRet |              14 |\n",
      "|             EpLen |            49.1 |\n",
      "|      AverageVVals |            24.4 |\n",
      "|          StdVVals |            10.6 |\n",
      "|          MaxVVals |            36.2 |\n",
      "|          MinVVals |           0.566 |\n",
      "| TotalEnvInteracts |        1.88e+05 |\n",
      "|            LossPi |          -0.048 |\n",
      "|             LossV |             130 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -4.37 |\n",
      "|           Entropy |           0.651 |\n",
      "|                KL |        1.12e-10 |\n",
      "|              Time |            66.9 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 40 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              47 |\n",
      "|      AverageEpRet |            40.4 |\n",
      "|          StdEpRet |            22.1 |\n",
      "|          MaxEpRet |             109 |\n",
      "|          MinEpRet |              11 |\n",
      "|             EpLen |            40.4 |\n",
      "|      AverageVVals |            25.1 |\n",
      "|          StdVVals |            11.1 |\n",
      "|          MaxVVals |            38.2 |\n",
      "|          MinVVals |          -0.879 |\n",
      "| TotalEnvInteracts |        1.92e+05 |\n",
      "|            LossPi |         -0.0639 |\n",
      "|             LossV |             137 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -16.7 |\n",
      "|           Entropy |           0.641 |\n",
      "|                KL |       -6.18e-10 |\n",
      "|              Time |            68.3 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 64 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              48 |\n",
      "|      AverageEpRet |            51.1 |\n",
      "|          StdEpRet |            25.4 |\n",
      "|          MaxEpRet |             163 |\n",
      "|          MinEpRet |              18 |\n",
      "|             EpLen |            51.1 |\n",
      "|      AverageVVals |            22.7 |\n",
      "|          StdVVals |            9.14 |\n",
      "|          MaxVVals |            33.9 |\n",
      "|          MinVVals |          -0.359 |\n",
      "| TotalEnvInteracts |        1.96e+05 |\n",
      "|            LossPi |         -0.0507 |\n",
      "|             LossV |             167 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -25.2 |\n",
      "|           Entropy |           0.646 |\n",
      "|                KL |       -1.49e-10 |\n",
      "|              Time |            69.7 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 37 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              49 |\n",
      "|      AverageEpRet |            47.7 |\n",
      "|          StdEpRet |            29.9 |\n",
      "|          MaxEpRet |             177 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            47.7 |\n",
      "|      AverageVVals |            25.5 |\n",
      "|          StdVVals |            11.2 |\n",
      "|          MaxVVals |            37.5 |\n",
      "|          MinVVals |            0.16 |\n",
      "| TotalEnvInteracts |           2e+05 |\n",
      "|            LossPi |          -0.049 |\n",
      "|             LossV |             201 |\n",
      "|       DeltaLossPi |               0 |\n",
      "|        DeltaLossV |           -7.66 |\n",
      "|           Entropy |           0.642 |\n",
      "|                KL |        1.03e-09 |\n",
      "|              Time |            71.1 |\n",
      "---------------------------------------\n"
     ]
    }
   ],
   "source": [
    "agent=VPG_Agent(env=env)\n",
    "agent.train()"
   ]
  },
  {
   "attachments": {
    "image.png": {
     "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnAAAAGKCAYAAACM8nxBAAAABHNCSVQICAgIfAhkiAAAIABJREFUeF7svT2QJNe158cBqVg+fQSLlvgMBS7kLNdi0VrIwoWhWGplsGg9yELSEj0WPXpMWHqGIli0REtIRMjgKkLxih5l4cKjLBY9rhwkLGEjpGDREl7IgP6/nnMHZ+5kVmd1V38Nzon483yfe+4/q7uK3TODZ9/4xjf+Z+GnQkgwEAwEA8FAMBAMBAPBwONn4MNn2vHvhH/x+HeNDYOBYCAYCAaCgWAgGAgGxMA/BwvBQDAQDAQDwUAwEAwEA0+MgW+e2Hel3PeFz2dqkuI/ENB/E76YqburcNbgccFw7vGvhSTwE8fjgp6vQwm8/BvhPbvsZze89Fp9/yDwWmEGrwNivG6SwGskeBcJIcFAMBAMBAPBwH0wMOiQT08clJXbC18K2PcpvZ27aw7lQ0lqYviD1fdN7r5dzt8sOJR7dMIowHE29Ob7GXxYggeeQ2d1aGKDkIRWegUA5yBJ4BxmLRVqD0InMCeZvZWurxtqitALVTYyfu38MIOBYCAYCAaCgWDgQgzwhvxXgQ8FvOHOSbYa9H0KO03tRryfWCRb/VRuovyioZ2m7YXDDXYo6ml35tkcBf9css2Wekk6eX8S6KnCPqAVathx3SYmfGpGIU3ktorxbKr0MkCVLIOakGAgGAgGgoFgIBi4MAOd5vFmzhvx/sTsbDXoxyBFS/QTi2TFuMtUbqL8TkKcfe4OZWZn4v65ZJst9ZIki5NH0OywMr9VnQIft8EJf1SM2jkhX6WXAUKCgWAgGAgGgoFg4EIMfGtmDj8h2QifCj8WkjDO1E6F1wryZ5+OwidCEpjzG4tJXcXqn6HjwyJ1Vej/joCmh1344PF7YSXQhzwTipCEXwnvCOyM5s/lMbeV2o/+szBaQRs/Ks4c4uxGHTYxhBg1DyXjgoPZF6l7drJP7V2U/1BYCwcaJ2Sj2JtCmcjV0P5Ejtk82/q8k2zm8SzYF9u/FuSGBAPBQDAQDAQDwYBn4I0JOrJiRRiFjyzfmV6iBhUB3pD5gPVXoRN+aFrq6ldoRSDPhzJyfGDgzR1B/0bYCb2QBT4U0Meb/E+EYnGpbySBWchbwrtCnWXhK0VvL1BLzZ+EJCDkfipwThZ64W8Cu1G3cTHq+KCYhPsW7gX6BQdvVfORcLDaJH080TdajrvOCWcj41yB4pw7JclyxSU72bwGBoHZcF6EUzsoHRIMBAPBQDAQDAQDnoFBTrJAlv5S4MPKlNQ8GuENmPrefFQR+ABUJcugpnMxTD5k+Lre1a1k7wX/pt6ek62evlZqbnSJGusnYmVhrHN115mc0+58XQ97AHoBH4wAfHjJcursXjagjrgXOB6aWOtet2OvBmqWCvWgCru3/UWx3NQc5ScXCzMYCAaCgWAgGAgGjIFvNUzw5pqE0eJF+jOLbaT3Fp9TvOm2wkwfz1YwNoXUvCMkwecO8slx/m3luv3r/DJx0FQsTdRdOsS5/cKh19XBYzoxi2eFjMJ/KvCrzir88yD/j+CfpUu/ZCZ540xyST81B4FnvpuZE+FgIBgIBoKBYOBry0D7K9ROTIxC78AbKdKZPqVGJfnVJ2+8PxZ+LqyErWtKzp4y23w9f6r23Njx3IbXrH6v+7x54k7ZckX63wr/i0N9hsxA1qanVJ4K3iDGayckGAgGgoFgIBgIBhoG2p/Adcq3b8y8ifLn2PhAloRRuE6yCpjzicAHOi8HOe8LqYlXt8zEzw0zJ5/b9JrXD7pfb7yUibtuFPtIGA3/60QNOWq2QjeRJ3SJD17MGGfmRzgYCAaCgWAgGPhaM+B/ApfFRJlg46gYb9gIb9rXCR8Cfi3w61A+9P3K7Nq3k/FngQ9xVdZW84GLnWuO1nCJDw/nnv1Y6q+7O8+yE3gGbS1xnsN2wWWooZaeVogNbdD57bk1lVxNNvvUnBNHRCoYCAaCgWAgGHi9GXhm1yvumgfZ/k3c52rZ1Bv4oCTYC3xwa4W5/I1OhDfxOoP4WtgJ5Sr73CZWZZABkFpr7lUcIBuBuczcC0Vo68kNwk6ogk98SYye7kXn81n0zwm1oJVBATAlScFOyMJRYLdemJJOQe4IqAPDVKGLUdtZ7afS71qO+3PeElmpaCskgRn87VFkEOqMTnYWqN0LRaBnbXYvjRRhMBuVhJ1Q57hUmMFAMBAMBAPBQDBwaQZ4cz5MDN0o9qWQJ3IRCgZKvDbiRRAMBAPBQDAQDCxnoP1LDMs75yv5m4urJj2aX/V8d2SCgWAgGAgGgoFgIBgIBk4y0P4lhpPFC5L1116DakeBv/zwXYEPdPz6lFhIMOAZ6MzJ0qMhGAoGgoFgIBgIBoKBYCAYCAaCgWAgGAgGgoFgIBgIBoKBYCAYCAaCgWAgGAgGHogB/hbqfyusH+j8ODYYCAaCgWAgGAgGgoFg4DwGDnyA+7bwH53XF9XBQDAQDAQDwUAwEAwEAw/EwP/3QOfGscFAMBAMBAPBQDAQDAQDN2XgmzdtjL5Hw0DWJvz3TT+74434Nfu/FPip7fGOz4rxd8NA0lj+TUae3+v2DLnbDwT+1vvnwl3IfX4N8Df3fyR8X/jLXVwmZn4tGOB19MXX4qZxyTtnIOmETuAf9d0L+c5PvJsDBo3ljfAcWdrD3A8XDuaLsxPg866FN6+D0LuDto1/1zvE/JszwPPbWftw8zEvdSZ5Hwu8DpfKOT2DhvL1sESYS31ZUnzDmqy+Uehv2L+0DT6LgD4KaWnjHdQNmrn0GdzB8THyFgxk9aZb9EfrI2fgLv4h31NXHpUcrIA3k3Kq+I5yWXPBbYRvquAcmerpJgZQx38rdolQOywpvEDNQTP2zZypO13gqNd2RP9AN1vp3EHY2vmjdL7QLjf5ye9UTz+xD3vyGlsi1A5LCm9RU9Tbfg3cYtxs60YZzuLua2Gcrbxsop8Yx9lLn8FE+9cyNMXjQxCRdCjPL+Q1ZeCcf8g3iwOPp0oJdyi3XL6+EZ4zZqqHXYZmCLuBpyDDU1jykey40h68GT+E7HQo8MIu5ZbLjOrvzpwx1TPHS3/m7PsoP97DIcmdAV/3IU/pGdwHH7c5I92mOXqDgaUMnPMTOL5xlaWDz6xLqn9HWBmwiXnJcuo3GXT7DYfeto8eYlXwf+78m5hz59R9kob6MzljqmerOLWtMKftpybPxNv+Kd+fX230lNR8vc9UTY1lGVN19Q5zZ5ya6XNJTn2mp/auuTne6o5ov1Ptq3l/tr8beT+77uVn1d6pmcR+3Zztz5rr8a9n9mnPIE/v1B61Nsl4X9i/6H7+Whqdf1OT2XUHZtR7EEeysDa7qiTD9+B/2NTgtrPa/nbuxIgbh5jtuT81KCvpd6m9bc85M9veZPvACTLFDfPrHrXeyl9StdfvTP05z4CBWYCjVojX2X6nts77/j7VRk8J86fO9Wdh+/46c6qPedQjSfA1p3aZmkkMHpkzJbWnnldrsuvBruLPwAZLhTP4M6Hn9CydHXWPhIFzPsAd7nDnTrN/L6CzwK8QB7OlrmKdwBcHGklCEeoLdCP7A6ETkGR2Mb/WoXlxZ6ux9NWvCbrqnND0Tp2zU3wrMP+ZcHAzNhM9pKnNBvwkMKPgmFBDjHmfCUVY1+RCzfm/EfgQgf03oRN2TT+5XoD/ozAIq6amuklGJ/gZ1A4C+hMhC8xMAnVfmi91FTtYfO6MrPxHwiAwh72Z8yuhCr1bgZ3JF6HOS7I7gR76szAIyFogxp7Ul6voc8lSnVD7jrJ5pkWgh976LJLsKuR6oc4cLJGk6eecbLDU1bxeoCcJgyU44wNhK1DTmZ1MU8+ddwK1c0IvtUdXQL3353qvi3cq+L1Qz+csXmdbAfsgUNMLVfB9T5LPPdBZqLPQ3L8TqmQZgPuQH2pioV6prhd4HXauZy8bMLPGOWPb1LmWKzNbzc4SSXojFPOrGmRwNvdkJjVzwg7JkKXp64SWZ88N9dSwB7PpeSYczJa6kk7oBfY4mk88WQydBXZA0O0zIDYIo9C+/rJiW2EndAJnbAS+b58SanjdnPr+tFKe2QfhM6EISUDQnVDPxS8Cgs38T4S/CXuBWUgSOoE+ZhN/JnBGFtZCPQu7CjY93J/6vSWSxZiTDZa6mlV7yNce6rZCZxqfOmp6gb0BNjOuE2aBJHwkdIbr+iL/NWAg646luedG/p/OvDvfQJnlpcjhhVullwGqZBkH52N2wscu1ssGVbIMzvIydTZ5avmiWSK9ikCVLIP/7mtyMXb1X3C9fFAlyyjO96bfmTpmV+lleJ6It3d05Sf7irJ8sSNJaPll//I8ffW/vaGGsoxSHWn26p2PPTh/lJ2c3zl7zuyVAF7Yc2OBTrq4JDvUOxHOQn02K9nJanvpwWxUEbLzseGVnir49Vxi/r7UcY6v38vP1txLF7Orqj0+TM/a9RzMTtLUb4XOYihqs/NbsygAesMgveT1orJFUlSVXWUvu+5MmFxxeUx84lV6GaCVNr5TAahSZGTnYxO7TqjpXJG3j4pnyyVpfC+9HFAlyyjOx/T88rz8ziv5Y1Pfur0CwEuRk12glw28jHKSCxTZ2XziPBfOR9B+z14+aKWN+xm11seygsdmyJLXG+d4nhhRhK3NytKj2aje4EJX+bXA3dBILxSzUXuhzsTPwtT37o3r6WUDhNmfmrbQ1d6dOVm61ITpJT0Hq03SgPN7i6GYkZ0/ZQ6upnMFRXaaaojY02bgWxdYnxfHpWR/zaBjk6f+Q4EXd5u7ZtQr6fJK5LwAO4yuhX2I3VaKBrxrs34gzTebfMOhLUfwtxF2psdm7kH+OwL3aHub0iu3E5hXpZdBb5VBxtawdKZrf2HuZXUCehCKwDz4+Y7ZUi/kmazRvHqPnflJ+k2BviwUi6PqTypc6CUe6izyWRgFdqhSn1VxMW9u5IwCHFehh/jBAmOjuTM5etD8ZKbWWulLai2PecWiW+mV2Wj8o/k707dV+9sOmOnvLZ6k6zObKT0ZHpSFEzRS+cBeC6NpXhPgNsI5fxL8M2b3uxDmjjOD2YPnDBD0d81eqrIK4arOqH345PYWONTEmbqdy7yNsBOKkIWVwNcYd233J1bPrpreQUjWw9cXM7zgjy5wlA2mZK0g9f7rnNdImiq22JKeet7o5gx2TpHm6xw9J50l5mqS8pxR6ygfLIYd8gQZuMQHOF4U+we6e33R8wVSHmiH+zh2q0NGYSfwYeQnFzr0oDnMRlYnZi7ld+rNrj4jxg8CZ/ZCFvbCTaXui+YOzGXeZxMDybWSFOgFegaBX6+0cmwDJ/xkMz5xNd6eal3S0+4+atBa6AQ+2H8gZKGtU+hKeCalOtKdsDV/J90Lo7CxOLHHKkmLdUIR9sLUM1uy+6Ai7pkMzKqSZayFQbju+X3Vddpijp/17HT5nWRXE1OPE7HrQnwAmpI0FbxlbFR/ncv+vxI+E3h26LeEJdKrqAg8Z/puK6MG+Ofp7anZScFze6inrxPeFbgzughTslFwmEooloRR2ArFQH0nMDfkiTLwxhPdu669NqPc8B79Dfvuoi3PDGXHo1B1cnXenmk/Gc7KHqxilF411cn80sTnXH5i1c7w/qg8s/jm0dbNzZyK01v3HsxGH11xmmp0MeorfGm+pm8uXZRYTySn7skZYC+khT21rJcxCmh47ExLTYr/kMN+cFSsMkuPZhNn3kPLqR2KltsJ8OYlN/4Sd1BRJyRhtAb46YWtcLBYVanxl7rwylwvq6XNF6wbZ2ZN7dLP1DKjvQulSSgzPbcJc1Z9DjvZvJZ7AU6rJBlgTgYlioCmr943yb6JMGOKgzq3nZkV4A7n9DBja4N20huBD281ZqmXFOcfLZKli9mdNOePQhaKgKCZG/KEGbjEB7i17t/dEwf8GoIXapVexgfOH2X7fHK5an7iYrywq+xlcJf7klEHJTsMfTTb70+InWqu+lW3tTZiVvlf49D7vtBb9SBNLJmP2gieX5eaNHtFt00mN/4g/9dCaeKn3OSS7MhevcXW0keXx0eSi02Z5A8uwVyk9rvUIpNZRciumll1HrkqnIVPzyjkmjD7uh06V8+Mo/Nbs7hAL7tz/qlf5VFH/SVk6iezfi77wwlS9dS5zKl3pc5jqv5UbKdk1xSs5Y8ulmXXD8DJ4lXXsrpP9TeuH3MrtLGuqVnicg73rZIb/7oZg9XTVwW7zvTzR1fjTeK/Efx91vKJH2Z6zgmf+v6UmjPwEbTnxcIvVN2vBvCRqtv663zuCQedK0yys/lH6boPMXx6PhI6q0FxfnZ+azKjc0FmgDnZKcFMJAmjgN8ZpF76Nf6RQMjTZuCbZ6zfqzZbPf95l1HgRfA9gf/kyx8sd0plJTvhmZAEXmBF6M0nxswskMP/tnAw+y1p/rMg5LLFd9JVqGMXdkoC/wma/0ZYCUVAqPmZQOyPwvF5+KoP/3Pz51SnBOcngd3oJ/bPFivSvdmcwb7ZwF7MH4V67tuyicNfEn4p+Fncgd3IUTcI9HwhkNsK8MlO+HWuzJcky+PNiJ3pZ+YHwsFV/U72e8L3hSSw/z9avpPOAv3EwUZgpyQUoZ7P7DqDOLtWoebvBc5aIllFnIWwN/v9TPjcYkWaPTiPXQbzSVPTWTxJj0Llhz2Y5fvY+w8CtZ3AudhF6AV4JkYvZ2bh2xY7SHOnTkgGapmHjMJa4Dx2YAYyCJ2QDF9IFyEbiNNDDEkC/W8LzMP/rTAnnPMzq2O/ei71vxD+0RqTdBYG85nNHatv4ZdULy8J3NNz8j35lRdmfNfqinRvdu2pXFBHrPqd7LXVMusg/FmgLglwwr07gbM5ExvxfFnoFcU5Wfily3AGZ9LPvC+EPwqd6R9Znn3oH4XPhWSgF/8nAv3sNQrM+JmQBGp+JzB7SjoFs0D/94ViRcwiTj85dsBG2KUX6veNUfZWSIa6K+eyB/3k6n4yXzy/lWzqyXUCZySB8w7CH4QfCexG7m2hE5DsbPKj0AvsRS0zmT0lWUG+P7EbtZzxgcCZCPo9IQnUcBfqmMfcXqj3l3l1NkLfRmAfMAjM/lygtxOWfu/+tmqZBwfMTAb22QsIc+tZ1FOLkPc9K4tl6e6q4nnfKM1eSfhCWBvelv6txaReETjgXOr5P+Z8zSG/FJiD/FTYmY3qhMH5YQYDd8ZA1uRyZ9Nf/8G9rggeWpIW4BvZUulVCEIuzwBvKryRIFnw39yJdVeZkGDg7hnodQQIuT0D3cwIvt75gIdkYTA71BNl4I0nunes/bQYyFq3fjjA5hvJUqkfMJbWR91yBgaVdlaOxg8JBoKBp8tA1uplZn2+B28sR137f9hm2iL8WBn45mNdrNkrye8EflTNG/ofn8jej2XNrEUA3B2F8QEW41cAPL+DwK8ZlgjfbLLAryIeau8lez7VGn7twrNYC3sBv0qWwbOC95Bg4C4ZyBoOVkJ8nd+OaXgsMyNGxQFf79T4r/eZlggHA8FAMBAMBAPBQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDAQDAQDwUAwEAwEA8FAMHB5Bh76LzEku1L9d2ouf8OYGAwEA8FAMBAMBAPBwGvGwEP9MyKdeBwFNP9tvc0d8rrV7C+FfIdntKP521SdnXmTc+m/CSf0cF/+m4FJeGjptMAo3Df/D33vOD8YCAaCgWAgGHjtGEi6EW/oncBfZ8bmQ8ddCR+GOOO+hPP2AhrhQ1Vv9nWKHmoHoVxX3OThMFlsLT0Kual5CLfXofEB7iGYjzODgWAgGAgGggEx0Al8GNkJ/Kc6bipZjff5hs7O5abL3qCPD1J90zfK58PZUskqLEuLVcds/vuiXnhOpYk9hNvr0Pt83g9xxzgzGAgGgoFgIBi4VwaW/gqVD0EHYS/wAYX/hiKxu5C1hqaZwT6HvUSyisqSwgvVVK78uFEOe9yVJA3muXhODvLfuasDF8zNC2pWD7zjghWjJBgIBoKBYCAYeHwMLP0AxweDnVt/L7u7wXWy66O/F5LAGzk/QeJDB2dtzEYjxIrwJ2ErDAI74F8nWQXFiphH/+66plvkk3qPE/3c4a7koMH8x4vRVTjv93d14MzcpPjHwl7A7oWN0Aq78ezIszf1/ySs2sLwg4FgIBgIBoKBYOBVBr71amgyslN0cJkkezSfN2N+pcpP5a6TYgXUD0L1md8JSTgKCJo39R8KByEL/CpuI3Bmb7bUrKyU+YFQhCzshU5g3pQky0/l2ljfBsx/cyZ+1+HKG+dw7x8LP73rQ918zuR5PRPebeI8gyrU8SHvI2FrQZ4Lz4R+31t7QgcDwUAwEAwEA8HALRlI6h8F3oiRtdCZvURlFbV/Jgq/NM21bufiU3WnztwoyU960Pcl7d04twj9GQtk6zmj5aXSQV530+Yb9mX1cfe+6cf3nPChbYojnjNxXk8hwUAwEAwEA8FAMHCCgaW/QvUjBjl8IDpa8CBN7KaSrmls39A5b6lkFX4mMKNf2nTLuj/fsv+27VsNKMJw20Fn9ueF9auZuvp6msvPtEU4GAgGgoFgIBj4+jGw9FeolRk+HGwE3myTMF6AsjrjOzOzDk28vtHPlL8UzvLYuRjwsddCO1ehqzt1GAukn6kZFWd+afKtP9N+qzDPhnvVszrZw60mLm/mzF8tKK/PLzW11T/n+S44LkqCgWAgGAgGgoGvNwN8EFo5CjqzieUzqKG2/RVab7Hk5hDjzdzHpn5F51peMtmL+ipFRhaSsH4RvbzRaeTgxrKH/1CC//Nrjs3Kl4ka9t5MxAkRZ+47DltXSz4735ttjnO6ppZZq5n+Gj7I4FfWXvZy/PNmxijw592qEPtUoDYkGAgGgoFgIBgIBi7EwEZzeBP24A0dIVfMvk51KuANnjnonWtgXhF+LfDmzpv52vLojwX6eKPHvk6yCphRhfmgc7G7MgcNzsJK2An1HpyHPQoZp5Ek/1cC9/+r2Z2rwT46v5pJBvXtMxpcLbbnw6WuPnD2LrCRfXA+9xgFfw+XfmEmWYNQBO7RC5xZn3ftT4oNAmfwvIvQC5wTEgwEA8FAMBAMBANfcwbaDwStf5f08GElC3NnkruJ3LSPs27Te5NdoycYCAaCgWAgGAgG7oCBc/8M3B2scKcjj8301r/Lw/np0pxkJU7l5/qIz30gPNVTc2lJUdQEA8FAMBAMBAPBwONm4CZ/C/Vx3+jxb8cHsCTc5MPk+oZ9sLIRCkZIMBAMBAPBQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDAQDAQDwUAw8BAMfPMhDo0zg4Fg4NEz0GnDLHwu3OQv3GzV9/Yt+h89QbFgMCAG+Mthf7kFE/ylNr5OxlvMiNZg4MkxwBvEIPQGfAR/iexUxD8IvLbiTroI6K+T8A0ELuCPf0D4voRzK/f3deaSc9jrnSWFN6xJ6uMfOeYfMOb1B7B/LpB7DFK0BK+H2wrP9yjA6XWyU4H/eryu/qnk4WDJ/R/yPkmHD8a/3yM/5FJP4OxeO/J8W8kKdEIvpDY54VM7NWei9Krurl5PT+1rEB6W8jbFZcQegAEe2p+ErjmbOC/AcsZO1GZX38sGl5B8iSEnZqQTuXNSvYo7gXnYc5LnEjeMb9V3vGFv25bawC38jXr5L0dcWurrk3sDzuiELPTCKBDjw9xDyyXvX3SZvPBC4xm1C0c+eBmvcZ73Y5ekBf1zX5uPDnmVgazQ1HMlvrNyuBtebZ2MULeazLwc5PXULai7riTNFBwUzzO5xxaG//GxLXXBfS76HJIWY2Bn+oJ7njVqr+r6BdI28gVT2uAJn9rs8r1scAmZ2/ESs5nRXWhQ0Zy8YNZd3Ge14NwlJZfiop51qb387kVOtgB66kPSYPHO6h5KTe12012KGvPC5nNqF4588LK7eC3d1aXa5/6Udr8rTubmDkq0/OB/2sTL3IAmnuVvF9S2Zy5omSzpJqPP3z/Z5anIpfh4jPcdrlvqjesKLJ+k+RVPEQbD2nL3qTjzx8Ju5tCD4uChhT3vkp+s+ekeL3lX9zle6A7dhebUMZfaq85byeiFYoEk/eeadLqT/ZmQJ3IRepoMXPq1dJ8sPOXd75Invp5By89GsU9cPJ+xRFHtdkF9e+aClsmSbjL69IKX4uOx3ZzXF591Tsq3Tma/SjLsLVc7ys7CYWH/pcr4AkHGEwMHl2Pvzuqx2ffcnbfqOdpM9N5sFLnR8kl6ELLQC/hopGpzXyh26oTRIhvzcWvuIDsLnItNTS8cTY/SgzAn/gxsZgCkF5LQCdl8qZekxpPL97KJb4VR2JuPPQicsxGOwlogX8+U+YKbN2W/S8AkW6727eRjI8zJwiisBGYiW+EdoTd/kB7NpgfgoweBeUnozSZGrsbw273IZ6H27syu84mvhCT0QivkiwvSN7ZF5s/FZ8rvNbzRae8L3NXLB3JKE7upm9SYBc4Agw3aSn9HeGaxUbrGPnN1Vn6lyPNNkP2YlYVe4B6dMBiyNLWjaamrHK+DnwjUH4UsUIdUe5Q9CMmAvxeQzjBIgyzUc/CTsBYOQu2ReRXbCCOOCbFBoNZLktML7Mo98VfCURgEL1lOEkYBeydQNyXkfiD8QqhnMrcX9kI9o8jeCuc+G7Vc7cAPBpjVCncpbfAefO4y9cZ5VJzXArIRitle9XLor5J8coE9qqbyOlXeKQgGQ5bmvNH8JL0WDsJemBLm0zP3PZMeajbWnKV74Wg+iv5RSKbnzqKWWZ1wEJi5sx5iYDBf6iqPX2vXsr8UPhGK5XlNfldgTi/479XEyNevgyy7E5AkMJ/ZzC1mS93oa50ZzGIOc48CsV6ospKxFYqQhb1ADzbxURgE6oiRoyYJO4v30sggjGbfSnHYUUAj6I3Zd624XFl4CHsdbL/aQv/a9TMrO7+XDarsZHTOpz+b30uDKoOMykM2c5kjAAAgAElEQVSWXb5KzVpbZfw+vauk3+cO8leWp87XuraXTOp9H8m9kF1VaXyXemFST10rGwVGIQvYWyvoLYbLDqNpS79QfHFWyTKK8zvZg/lrae5RJcvgHlX8nLka9mAGGkEXYSMkYRCq+Hnk/Nlr+cUKhxcdz42+8edc+rczyVO5mZarcC/8agHSqSHKwcsUn53i7JyFnWlsQM+cFCXgeIlQ63nxXNO/Fzo3KDX1LvXCLLLYFxmEZDax3mxUForzMUfB7zPI71xNlv0nAV0F7jwfU+d8qhru5nuqTe/ocvR3gp/p0lcmOc7NLtHLprfKRgaxKvQU52O2z/2gWHY1+Ml8+j92uUF253zqts5vzaxAL6DZEz2Yxmb+OZJUvOT1T82c9ErAUxawvcauUnMudPU84e/XQt0DfrjbUulVmK8pZh51Vaj/VOD8Kkdnz5nts651RYbfGXvrhuB3zt/L9me71NUzPAhoBD2ajWJWsTj+RmBela2MwfmY9FRhXnuPg6thFjVJ8HPpxyde5Shj6/xBduf8LLvlmbNP9RTl127GKDuZv5FmXjafPdmhCvH2bi59M5OhXBxdhUUG59+ludNwvmEuEWr7pjDLZ/8qRQaxKr0MgCShJbBTrBcgeypHD5KFYvYpxY4fCsmKqobTQ9NIbWexXhpcJ9uJuqzYKQ6mZtJTJhLE2z0p6wTuUKXIyM6vpueQndjXS823OWb52vZZMIMe6rz0ctq+VVOD6+cN8unzcjTnIO1/gpCauimX85g/VUuO2XWnjexhasgdxTiX1+N2Yj67VOkn8kkx0MpagTKTa2upy02QWD2b3MHlu6Z2yqW/n0gQ8/EsvzR1+MSr9DJAlSzj6HxM9iNepZcBqmQZbQ+vB3hCyBezUb0wOH/ObL8GVir0rzPOTE0zc7cu1s4oymXLU4dfhfm987Ns7l6lc/aUuXHBuoPfpaZ7GeCuJemAtR3CbvjI1E47xbPlq+pklCY2yq/3ZPZeSE2Nd3tXP1dGDaiSZYzOx2yfY5O+cudqinKda+hlAyQJbR/89M/Tr/wvOe7spcjJFuilB5ckXpy/ks15aCQJ2eyq2n2KEpXzWsMOXdOH73ejL7uaXjaokmUcnY/J2cSr9DIAshZGs6vqZWzNydKHJu/vQr69W1P+jW+88UrkdKAo3Qk7IVvpFDmnp9w8y4Uh5pTU/Fzd1I/Hp+YlC2bpiqPsUZiaPVhOarH0qnwmfCr8VXjfOuv8LL+Cu4+WX6o2M4VLOZhpfykMJ60MFthKZ2HVFkz41CHoig9cjPtXKTJ2zp8y5+7YcjK1v5+XzMnSFb1s7tQJ/FqFZ8czXAvXSVbBZ8I4UQhfoO40VTPRdpFQ0pRe4NcT+4mJPkZtK8TgpJWDAqAXcptc6K+trkhzRvUXtr/yjXJp35I67ubluKCp7aGFeyHkktko4sX5S032+JuQDN+RHptm/KVcsofvZ37v5hXZ1NR52Kdkb8mN9MHs2uv7RjmcddfCOX4P/Oxi/vypPVNTi4/UezKb2Pg8PPu/q9nMfGJq5tSO8xNezkzNoyIJfO/KDtQWYUrYgWeXHeBjdMXeduErk96PhM4SG+li9ilFn5ckZ2xi+D9oYte5h+sKXD7Lbu/OmX5Gu+cZ45+XfmthR7I6FuDQIvRCFu5T9jpsJ2wE7ClZKwhJc+TwAlwitb9MFOeJ2KlQ3amtSQp0BmoGgT/cPgqcX4TrZG42fbflYOpszkMOU0mLDZbfmT+3hx8xygHFBatNfOXip8xOyUGYe85LdpmaX1zQ29niG+lBSMKpM2qdtb1QnVnMqJJlFOefMnslr/1/bKrhG+I4MYjYVshCEZIwJex/mEiUiRihnQAfzL6peD6Zx6xBKAsH+v6FLQ9Wxq7cjTuOBvybCB/akNH0StpzgV9zVnJSUX9Kdkqy9yDsTxW63EZ2Z36a6MmKMfc6obf+H+Draj84UcAdmYWgR7O9moqRP7qiykMNJRlzfa5tUY2vn7O5x1LpVDgsLH5TdWVh7ai61NQv7a1HDDIArwHPb80v0fS1fODPvUcsmXldzagCvh+X6woX5jvVDW3t0g9wvRq58MYGYHsyW78951I+Z26FXijNDpyxFg4YkkHozK6K/L6JzbnM4cOUn0ntxmbw8LNQCJq0PmH6V1+VvGT18piHcB4+tUXYmX2URohnYS8QSwQlxOdkUKJrkmv5zLipnDqPmeT5Rtq5A5LZWbq4uDfZibzfrdYT2zQ57/MsOPdokLqasxYKjgn+4Pwl5k5FnOUlyzkIvVBznDcIK+EozAn1a5dMsrdCEXoXx8wCuU6ghzOKMCX9VPAGMebzTXpO2H9wyZXsTiCeXbya7N1PxJeGmM+9qwwyRqEYXiRuaaRb9l+6fXfLgVn9fF0Um/N7aWKeS/yt5a9T9HVNUTJ/ND1IYxeDhWdVUgZU4Vm3wutnFDZCJ7AvfivEPmiDN/CZX6wvSYPqW/jq/HZXYtQi5Ng7m4/CLgLxmutIOEmyj03srlxeG+xxOOPMolr6kjAKVToZg/OruZ+IJ0uOE/VToWLBQXo7VbAgxh4bAV0lyyjOv7TJ7EFYCUc3PMsmd52MJwqycoX8GyeKfMo/CBZiQG8FnXQx+z7UoEN2AjtldyAPKAkHi5Efha35WXot9M7njeodIVkOnx+rUod0Qi+kK+/5eXU+5zG75vBrDk0cMKsIU7JSsHcJavfCKHSWowZsLCf14u41R2xK6qytJbM0Z/TOrxwQn5ODEkngPHQxDXe1nxxyFPjguzG/k94LlWcLv6J6RZJQ+7LsOpMcts9xThXmd1ZT4738tZCtaCs9CtQyq/6KFZ2shjh7IsTxqWcm/Ug2nxj25ir61RfqaP6U2inIT0U+FD4WBiELvcA5rawVAOToPbQFD+DnZo9ku60usMuoGcxjFuDOgHiVo4y9gL5O1iqAb54ltpciJ1uAs8jztV+fPzl8kFwen1pi/vUv92oesake6sFUj8JXPeyBZIHXRwWvl5p7XjH9v1sLJ2nsjSvrzCeHkN8JRWD2+1fRl+//pvy610F2LwwC9Ulg/ihUOcrYC+glsrX6WptkAO9/KScLzO2FsSbvSHd2FuO5c544p0zE2W9l6KU3TV+2HOF6l6bk6u6cOSdrJXgePJdkWPJ6mprHDllg56MV4DObmUmo59XXPGXca2d5ercCs6bkYLXUI0mgfxTWQv36wiZXfc73UvuPLriS/WPza32WX/dnZhX66SWPoOnvn7svfd0Sp7fd7Tqe2x7O2wiDwEywFQ5CEvw8cvUuaPxRqD9AWssuApKFjwVqvvHsKrRMkso2VrqXHl0bOe+71J2ZXIB96tllZgfy4CgchCrZ2aNs5l2RIpmrpZ9cFerX5hQXx6y5UTaYknpeVrKeuXR+Ug9od5o6hzr2HK2+1mRXXM93oZfM9j5JWVCl3SNbosbx25ovFWtfg+zJWaPBxlwpcklgzngV+UqyzKNAzstUD/OJVxllgDbOLGYi1JPHJ47g1x7ixeJTqtaR8zOmaomthV4oAr3Y9yVTz6WenWWUZpFOPjvumjgutb3pifRLIWbADXdP1oPfylaBqbPaOuYwE2HOoSlI8gE5gI0UIZuNGgXm+Fm+npoiZAyTUdr34CPJNKoI2fnstxHo21k8WU0vjT0n9ZllK2AWO7ZS86MSAOG8tdmoImTn+1lJccBs4q10CgxtcMbnzFGoe+L7mZ38uleRvRfuWvwOKx2WBL9TPb/IyM0y1Nf+Y5NjRi90wqbJ4dJLfjuRqyFmU4eMppNpVBGy8zmz3cOlr2rJU4fgVxllcFY9z9cRW1shveROSVIS+Bn0+9nkqKlSnE0dGJsYM6oUGdn5/qwarme2Od93UHESOA+hFiTzUUXIzp/qIYYwZ212MZ2kQRVqaw0xfM6kN5s/kjBJ0t53qTCDgbtlgBcfQHjTCZlmYKtwZ6kinQT/RT7ddZnouc9lb/t1E8cXxfJE/NyQv/vUOefOe6z18DUlc/Fae+4zmzrjprG7ejaDFmJ2Fnphc9MF76CPfdhriSQV8TWCFGEl5Ofui//tZd3X13dzdLhPlYE3nurisfeTZWCnzflGnIXfPNlb3M/ixY5B8839cD/Hvvjbi0uPG1SYhfom5ft4szouHXSijtlJ6ISpc060PqnUoG0BvFXZyijO9yZ1/ldJvm+m5eLhu3o2ozY9GFj6ePHNbz5wp9a8sJ1nMlgtfRuBe1Uhj/jYV9mwgoFgIBh4JAzwzaozPJKVYo2GAd5g9kL98xg3ISiraRB4w7qEJA3ZCujXXfga4Rlw305IJy5MbXbAv29JOvDr8mw8t2u79235Hm47IPqDgWAgGAgGLscAb8AP8WZ6uRu8/GdBzp3Lm1tIMPC6M3Dbr/Hb9r/u/Mb9goFgIBi4Vwbq/zMfdGp8g75X6uOwYCAYCAaCgWAgGAgGzmeAD2z89K2Kt8+fFh3BQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDDw9Bl49oSvwK+p3hf47/0hxcBPPPYWC/XwDGSt8I7wmTA8/Dp3ukGv6e0/51AU4x9kPN7pyTE8GAgGgoFgIBh4AgzstOMg8OuqKlnGVigu9phNv/tj3vMSu13iuTwVvg4irHekrWUXobsEkTEjGAgGgoFgIBi4DQOb2zTfsrdXf5mZwV5zuZmWBwtzj6+LZF203OKyfHhjxlMQfgLHhzYvvC7bn8w9hbvEjsFAMBAMBAOPlIGb/EO+vBltH+g+Sef+6sT5e+XGB9rtnGPzOcVR+6j+BfZTj4Pnyq/0D00RH0Drr/pP9UcuGAgGgoFgIBhYxMC3FlV9VcQbUfvThTNH3KqcD49I+wbph/IhzkvtIVaEoyV7aX4q8pGQBO72HedzT2LMm+rJiiOjUMxO0vy5POQD0710PWeU3Qm/Fg4CH0bbPxtGnvOSMJgt9YpkRd4RPrEa/CIwF9kIzFkLg9md9F8FhA8U5P4kJAEesgAHK/PRxPw+cq+kF+q9mEPtcJV5LtnZrZkUoKcK84vzvbmVA097od7X125cMXFmnRL2rD0H2cnA/FEgnwXmJGEQlkpWYWmKmfdzYbt0SNQFA8FAMBAMBAOXZqDTwCyUZnC69EEz8wbFedNdKrwpr62YN1L85PxR9k4ghwxCL+Qr7/kbPT1VqBsF37OV/6Gr2cjmg02V2pNdrJcNWuGstQXRpS1ofM4ehCz0Aj7ibc4/CGiEM3ZmExuFjfOL7LX51NW+rezaR5r4QRjMPkrXPmLVpg6/CFXwq1DXO3/KLArmicReMX8Ofpqo86GDq6F+YzPYEwym6UkCNUulqJDXAh80wftCL2QhJBgIBoKBYCAYeBAGsk7lDQ5d3AadbN4U70MGHVIWHrRV3a6p5c2aGVWKDOqq9DKIVUky/Icx4kXItcD02MSu6+lVD7ywW2li+Osm5t080ZMUa8+Hh3pPeg5uiM8R7lyOunp+kt3OLYr1rh6zE4YmluUXi62kRwFdpZ7hQi+ZRV5ukp18dveykbNvYt7NcooL9E39IL9zeUxqwBKBn07IBvbDDwkGgoFgIBgIBi7KwLcWTuPNNgllon5QDNyHHHQIP9U4JUnJUeDNvDSFR/n1P/xcU8z04v2xyc251GWhzBUsiK9Vw37MqTI6e6lJP/9sBboKcwFShJWQBGKj0Ak7gR2KUAU/C51Q+7/KPrdKE9jIP7RFzmfOXuBXufQC/qP2/6XwXwtV/m8Z/5vzW3PqHGbzfP8L4d+6BuL/ThiFtYtj+l2z/GEiv21iUy69/Fra9zObe3JfdqjSySDnz3bpMIOBYCAYCAaCgdMMvHE6/SLby/pUeEf4gcCfk8K+b+GNEFmfODifyJF6dk3ev9FeU/oifZOe2pzcIcwpDp3sc9/kmcFPgvycXv4gVIHHrbBx8bVsML6oet5DbBDouZT0GvRdgQ9uPxQGgf8z8Z85/MeypyRPBZtYO+s/sfwoPQi9wAe9JOyEU0LNEskqKk1hmmmE+8NMLsLBQDAQDAQDwcC1DCz9CRxv3vWDD7ri2gMuXDBqHm/6vxJ+MjF7pRg1CDvz4cNLklOa2CVczjn1QYBz54TeUShCO4P7+DvNzfDxg5y3Jgo5hxwyCHvzj9Kc2wlFqIKfBDSCXcXPcuErswjkvayck2UnYRDYoe7xf8r+H4UpYcc6I1lBkW7PIfd7gf+zcWpWbzOo9VLkMBNdJTW+S71kZnncxQsxhP2RN4X6bLLs8jwc/xsMBAPBQDAQDJzHwNKfwBWNrRhl84aEj2yEP5l9H2qrQ/hV1YdCfVPn3CR0QsGR7IQk8IaMUMuu9Fch5mdgJ5evtq8hzZwqzNsLxcXYr/Zki/sZB8WSxasu8ol3FkdhH53fmkkBfhrqZZTDTjsX3Mj2czgHqbG97J8LNU5uFFYYJmvpeq9kMfK+hvBOIE89Qn4j+D05q/ah989LZ/+3KJMtezTNOfSuzcfuhK35cyorwf8BqHjfFdK7EZiFJIH5O/PnFPXvCIeJgtFiWfqZ2Xvp4mo72X91fpjBQDAQDAQDwcBJBuobyskil+xkJ+f3snnzygJvSvcpnLkRjsJoemqHzvJJmjz1SG8aVYRkwB8t1uGYDNKjUISdwJyVaWJekpwsjAJ1a4FYMUi9+HA2yj4QMNk4u8iu+7rwlZkNNT7IGKtjOfZDmD+aXRXn7F2s9UllIQnsUMwmNghbocpoMRd68WzYgfM7S+6kk4G55PeWO6U4jzmAviqdjFFIAnN8rtZUvZbRWR2xJGSnia2EjTCazcxT0imZXEGRDRBm7YS6F/FeQIMq1GXhurNcS5jBQDAQDAQDwUAwcA4DRcX5nIaofTQM8Oym5DAVvKNY3YEPiSHBQDAQDAQDwcCNGHjjRl1f7yZ+WhLyNBngJ1xds/pa/vEer8OHRXa4zw+N93i9OCoYCAaCgWDgPhh4dh+HvEZn9O4uRTYIeVoMZK2bBD6I1w9uw9O6QmwbDAQDwUAwEAwEA8FAMBAMzDOQ5lORCQaCgWAgGAgGgoFgIBh4bAzw6+VB2D22xWKfYCAYCAaCgWAgGAgGgoFXGUgKdRbmg1z/aklEgoFgIBgIBoKBYCAYCAZedwb4c4chwUAwEAw8Bgae/Pejbz4GFm+xw0a9/yDwE4J/L3xh9ue3mPm6tvJi/dfCd4Xg53V9ype/F6+bnwl/vOVovlaReO3dksgLtPP98l8K/CW24wXmMSIJP7jgzKxZgP0usSOz+C+h8I+R8z4REgzwfe0vC14P9b0zqfazB6TtlT2e6j8jwptBMSI/kP5I6ISt0Fs81KsMvKvQ7tVwRM5gYFAtr7Ovi3BXvt5uI9maD7cZYr3s8k8XmPN1H/EbEdBdmIRfXGgmr7m17TZcaMeV5hQ390Jjv1ZjeC4P9bWXdfbHF2Z7p3lgifCDj7Kk8I5rfvhI9rjxNfkGPgp8QbbSK1Da4CP1+WK4b8k6sNzzoTyvpyr9xOI8tzwRv3Ro6uxLn3HdvKSCg1CuK7wmf9t+P34t5yG+dq654qNO54ntesXAJeVSM3nN1e/vVV9izy81JF9i0MIZ93nWwpUWl019jXGfqfjioQsL+4m6pNhuIn7b0EYDuoVDeP08Bnmxxzk/gaOJT8BgL1zyC2spKZz5ocCL6DjR1M/EJ0ofNJR0+kPwd9+X5o7r+z70QufxjKZkp2CZSlww9lh463QnvtbeucXdsnqHW/S3rQcFeAYhyxjYLCt7VFX8KvZoG1X9qBZcuMxT5L5ebT1xx6LYfXztTZ096uztxE63De01oLvtkIfqP+fPwL2lJflV5b8TeIgP8ecI3tO54L87Qdj3lPN/XifJ5w3o7wT/52+y/DcF/kwEPe2f3yC/WtDj/0wF9fw5syTU35Uzp54DZ+T4MTT2UWj/TBr5qX0VnpQl9Vmd7PZtAXswn13ZH+33lfviH7rli4n8PwvsW4U4f46Gfua2f74uKfY/CJw7dc86t30uKr8SP//78t8WeH71dZdlw+szwe8ld1LmzvNxZp96Rkl5/zrhbv7efuc6p31d1eWYNfWcicMbd/6LgF+fjcwrn5ltnF3+jUDffxB4JpUrmWfLRh1/EJjbCR+cPeF5Qy+1F+a+9tix8uafZeWWHK89fOTU11d9HU69Juo8vt448z2b5/ey0AuVZPGMvidMPYP6tVv3rzX0AfZBcwbn82yYxflTz8bzoJJXvqbac6ipQg5u/NcTsV8LnM8unpdsjQfra78PWfrq+wWvt/brv+Yrr8zmPKTU5Ak9tW+d1dmMuZ3qWOp5zbfPxx/LOXyf4tn0wkfCKGThKLxtmudBrX9mSf7c1yic1P2ybL4X1efPXv+9sBH+IiSXk3nlT72uyNV8/b6B/yOB/dgXqXe6jh8rnz0vqaDuUb+Pc///yc7wu3OnU197PINT3xvrLmhm1efGuf5rYSv/50IRklDzWbbnWO6VwEX7uieehVqfZM99H661owy+Tk5Jr+QHQhKYV5+37yE39Zqhpubg6plQn2ebm5qbVQRv7NgL7PGNc34CN6p+LxxofCDhYU1dzq+zc04vOwu/t9ggDQkImvjW7D9LD8JG6ATumQU/r/b0iieBXbA7ocq7Morza8/aYkmas3iAwAu7cD57ZcGf/XLlc4+aJFDP/OF5+MX/MosZB+EodE3+Lfl/Fdi5CjX1XGz6Kk/4Xj6SQ4xzPhF686lZCfWOU/ekhrn09jhOBtlJ+EzYm02amYAYe8FjLzDjlAxK0sd5nYH6Lf8jIT4KB/OT9Nwz+oVyndWhfijsBGax09+EIqwFzoSXQchCFXKAc5Ow/yr1Cm+eu051nMNM4sX6mEWOeWAQiN1U2BuMBuakGw6j79D0MrsIfsffyO+sjvxO4J7cpxeS5eC7mI2i9iOhE5hHD70boQrxQfizkM3Gx54TegDn/00oAmdV+YmMes4om5qDy3eyPxV8zyDf7+XKX5gfyNoKWeAu1PdCJxyF9hyFru6WBHalvhMQzn7mYOEXKsnKAud0Qi9UoXcncCZzs8BeXno5nUB/ErKwRAYrYm4SetfEvkjVLvWSmeRx17nns1ZuENj/z0K7O/c7NHF69gK5LCSh7kjcy0/lDAJzmfND4WMroJ/9PSx19ZoCfu+aQ3dCL7Azus6nB9kJ2J8IWegJnpBOOcB57DNYLTOyxZlFnFgSOHtq93cVL0KVlQzmsiM2fYOQDXVHdq5CHfX0fSbshWRJ9MHser65VzWlOqZ7aeLMQtpziG+EJLAbZ2WhlXEm3tbhszvCPM5jfpUsIwmcuxaGmpDOArXkam9NdzIAOe491IQ0c3rhIIxCJ9xI6uVZwg/BLzeaeH4TOyw9i72G5ohOfu9izPI+NrEqkHd0PmYRsoutrCZZDP9Ll8csQnaxXjbwkuUcmhh+amLehY/eBdiVnZEkjGZXlWWUJsaMzsW8PSqeLVfv6Uqvzt65ALXF+eRBK0WBzgU9X+zPvav0MobGJ1YlyRid35qdAtzRCzwhReBeVbbO7mWDVqbizEuusMj2vLQ95IamPjuf+uJ8TPbkw7aXQU4n9KZrLssAN5VOjZ4Xns9N540zS5RmZi8fIBtn47NLvso8l6mvr5Zv7+/Vxkzfn5w/ZfYK0lelyPAzsnz2YLcqrT8qkVy+c/ac2StxcMksm+fuzxnlr62GnfyehMlXKTKy86vZy/B91BRXN8j29yVFfbYacsXsqnoZ4JSQ3zYFO/n+rPb5Ts1jRnGJoZl7kJ+bxvZ1zAz6qiQZ2ZxBuq8J6XYedaPLY/q9yZcmj9sLg4tT4+8+yk+WX0sfXW2WzR5eRjmpiVWXePs9Y68Yc3vBn5vlAwRdzPZqJad9NtRtXVEv2++Y5VNTBX90fi8beGnPqDkf7xTcNX3s0btYke3vSM7nayk1U/Gar7r9+oaPUUAje6E3GzUKa/OJZ7NRda8ke+4ZtfPpI/aCh2+5gdeZOxUUKxqkR/OJYd+HHHQIP149JVzwKHQC9V5GOb8SehcsL1W83EP/d5p863IWdZ3QC/g3kawmHuQ7rpkHlYTRxbzZyeG8LFDrv9Fv5I/CdTKoAE7QyMo0ai3U+fAwxcVN7stufm935JXpd2hz9H4oeJ7ebIucT33L66eKcbe9MAr/JBRhEG4i8DI2jcyek94S7EDvqf3rjCxjFPy9eeZJ4Kwi/Fg4CB8Jo3ATSWp6X7hup6ya0XDqHGrOFe4wCNy1CNwHfUraPNyekuOppHK9sBLqM+I1hA3XVfh/0u0caooVDNK90AlJaGsVmhR/BgV/a3pH+eyGbIT29Y2fhXJVMS+H+dTVa6Bv8tRzXhE6003JtS7926aKucT213Z/VbCTOQhrga8hsLI0sR8Ixfw5NSgBV5x9FDYCcxFiiJ9voRdqbAML/F417OnnYu+tl5yXo3M2svne5b8HsH8SRt9kdrZ4W88czivCx6b56c8onBK/i687NE31LjXszy8KroUk8D0GfFYLz9CdandNPXv8WuhdfG5n37qkptb7WuxR2AiD0AnEsvClYSWNFGEveL6JZ2EUpp4RcdCeqdBzeeOFdb1RXMkouzOf4Yfr2y9SAQEQkk5M21iuEteWnuql1pPV9s75N+mps/yevCF84vBD2WXuUMXXApwg9PkvhLn7W/kLRf93hSRkoc6jYCMMwijwBX4b8fswdyccBfb2cpAzCu8L9KyF3hUk2S1Pz1y+NZkBLy2vnMMO2fK/kOaLa0qYcUlJGsbZzG2fW3sONYAe3sj9PX4qvxe4C3lyb5m/lr6JdGp6V/jAgbm5GdbLPzaxS7mjBiXhI6HeZ3PN8FO77NTbCfCI5m6n6jkK/gYhCdyf11wr180Y1MDenJuFfTtgoT+eqEvKfSawY8Xc9w32uK1wHnLTWbX/tntw/iDwnNrns3S3o3o/EjpbBr9KkjEIzGI+HN9GkjWz707AZy5f017IbQXORZ5IpWEAACAASURBVPcuSaz93jf3rGlLwqnvGezCDu8KB6unrxXOvZQwaydkgbtcx2s6cfBxIvfWROyuQ3VH9N4Og1d/tyJ/im965p4RuZOy9ANcrynDyUn3k+RFBjHdzHG8OOpDpRbfCz79l5akgeOJoe0evpSHitBfbZ+f6yX+T0InFGuotUn+aLElalDRVvB97NILnTAKXlLjL3Hr3ejdCb1waBqz+Xtpvrjp6YTR4qgi1Fk1vHL51uSMNo8POoF8LzCTL7gstNKe1+bP9YsaegHtJZtzdEHOBsW0r8fmHluBnp3Qmb8heabQM1zTw3lZSAJ7XSf+/1meqmVuFe6DDEInsBf6psKe9KMPQr9g0F41g4BGUqPNPalGZYvA/ncl7Jea4Sv5oBXuv1T4WkhNMX6xGDzeROhPTSP+eOawQfUVtNb7Jtnn7MaMrdAJcFmlyCCO9pIaf6m7sULOqPC92Zyj9E5YC70wWBx1ELLzq7maiBHinDSRI9YJo8BZG2FrMalXhF0uJZw3CoNwdEOTbNBK1wbMH6Vzk0vySxNb4nI/v8uSnlqzkrEX0PwAoBOKJYkhSeiEUeD+G6G3GL1JaIVYEfjJ8qws/QA3akLvpmTZg/lraX5seV/C5UE3cSAxCEF6IQuVRGKdxbHnxNd729dz5ypZxlEYXMybSQ7wQn2dXWcNFvOzO1f38oTn/cxgFpIM2MwYBPLEqxCfekEMineuDjMJ/Hi+SpbxmTnJdL3DV1UvWwe59T61ll7m1r3JM5c4QDYCe/Jj6B8I6Sr6XHop8l66xvduL4f6ej45fM5v46NiACFfe5LFqvKzvN2UveT6Ou7GfIS4B7GD5Wq+WGwvvXG5tWxAf+fiR9kAof6fXG7OTEpkYZwoYBZ5pM7dyy4WO6U+UZL9rpPsCtr7HJQr1w24Js9PdHk9wf2SD5XsMNpM7LXZ2fRSNajwQ6EsbZioa79mPZ/MZzcf27gZnFt39zUTx7wU2srrXIRe5gwW20n7cwiTT5afU70SXZNkDvFzhLO8JHOIH4UPhK0raOtrqpiRrK/G575GqVsiRUX1zJXsgzW1dp1Va9F8z+O1+qabQd1gxck0qnN2a3ImyC6BzQ5J2Lj4UfZoPpoahH3IIcn0bRRzRzeg7uJn8xMp4m2tP7eXs/EB2Z1AvEq9gwtNmknRw2Tm1WB2IezReuu+R8snaYCszc7mo0ahCJwL/F2yfOaRhwv6q3TO/sY3vXPC5gAa+SvX7wl/EPZWz1+J5ce4xO5D+CvHvxP4q9WdwIv9LSELg1D/SjKanX4mIBA0CH80v5dOhqN0NkAcf5Wa2C8F/vo8BP7FYp3Z1BD/vkBdPVfm1U+P2I8a5pF7W2DmKLBDzRNjNvI7YSN8T2Au8Zq7KnBCH+e8ZzHO+a3wM4E55NHk2SMLXwj/SmDvPwhVqOXMf3Qxzv17IRnoPQjsh2b/twX6mM/5nekkXQRmUEMewR+F7wrsQBwU4T3hDwLn0EMNz/VdoReYX4TPBc7/mYBk4XcCfVNCfBC2lszSRTgKnP19gdnsMwp/FBD0jwRqmMHu2fA9izPjl4J/jXBOMpDPArNrz0E2z60TmMv5g/mcAUYhCeyF/bmA7IX3BOL0IUVIwijUczjrtwLC/nx9ssv4PPTK/3aKbAXmcn49j8JeqGcx949CZ5ra64SZda6vPcj5kQWyNHPR7DtaPElzJnWD2b105fsPsvGTgTtiZwFhFucwg9dRfU39VHYv7AV6poR96n7fl70TNkIRktAJ3Au7CL3wzGLwUudiE+es66RTQRbYl92Zz5nsnYQi9AJ5fM5g/u+ErYCsBXb/3PyDdCdQSwzgZ8GfQ4zzklCEeods8fekfyYwB0H/QegEds0Cs98WEM6dEmrIMY++TvilMApJ2ArwtRa+sLjUK1IU2VhNlv6t+ewNyH9fSKalrr6f4HN+vQfxUaAHXcV/jXIn5nfCKHzP7JV0slgv3b5XcNcsIMX0H6XZG2HOIGShCKOQBL5e+d6Lho9/ED4SvhD2wi/NhqN2b4Vekt/J6wR2hY/Kf5LNPGLsgeaOCNx822Lk/iAkgV24I3YRerPx6ckGeugn1gm+h33fsxz7D8KPBIRzEGo2AnMGIQm94Pkd5VPfCZxF/W+Fg4D0Av1JmNqt1lH7M+EfMa6RvynPbm8L8AW21sMZvGbeM38lzT7MZs8vBPakh73AXkB+J9BHD/nPhbofNRuB3rXwF6G+h2OHnMlAUX0+syfKlzPAC5YXcitjGwh/MQNZlWlx9fWFRSU8oyUzqRmuH3lnFXzT201M7xTrJ+KXDiUNzJceGvNeSwa2ulU3cTNev5uJeIRuz0DSiOH2Yx5mwhsPc2ycGgzMMnBUJjVZPiwQD7kZA/A53qx1sotnsRaYe52MKqCeZ/gQMvV6Yg/2Ge9oIbjZ2eyNdLmjc2Ls68XAqOukiSsRIxdyeQY6jewvP/Z+Jj67n2Nem1N40Fk4CHuhvDY3e1wX2Wod3nj5cyj82DoJgzAKIecxkFTOhxVes5cS5oFx4UBqe4Hn+hCSdejaDubPFvGaQgbTl1ZJA3sBzovpS58R815PBja6Vv36qjccZexfz+s+6K34ngCGB90iDg8GgoFg4JEzwJtSfuQ7xnrBQDDw9WGAD8shwUAwEAwEA8FAMBAMBAPBQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDAQDAQDwUAwEAwEA68HA998Pa4RtwgGFjOQVMm/HcgfZOff5gkJBoKBYCAYCAaeHAPxz4jMP7KkFH/zh7+1FnJ3DMDz7p54zjpnK7wlDMJDSdLBv76HwzudcRCOAhxzrhf+EC+5QUgvp8ILBoKBYCAYCAaeBgO8mU3J1/0DXDdFyh3E7oPnor3XwkrId3CHqZFT57BDsT2mei4Z22rYpzMD4YF8SDAQDAQDwUAw8GQZ2M1sfh8fLGaOfvAwb/D9PW1xHzzfxxktXXOvq7burvyswXP3jg9vd8V6zA0GgoFg4I4ZeMq/QuXDxaWk06D1LYelM/vZ/5J3uO74m5x16Q8fcHSTPU7d7b55PLVL+xrgNfX+qYaJXDtjouSs0GjV7dys+OGsSVEcDAQDwUAw8GgYeIof4HqxtxV4c+QDxsbYTNKDwE8biFUUy88p5mQhCb2hrSUPOqE0Sc5hj2SamlPCeYOABp0VZ+mDUMxP0nvB//SEc/Dp2whHYSUQx94aRukkIMR6gZ69aeLE6NkInZCFIlCHdEISstCbLzUp1BwE5mEjaPxBWAlbIQmcV2MyJ6VXtJ3F/OKqmbkTssBM7DlJSvSWRANie2GK36w40gmjwGzs7GyZL6SXtRWS0Alrgf3YC90bkjTCPM7N5qOwiSehE5hXpZdxFDqBmVkowlq4TkYrSE0hveW65sgHA8FAMBAMBAOXYIA3tb4ZVOQnF6sf4GqIfHb5KZM8dVPSvtEeVEQ9koTR7KrIr5qYd6lPLsAbc30j7mUXl8P0HzDwyXdWU3WN75p4ll8shmKv0fnkag/hThhcvpcNlgh3GJvCrfkb6T+5HGfWXA1P3TO7HmYU52MTq8LMzvlTZntGVlEbK4oRr9LLODifXHE+9xgaf+/8dn5NMSObk6TxvXRyPEfkuWOVXgZYIqOKOlfI3NWSxqgJBoKBYCAYeJwMvPE415rdijee0mTx/Rsdaf8GOjvsjIQ/8+j6Otk+R4r82tV4M1t+dMGN7MNM/Vy49g9NQb13jcNLcTXsBlYuNjZ2cv45JndgdrYm5hSz0T81G8X5fgeXmjWZXSXJeEfYuxj2xvmXNMuJYd3EHtsT9VMp6tszRsV+3hQfGj9PDZuIMStZHI3v+bTUlborDv0ZYQcDwUAwEAzckoFv3bL/vtvfnDlwPRO/6zDn8kaY3UG97PaNtqapa984i+u9tLnSQMC5VbYy/A5zu7qWxeZOlZ1QhCwMAsJ5G0Mx/3nmZv+brC037f3Nxl3b5flqi/k35Xx+bAsW+GvVlIm61MRuMpsR9HEGwnPgOU1JVrBMJSIWDAQDwUAw8LgYeGof4D6boe8wE79pOKuxLGjm3Lk336l2Zr4/lZiJrWbiS8OjCo9CWdpwoi4vmLNXDR8O2r2JsUdv85mFJNOj6VMKnqsczEAz95LS7n7d7D+rYGlPVm2ZGMg9pmbMvd4nRpwMjcpmoRP2M5VrxfmpJnyyT0gwEAwEA8HAI2bgqf0KdScuNw2f+MRvI7xprWxAOmMQ565dL61ZmJtRlOO/AEBNleT8g4tj5sY/12U/+PGylVPves68tKD4qJq98LHp2gJHxfXX85OLtSaz/J6+ltxvhE3T1LVDrvEPTZ4zwDmyU3H7obxzA3je9R7+Pv4MZuTmUO5G/BIyagg/KUSwvbBTPYcPo0j1k+y1xVqVFejaYPjBQDAQDAQD98PAN+/nmIud8kdN+r7wtvAvBP5c1W+Fg5CEX1mON6V/L2yFHwv08NOMUZiSzxVkJr+i/VfC74Qk8K/l8wbWzvueYv9B+ItQBM5hH3q/LbDnnPxBiV6g/rsCs4sVs8ePhP9XYBbynjB1/t8pzr2RXuCe7OXjzGNH9mPmuwK7jUIveG6SfPjjvtzhE4H5vxT+Kvi5cmdlVOYtAQ69dHL+L4Ed2OlnQt2vfW5Hq2Hv2vOF7H8Q6m7c40dCEnhufy8Ugbop2Sn4tsD9eHajQC3P4PvCfy7AORzyYYeaLPDhjPw/C+zFrpy7Ev53AY44+z0Bjrk7MWoR7vgzgXp2Jt4LzCX2fwijACed8KXwE4G6nYD0Qn2+7JUE+tcCPuedEs75rwT2aOW3CvyjwPncswjU/U5gB/LYrWwU+NFMrq0NPxgIBoKBYCAYCAaCgWDgTAb4AJdmevYWz9K1rsbWim0tjh0SDAQDwUAw8EgYeGq/Qn0ktMUawcCTYuCobceZjYnz4YwPb4APbJ3VZulRII8dEgwEA8FAMBAMBAPBQDDwSBjgA9og5GYfYv4D3SNZN9YIBoKBYCAYCAaCgWAgGICBzQQNg8X2M/mJlggFA8FAMBAMBAPBQDAQDNwHA0mH8OvTVogj6Km8pUMFA8FAMBAMBAPBQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDAQDAQDwUAwEAwEA8FAMBAMBAPBQDAQDAQDD8bAswc7+WYH92rjb8x9R+BfjecPVxcbtZPmH2DlX77HrnFLL1a9KrOwFQ6Lu+6usO5y1BHd3R1z9mR2AZ4nng17jmdPW9bQqywLj+XZLNt6umqlMHyV6fTi6MZmwPt1klUwGq6rvXSes3lu7NldaDhzmMs/5Mz3A89BkQ+q9DLgGzmYpn4lDMJoMa82crJQ65PsYvB1p2x6+Mef+b7EeRV72Z2wE3ohC1uhnoU/GqTuRDpN5Y7sMlxzQlae3dj/LiTZ0PEuhsfMYCAYeBwMdFqDf61+SvhmuJpKnBkrqs9n9qQz6+fKp+bwTbbMNTxgnJ2yO3/JG8Ft123PvO28h+rvdPBt3wx5rfO1wOtjiRxUxNfIfUiaOKRTrEzEbxPiHDjIzZBB/odNrJdfmliSPwq5iffyhyaGS6ybiE+F1gqOAtoLPs+id8EiOzu/zbvURU12ANfJUQXb64oW5tNE3V4xEBIMBAMLGfjWwrpatnH1RTZf1Pct44kD2echdko6F5za7cTaL6U6eX1T/BB3Wrqvr/Ovj5v0f516Bl0W3EZ4XZzzU/Ss+vt6LXU6q28uNzb+Jdy5mZz9qfCRUE4cRP8g8GGP/wwawuu4E9Jz96X/JQ6Ho1Am8jW0kvGx8FPh0NTh74TUxL27PpF7iBT3uZRkDRqaYXAeEgwEA2cw8MYZtZ3V7qX55lL9M0a8tqWX4mL72jIUF3sMDBzvaYnH8Doe7a5p4Z35b+JW6WUMJ/rIXXfHTjX82pTvl1MyF5+qfZ1iWZdJr9OF4i7BwEMxsPQncHzB8Q2LD24I33xGsx+rylqsE3jTYt+VwP4H86VeyMYsapNPmE1f7Se/E0bLbaUB8SwUg9SL/wQRc5nh+8h72cjpBGp7YRQGwUs2h1mjsHdJcsTrWb3ZruSF2ckCtf8gOwuDMApVVjK2FmMu4s+sdegs1Fp0lc7ZmPSvBeJvCvyEZBCS6VG6F9Bzwl4bgZ2ysDO7kwbE2WE0zX8I/vcCda0kBXqBXfhpCbsh6F6o53SymcnsKtVfKTAKe0t00vTxZzV/Y3Fy9c9u7mTTy5/ZfFdAOgPxtTA2PjU17s+hj56D5XuzqV8J9Ywsmzp6ByEJvcC9fyFQC9YWl7qSJHQC87FHgZpikHpJNvI4ZxR608NLFV/9qjArzty9yxNj/tF0b7YrWWSyBx+gyoJqzhusbiUNZ9xhTtj553NJi3P+eKKG+w0z+aw45++tBr8T6CGWDOyBf0pWSjKLWmxkMF0V8WzOWroI1FfpZGyEvTBYkJ6tMArYaPJVeosdLUCOGcSJ1fwgOwudxZmZhF7gtVm/bontBISaUehNXwUlg+m19PsCr+uQYOBrz0AvBgBfgCA5RrA759+1mXXAlzOH9E2cWr5Z8AVdZS+jd37X+Cv59GRXU2Qn84kfXA6zCMS9MIc6NJKE0ew51SsBWskK/FVIlqg71jrypTrS+IPzp8ydgnuXYOYoJIvhF6HuT7gXNpZHFSE7nxyxKlsZg/N72cSQtTCaXdVOhj+vSV+5Ragz8KkfTeMzA3jpG3/K5TXVuUSR7fs408/F9/mD/LXrT7KPAhrpBZ8n1r6Oi2LtGX9SLFEsWQnM9FLkZAugq01oL/SWQ2F7n3ntvQ+K+T1H+dQhW6E3+5SiZqouK+5fx0m+vw95dq6ykTE4f87kDnygeseAzRx/D3p7AT5r3Y9l7yxOHskC89BzUmva+b6ee/VzAybiRTHmVullgCpZxij4M7lj/6Ji2ihNT2566GeOF/zUxHr5oEqRkRs/mU9/1+Tq3r3ioJWsQHHBlWyeA7rK1vmckV2uOH8je3C5MIOB15KBNxbeKqkuC0XgC2cnEEP4wsxmP0Z10FKgSv//s/f+sK5cabYfpJbheYnFiTwDB9oK7DeOxIk8mXZHb14kNhx4MpUiT9aczIADlbIX2Gg24GAylbLBS5odGGhHKtnJGDDQ7Mjz4EClwPDYSbOdeB6e4fb6nbv37X33rSKLPCQPz7nrA1bv7//+9qoiWZe8ui2FN4IsnKUr7L30Mp/Qtoj30j8p7CmVPehDP2RI+jLZpy6/ST2oo+eHRQP2qmf8/MgG9GC+LNj0oBfSCMTxZyG+KexaLXOJtUJXJNEPIKyDsEr2Qmsv1D1S+I0l98BJPnVNytgUOi78+OZIfyCJPZdFHHuo7FjYxFbCVqCuF6g5JvsigXy+RRqSj1h53cd69cm51roQ2rGk5Mt7dUUOPuqQIHwk5DzmiQ+R86W8jwe1qe/jvmi9lX7sPs7p9P0u4edaVwLz1gKfOY9vd9ZCWyTls9Z1pR2SMRxIYu+cdyDtpBD7lWdidjAlyxQoa3r5vhQWRVEZx429KeK1Sl/QFwH0Jvl5MO6K2Fb6vrDnqOR/IzRVMn72Zo++iKHHZLNfU8SsmoEXycAHM08VlLcT8osQvRUagRcLeC7C7Hxo8CaAoA9Jn1o6BX4q/CBkDqZysz9KGQTWLJ2UufVF2VGVs/RCLDK/Olr1dgKz5R6scFUKNh/ocyQoCW7LHtuqsJPdCPij0AvnyKCiKGwEdPZshE5AONccGaqk/kARsZWwFnLdYqSes/1M+HEVmzL7qcAMf65dKrcVwoyaQymDgtzznAsO6dsfKnhkLKb+rFnOuY8fM8ZOxTzkMUM/0SjIn98L/hvp/3GR999K/x+FQSDvkMAn+50rgwp5jUWhH2myGvFl16FrSS96TgmxQWDNgg3o+90f3A/aprLnml2qpb6cdyGbB/ZYNILHfWFbNQMvnoG5D3BjL47wROwcepEeiuVxefEjw8z5yf9W+EKAh1KI1Xs28nVC5qwvCkq9cI+qy9RjNFg5h2SX/Ut9To+ck8+Te5a14YRGuZ6amrfcppOyEcgBNZc5b846FEn05IG7F0r/nD5zc9iDedtUwPVCgjAknQU9Co3QCdeWhTb4WmgE5stz7c7ceK06QD292xP7sP/cvcnbC32xR6mfuPXZ6VtVfnqgeqVYl+L/ndZ/VuT+70knzvsGnHGmMYlyzuVmrJ7eyFSPYawo+aZmIrw8UJf348GxH8mL8n0y4h9z5X2m5qc/ZySPvpuiCXsQt5iBd5aB92eenBdKqHKHwq5jM9uelcaLnT99NVU1L/Qxqb8xWimJPyHuE9DxlVL2igrwJ3L2RUJaWZZJHwo/fZGtUPeNRV5Ke2PJtTjLGcZySx975VmyPx4rGolT0yX/1PzfjNRNufiJ6thc7Mc3VP1UkxF/zQ08M28W9I+FRuj/4L6oxgNiuWdM3dkzSysl84hec1GkXkzdqBP3dJ6NPYdHdKe+Tf1Y58i+SFrMKUg5zByr/NUJ9ZdKXasR9089C/3zPJu02aD1fy3wfyd/r5WfcnmYHpNGzm4scMB36L1srAw+QxVYyub9czdWkHzkdAfi1L4nhConyibG+2UZW8hepdxBK/0R/NiHZKNgW+X1sjlDqArzHvTNe1QpNs3Ay2Fg7gPcNr1YeMHw4ohCm2hotBK/pfBCBTFtyroWNiND8GZCLhKTnm18uVeQvhDog3wmBGGX1qVW4lHgweRzIctWCn2CQA7SC5sEfLl+eIiOS+6T88kKArPw5s23AsSwEVZs9tkL61fuhxnxHxPyci4fNL3ADAg6fZtX5kMeyHsspTNT5ilIZz58xJBGWAnxlflgD0nPC3v8sbCr/FPmPgVCWhut9Oirgk72UPnGzIWcNZ9RPs7BNQ4J5TWgzzdCruOMvfCJMAhR+EWymRdwPjjOfeo9l4od2nNszrqGOej7rcC1QP9SYP8g0B8fdWP9YsqhlnyE9dcCPQHnov6QbBVkFvbIuUF6Pjv9x/bfyL8X1gIS0zq10JvzIfTMe43lx5STOWD/KWEGejHHqkhCb4QokHNMqOdBYyvQD1kIjbATco8onbky7+R+kpDrZD48GNETiUIjrF6Zo/9Lf3I2wkIIAvX4SiEvJgdx7O6NjDcN4iuBvkGgN3U7Ice6IkbuVkD65A9alwL56JwdDvCV0smIQv+m+/W5gvwLYV3kNCP5VblNM/D8GXjvxCPwwkB4MfLCy8ILqLSL0FXVlbrzgu8T6s2iHK2Q85hxVyclmz6coxeyPkgHSExrr5W8IJS9sEEvlEIu/YaEKvyWGeQB9N4nHTsLfvqVNnlISMi1OWdsbZOzO1KT52cP+mZhBmLIkNaQ1qncPsXLhR4roRuJjbmCnIOQ92cm9quFnv1ErMxlf3ploV9pDykQipw+6fUM2OSTS1+kT2tMK8v3wseFzZ5ByDVDiuHLUs81pybX9lLoBZC9MAjMm6Xun+PktEXeKtlRK32mJCgA6EtetnN+vV/OI55zS1+uK1dmWxQO9qFmTGLlPNY7p7MHteyzFab6V+3fMKldCUHoU499kRELfZBOPkDIY88otGlFz36ps4QapK+y2YdeIWHQCmppkyOvOR6lTM0yFWPPpcC5yr3pOdYryD8QHJGYfH0Vy+caKbHLDJiB58BA1JD9cxj0CWZstSd4Cslv4Oy9uuAAfCjQG1lfsO+72mqrg4eRw+OHa8vtGIjaqr/ddq93yq/PVh5gMQNm4E4YeP9O5rjGGEFN+Rki/zRxjT2ea08+fKd+srjFmRptAsKFN2tTTz50eMiwPI4B+AShaNNIH4Rd4bN6XQaC2uefVHnd3krYd5M24z2ju9XG3scMmIHjDLx3POXZZgRNDrL0hf6uq7wZLxIJe61P8WG80r7sfcnrwpnoS89BsFyGgag23DODwL3CarkdA0FbgSx9oV9b5boD9hyuvZn7mwEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADFybgR9dewP3NwNm4FkwsNGUfyT8idCmidHXwj8I++TzYgbMgBkwA2bADJgBM3AHDATNEIs5yoe15g7m8whmwAyYATNgBsyAGTADFQPLwo7St4VdxkycGTADZsAMmAEzYAbMwB0ywE+p/GxqMQNmwAyYATNgBsyAGXgmDOw050v81q3VuRY3uAaP5S4+YkbOd4szPmJEl5qBu2ag9Wvorq+PhzMDZmCCAT789xOx5+zuNPxjH6zmnH+jpMc+QPVzNjqQs77ADAfaO2QGXjQDvH47v4Ze9DX24V4YA1Hn+VpoX9i5Tj3OSgX9qUV3ns815aHm2tJog3iBTfpH9sgfQI9s43Iz8M4ywB/2Hvs6vAl5799kF29iBu6XAR5akI/vd8SbTBa1y15oBfSXIJwDbG5wmHt5+OUa7tK5b3Bsb2EGXhwDvH62wi3eN14ceT6QGXgKBnpt2j7Fxt7zKgws1PV7IVyl+5tNeXhrLrRPf4E+nJ0PIIsZMAPnMzCodHl++fUrP7j+Ft7BDJgBM3BzBtba8TthmLkzb9QfTuTS55DwALeZSAjyR2Ev9CkH/VRhjyzbI8X0D0dyHDYDZuAwA63CPxN+fDjt6aL+CfXpuPfOZsAMXIeBhdr+VGhntm+UtxZ4o/6bRNZ6eAAAIABJREFUtH6R1jlv3kG5O6GWRg76dsIgkLMUTpU21eYHtzijAXvNyZvRyilm4J1koNOpP/br6J289j70M2Og17ztM5vZ444zwEMT13OOBCWtUiIPV/kBix61LFJurAK/rxNlB2EQqMlCXmkXoddqP+LcFb5WehjJqV3kxdpp2wyYgZMYaJT97UkVN0z2T6g3JNtbmYGCgY30Ty7EyKA+fGN0SHgw4eeASwn7DZdqduE+jfq1M3tyBoCshFaIQvnQRIwHL67ZWmhTfE9A8l1ay4W8Xsg5S+m/KWxygxCFDuOI8Pf5euHnwlDkttKBxQyYgcsz0Ksl/0JBEIbLt3dHM2AGLsFArybtJRrN7MEHOt/IAB4MTpGg5JXQCTwg0APfMelT7m+1Lo4lF3Fyo9AKg8B+nXCPknk95Xz5HPlMjRyxOlwrG84R9Jh0FviopZcj5xNbC5sqCTtWvr6yMTkLvTphqOK7ys5mKyVOxOw2A2ZgPgO8xnj9WsyAGbhTBnrN1d54Nt4U8kNcPHNvPty3Qjejntx92vPbGflTKY0Cg0C/exM43Z0xVKMaapFWaJKeF3jL0ktZFvZQ6Fllhlj4t9JXAnVBaAV6smJn6Qsd/y8KG5W+SBQ6AbvBUUknu5yxjts2A2ZgHgNrpfXzUp1lBszAUzDAC3TzBBtvtScPcad+K1aPyuyL2jlix7Qfe/LGdK7wcPCY+nP3PVYHn+dcx53qQmrOubpioyideJuwL2Ko7BkqHzOsko91EKJAb2Qh9Ekvl9IXFWiKIH1Ku5Wd9yjSHtS+dtypHe9kLq7H8g5muRc+HkNFuBMuOcPU6wOeueZzhNyxb9nn1DrnCRngBb0WWuEeXtxPSMWL3Tqm68s1zsB3K+FNZC/wBvGYb8XoM/ce3aT92HNuzRgfj6mt+8XacaYNl6szatuiJkhvCjtKz3F6d0UMtUko3VwPckEQ4GqdVvLwtyiV9JUdZTcCKyill8E+teDraucJdqtcZi1lI6NPGNIa38g43WhVEk4vu1rFU89zzf25J76vmOOe7IRB6IWdsBUeI+zDvXIvMjXPlH9sbnJ5r4xjQfvuk4F8czMdF/CxN/Z9ntJT3QMDUUPwBgHaGw20S/vxps79/ZTCa+2xD5PMH1KfeOHD0K9NPXkfCFX/c94f6BdTr7JfX/U+ZHINkVglNbJXlW+uGZS4F9qiIErnjE3y1b3xk7NO8TkL+dTdm3RPNFDUvs0V927Vm9dYKTEZXLelkO2cgz/HyrpDeqcg98o9Cfdrfc8yXxSamYPuT8id2dJp12Rgp+ah2IALaDED12KgVeP8ELe81iZF3yCde5o9v77Bfse2YJ7HSlSD+kPqsT1zfStlLUxdG2LsP1dWE/36uQ2UtxWiEKqa7oQedSrn6IVNHZDdJl9eMYOQc5lnrnRzE2+cd+p1vNR4p3B36p5BBZyL10asirFXQlPFWtkhAX2OkD9238ypvXZOP7HBlL9OJ6+tnbbvk4FGY3HBsiykXOuDodjG6jvOwC7dZ99r5Z67tjRpP+5t9OcufBA95eu01f6PvW79Iy8CH6DnzhBVuxQ4Rz0HsbWAbNPKwn78o8mfVn76TElQoJsKPrGf2crz3WKca/ORr9vYA1y+Dq0OuioOu5PONeXa5vqF9FDk1Gorx6HrXuff0ma2OLIh/vLcIykPrl649X0xNctr/wdHM97NBC5oVxydm5J/w+m5SaOB90IQdkKfDhC18sL8QdgIjTAInYA0wiKBOnQ4WBNMErV+KXwjDAI5+DphL5APqN8KWYKUVfJHrRuBfKQRPkr6V1qD8Hmyf6mVXi9ZMi+c+2vhJ1c+bKf+UYBj/o04+H0sx/TkGvL/aLAUkCC0Qkw6/k7Ie22kfyJQg68RmGns3jo0H32/E55KWm3MDIdmPDYbPc6VhQrhcn9mA2anfjVST2yb/GV//GuBdZfiQWsrjPUhBX+PMiKLFGcP9G4kZ64rKLERmCv32x4pHhSn7hqyVlPmyOjTJiutWa/3hYPPhPdSzlAnHLFjqiPtd0Ko8odk90VsKR2eeC19IbQphzmZZ5PsemmK3DoWUy31g9DXCSfYeY6y1/5I/U7xOLIvc9Bve6SePOotd84AN8XvBT5Av0z4VuuxC3zJY4Vi7zzD1Dq1b6PAugjmGzi7ohR83LygTQHWrAfp5CDLtJZLK6MvYuSQ3xRJQxHH3QkhxaPWPul5aaWU/3QC8UWV85JNrgX3H1jf4KBwyzVjv19faD/6bYpe6J0Qki9q7ZOeF2rwZ2mlbIWQHFFrn/SppZ2RM1X7rvu517gXkEb4PunHljYlcI1zPSs2kn3JfFhaIZaOQu8KvTmQN1H+2r2U1hdJzNMdK0px6sLM3Llp9GQmZCXsk87SCrGws0r+VggjsbmudZHYS29nFLIfeYsqH18+w1ibfswpXxTyHPQs398nSibdnSJNigatvG8dmik3IqfPRrVO+cu09kD9RNvruz+4/hbPbgcu9A8Cf/LI8mMp22QErRuhFXY54YSVG/jQzUSrQfjqhJ5jqfSIRYD5sfvCx7ce+Vx5JadNOYNWcoJw6Kw5xlr2pA09OHOWfaH30j8tbNRWwN8I5K7SquUt+ffkee8t7304/l+N8f+dMQrXgX9t/6cCD+29cIj7M7Z4owSOG4GHt6WwEdaPbErPvuiBHYSh8HGflELOmAzJCQfMdwn5kZqAd034sPt3I4cO8g1Cvgbo+ObIVkkrYSPk+iid64UfNEIpSxl95ctmlBKEQaAffc6RTkXronAhvT+hUVDucEL+odRWQc5RnqUvCqL00ibEvDzo8Nkz4DhDOH93Rt2gGmaNQlvUL6X3yV/HirS3VHK595C98E3ST12iCpihSYX04lvFktepnodywlTRvfv9APf2FYrVDbGQzU2zTamDVnysU9Io0I0Eo3yt0CeMpFzMxR6DsE5rSGu5wXelkXRudM7bC5zzB2FIsbnL/kAi8zQpPtV3pThztMKhXsT/JPW6t6XTQP/TmUNxrijwkMMKF9cU+vPz5c8EuOcaPVbq61afYTFjg7KGfh/OqJmT8p8r6V/MSXxhOf+o8/zXI2dqkm+Z1jCSM+XiGtXXNsrH9cLfjBQeuva98vn2bxC4J+mTZStlVdhTKjns0RcJUXpbFUz1G6q8x5prNSjnZpa+arqrbHL+WGiSf9DaJX3OEpQE2LuUWNlTJtyMCedohb4KxrHk5KMXaNP6RZFLHSB2TDhLVyRF6X1VxHyLKq9KeTmmH+DGr+WucDfSN8K+8HGDlHbdJdSOZPdawTEJSvj8WFKKfzWR18jPzQyQZVqPLb0SotAIQcj1Uh8t8PatwAt4d6BbUKwT1sJW2E/kjn0YTaQ+Kzfn5dxwtLnR5OzXCqsb7feU2/xrbQ4sr17r+V4r+eD9Jwr9GSQtVLMWulRL/7nSKJFa1k6gFxLSOmdZKmlXJObaQT767QVysMckHIiN5R/y0Ys/ePRFUpTeFDYq8/SFD7sT2sJ3irpS8roqoFc8pUmRS10vwCuzIdhZSr1wP6jUBIG6jdCkVcvD+80WZYZE5VCfBbsvbFT6lTlVeNQcRr3PwOkHuLcvEjcbNxqyELghQJYopReCgJ/cRrikDGo29WA2d59WiU2RHKQPQhT6wl+rQY517byQDV/lV97slSVK6QU4h9NW2At8K1T+iU3mi5dGJ4SreKOTwjk/17Dn7kZ7XmubS31Ld6357qlvfq1tLjzUNvXrtA4jvfsRX5SPOZbCPuncj0gUWgF/I3QCwvxgSHZeyNsVvii9TzY6NY0wpLXTWguxLCEppa8IH1TrGnqBcr5+pAPx1YgfF/MHoexRpkYZfelI+qD1oxH/HBecdgJrFMb2Zq5atnL0wibVsOa8Vnoj0HNI0PJwD4z1J6/0r2SDRuiENtmD1txT6oNEYUh6uYQR37muhQrB2D7n9jxY9/7B6LsZ7HXskI7eal0L3AxZYqF30jfJxv9pAi+SrLOGlHPLpddmy7Qh6y7ZzAL4eS7PyU2XhbxB4JsykD/Y/5DxSgtaqOcDk/70+CwlcWbsKOQ9gvReYAXISvhOiMJ/JFDPnnsB2Qrk/FRgjylZTwUu4Gf/cIE+c1twTs4ThczD3Npz8+CZv3fXn9ugqIvSy2vOebC539AXSaeE+wSJQs4hPlbDPYDkmmS+tVB7D8Icj52lecRBwpH9o+JfC7x+S4H/8nUcqvgcc5uSeq3DSMFOvlj597I3ha+Vvk52r3UQ8HXJx4LO+0Ut7J+5Z43CkHzs3QnsR3/0WhaVg5xf1Ekn2Ly28hytdGYoZZARK99WNnOE5Cee52LusXMT/1wYm5X9eY0FgddQ7iV1ljDzPmX2hV4WE6/7ktulpKCVObK9kU7fVhgEhPpfC+tkl0snI6Yc4kNCziHeC23lJx5SDL0U5tnWzjNt+oxxf2a742X+Bu5tjvZytcIqrdilRBmbFOMm2hXB95LOmnVcdY+i5Gpqo85RYMatwMy9kGf5jfQvhHLOhewmQctroRahTynfyABZ+Hbtx0LZkz2yDFJiAnPQtxOWwn+fVv7eyyBk+UmhT6krBfKMOYeefBCxLpKTPXfCL9Oacw+t1FM3pKSolTfA/E1ikM4bI3win6T1q7SesgQlb4WVwJ63kE6bwAnrpaS85vQs7xFsuDp0n8ytqeft5fiydj6BvdCejbB+5N7hEfVDsT/Xd0x+PuaUL7+OJ8KPdvfq0FRdmJF7Hv8gbJKd05ZS6nOQC2oZ5IB7Yn1a0ZEhrQut7FcL+3SVcyW7rRNPsJmFvoPAnn1Vu5XdVT5M9o0JvdY8b5v0oHUQsiyk/CBMvWd+p1j9uivKH61yjiiwZtkk31LrTmiLWJTeFzYqZ/xzgbPU0sqxEqJA3y7ZeT/8vTAmxNYjAfz0OibkMf8hYbb2UIJjT89An0bgpglCMzJSO+LLLmKH4gdKrx6KE7NN+a8+0MwN+iJvIZ0XJC9WXlC/T3pMa598X2udI62SYpGYr3t2EcdXyq6y55jMzZ8845zkC+XAUXehXvfQBu643uGJh4FTrudjpX1sA9V3F+hxjRYbNQ0nNO5TLq/pUuIJPXLqUgr7c43qfp18YaRnnTeSMuqqe+2Uxf61dHLUuXVOaZ87z6Gej43B5/aEJq1yY0Ko6s45X6cecEtPZsmC3hX2Mf9I6lsPn2M5+M6Ze6qX/RdmgJujST1juljljZK3ayf25eJSD+7xQnOWrRCFLEFKL7BeSi7Zi5n6YjD0mGxWPtDra7RO/iblHVpaBWNKCFrr69bLR79S6pwqPGrCezMamefkjHFe6kMWM+4E6s6V5tzCK9ZxveMV+x9rHZTQHkuaGb9En0Z7nXM/zhzx7DTuu+6EariIVX4Y8c1pyd70q3lZJn/dg32InSNDUdtKB2NyCh/kNmNN7sDHXHO5ynzHau5zz0YfakPVrx3xkbKe8FflD+ac95WgvLlnH9vDvjthoLmTOc4dY6VCzpDXc/vUdfkFywuH3peSvmgUC5099hOb7OQv6ybSHt5wYwqOzcwLm3OV0lT2MbNTwuZY0pH4VvHFkZwcZl7OPzd/rC091mOBJ/ZxrvYJZ9hob7i5hFziHFzj7SWGuUIPeFo9oi9nu5TQaz3R7DH7RPUEnDNM9M/uuXw8Zp4jI1wkvFGXx8x4jKdThoR3UEuUo6mdE3aQf843+48588TWdpuB+2Fgm0bhRu8uOFY/0auVfyrG/lOxsh09YukodPz7idhcd6PEzMvcmjIvyOAvznYzi+F+EJYz88fSGjl/K4Sx4BP7Ntq/mznDWnnfTuDcs+0O7M1+K6FJ67E92pFe1FAf0zqS8pbr0ExvJd/Ywf14D+I5LnsVwmXbnd1t6rqeMl+j3YezJ7hi4QdX7O3WZqBkoJExJMdS69QL65KsRTXbTjQM8vcTsblu+j+mBx/ma4E+pwj5H6W6JhVO/cXlsi+c88DCnrsycETnen0oMC8Iwi+FQbg36TXQz2YM1SpnEL4SONO2WCkndo7sJ4o6+UEvBAH+4fVUaVXQpKKN1ij0yZ5amCkIw1TCE/qn+Lr1SJ7jsozfy702dV1PmS+Kmv6y9LibGXheDGw17lqIAh887QXH7yd68eJdHoiFFGu0Mt+YtHLGsYB8vcCZzhHm4lssvpp/LIaZA3DGx+6V65uZe946baEND133PE++L1hjco5dS+L421x4ZO1H4o18cJ8lSBkKe0ptqwCzUBeSP59hqj77eynxWJLjZsAMjDIwyLsajTyx09/APfEFeIe2j8WL4JQPxHMpYr+9sBtp0MoHhhTrtZ7zAv1UdZzlHAkqmvpnHE7tN3bGugf7kTcnt64ds7sx5x34uOZboREOXZvMw0p5rbAUFkIp2MRyTpTev5HxpkGPMWGOTRGI0vsqkZxB2Fb+0tyl+Pda90L9rSu1q5H6YcRnlxkwA8cZiCnl0OvyeJcrZfgB7krEuu1bDPDhgwRhIWQ7uS++8EE29qJr0v5tsWOU3hf2HJUa/r2sc8/BbGPzzdn7nJxBRe05hc+wptPMYD1j9pByuCdr2cjR1c4D9tS9wL8NWMai7F5gz33qh+/YvCHlkAdagTpkKeReyfV6CdKG2mnbDJiBoww0yuiOZj1RwvtPtK+3ffcYGNKRW63NDY6/0h5dsU+QvhH4kKs/KKN8vdCknDou91sS5enf8tpxDwxwXQahOTIM8V2Rs6jyuYf4mZtvWsEcqXtQUz7oE/9M6AX6Y0dhKYRka3lL1vJsCm8nfZ9saokPAvqYELOYATMwn4Gg1JVQvu7mVzvTDLwgBhY6SxRYLy191ZAXHH9X69uETmtT5ZTmkOLMFoRyxlZ2FLKg8wHMz1i/EPhgDzno9W4YiJpkODIN8ZByyOeaZuFBqC/sIencG1shFLFSHYu1SogCtSuhT/paK8JeXdLLpS2MKB0gQaCWNQv70n9MdmNO+8yAGTjIQKsouFvxT6h3e2le3GB7nai/wan4EOPDDAwJh7YNRU4rnQ/GY8I3Kl+kpPeOJV8wHtWLB1PkMT/fnjNSUNFSgN/unAY3rum1H/dAc2DelWJDmov88u+UcU58CHld0vdpHdJaL+wZha4ItNLhLgjE+2RvtCJR6JM+tRAPQhSYIdfm/EXyZzuvKyld7bRtBszAQQaCorx24sEsB82AGXg0A/0jOjSqXad6+gRhWfRrpcfCfgqV+RqhE1qBNxY+wAehEc6R3QlFC+V2KR9u2hNqnzKVuXuB9VQJKmhTEVznHpmLKB9+1lJKrqrQpLlVJAgroZy1nax4M7CUySzUxqom+2e2cpoZMANioBd4XVnMgBm4MgP9I/pH1YZU32gFpbQyYuW7hsmH71pgvwzeQMAqbdglO5mvv43J9tw1KJFec4W5miJ5N7fwDvLgrz9zjqi6RuDaZOFabJO/cL+hkpOv2VRO6Y+pH7OW0lb2IZPcek/6rQ8VOWYGzMBbDPBaat7y3qHDP6He4UXxSDdloC926668c1D/nwrlB/XfyN4JfNBuhP3IDMQR6rKO3QsfPkTmS1Aqew0C/ViZaUq+UWAhlPuOzThV/9R+5ua8m7SeMk8/khxTn1brVoCbocrDz55hJFalPpj9mPNEX1vlM9dKqP0ntnW6GXinGGh02kHo3qlT+7Bm4AkZ4MPqWnLJ3nyo04+1ljFfmUPdrioKsvvki1o3VXzK7BQIU8ERfytfLPy99DCSd88u5r3EtVylQy61Zn3q3JfYb6r3Mf9T7n1sNsfNwL0ywOv62Yi/gXs2l8qDHmBgfyD22NAle2/SMKEaijeNKNQfup18Q8oljl3KWgZABmHurCHlv6p8tXf+jyOyL6+/SQo1WZhzKOznoF5q3m067E4rOCRzr8ehHufGnnLvc2d2nRl4agaOvaafer439vcD3F1dDg/zDjCw1BnrD1feNAahPXD+VREP0huhE/IbTpTeC6xB6IQxWcg5CKz03Kak98aSk69PuZi5/kC6Q2bADJgBM3BtBvwAd22G3d8MvMlAlNmPkIJvLWxGYtTwb88hQ0Kb7LxEKfw7Zj8X9lWsNIkNQhS6FOjTOrUQzw98QXozlWi/GTADZsAMmAEzYAbeNQaCDtwKfDMWBey50qX8Nq1z65xnBsyAGTADZsAMmAEz8AQM8O1Yk/blW7zlE8zgLc2AGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBm4ewZ+dPcTPr8Bg0b+RPhj4R/T+FHrR8IPyfbyioHMy+9k/pNJeYOBpax/Lpib+7wx8vV5T+PtHzniWvV/P7NHVB7vJb4vZhLmNDNgBl4+A62OCB4rIfXpi0Yr6b9/bOMZ9VE5v56Rdy8pzDsIrLcSrsUvbrXZI/bhAWEQYtGjlQ4s12MgqvW3M9pzfXZCOyP3UAp9eG9YHEoqYty/gxBn5jvNDJiBF8rA+8/wXLxxgUsLb8bgsTKoQV812T626cx69v5uZu65ae25hSN1vXzDiP9SrrFZ2e/aHF1ifu7FmptL3aN5vigFPFdprzD4oJ71/cHDVaz24lpc4nVNX75NW1X9p0z2HKaC9psBM/DuMPAcH+DmvtGdehV5Y7zEG/Kp+14yf1Cz9SUbVr3CFXtfo/XYvbLTRptrbHaDnpe+R5c3mPlaW1zrXhw0cFsNPXYfXepcCzXqhGvucalZ3ccMmIE7YuCDO5plzii8yX0ufC/wswN/p4w3wA9TMfYgBIG/J4L8RuDvpCH8yZoPLfLLP2UH2fnvleweMv8gMcXIHwp/kP5p8tV/Yi/SRtUoL/MjzLcX8lz5DCn8egnS8pnGzskMUaBvPQ/+fGb2OibMAmfMhsBJEPjpEZ1z861ByRVxauhf7y/Xw3WiDiFezxHk43xj5yc2xXXZN89L788F5sl75lh5Lyj8WpYpn7zyXGVOnQ+n5FOLUIeer0HpTykPH9Q5zpy1BDngIc+b49lf875QAmdizbxmH7X4mCPPmvdkji+FVmCeMd7lfkOiLGaj51DF2KO+Z3IKMerYY1fVRdl5tiHF8DETHHCWXJvjQb5fCPT6VKg5keu15LPjYH/6sR/COYJA/2xHKexNDKH+S2Ej4B/jiZ6c/dAc9KplJcc2OX+tNQhDnZTsfI76vsjpxDkH8bJHrmPuvVDPGeUrzyvTYgbMwHNg4Dl9A8ebJG9ArO8lZI6/kPJNRfjPZX+cfKy90CabNzNs3tyy0GNT2OyzFdiT3uTmfNZV8vOm3QvkzxVyfyn8uCiI0pn5kBw7J337qsFG9k74TkBn7kOyTkHOPAjbZAetfDi8l5DcD0sjRIEzkdMJzJJlKYU+zEAOe4TX0Ve8Yo/NSG0UmKfmmhp60RO0Aj72Zk6knvdj+foUy0snJQjssRfaHJhYO/nZ4wcBHcHOYJaVQC8+mPEjrbAVvkvr8pX74X/J6YQgEI9J1/JafiJtU9gh2XCee+JD8jmjdOYgpxeCwF74mP+9BC0HpVV0J3wjMENTZBOjHzEEO8taShTghBxqs2yTQoy8JtkLrfTCRuds1K1SPGjlPHNn/0q5f5Nqc+8fJ5uFvf482UFrX8TI/50wtddSMeZixkZohblC7S7hB635fGU9+3cCK3uQE8oE6WthL2QesUv5uYxWoDb3yPZOPs7HajEDZsAMXJUB/rQYqx2CbN7ASmkqm7pF4VtKHwpflN4X8a30prLbZLN2RayXzptjligFXynsX0orY1M4qA9vprxlEeec5TnWVVa5D3nluaPsvsqvTeJT/VvFQClLGbvKx1n4hiTLICUUNvnkIL2wSTrLWtgWdlvZvexcSy7xLEFKTAZrzXnOK/2NnF0OaKU3801JrOKt7HJ+6nohzxhwSJZCvW95TnpwnlJ6GbFwoOPLgp73wYfevY6+2q+Mkx+LeG0XobdUZucMSBR4MM2yk7Io7CbpUSuxLNj7ZKy1booYao6h91Wc/G2R30oHc2SlpHLeTjb9shAvpb5vegVjlYPZCl3hJ6cv7GNqUyTARTljDnHmMg//Tog5QetQ2fX8reIld9SW7wu0qm18FjNgBu6YgffveLZTRhuU3AtNKuINuXzDyr32WdHKm+BCWBa+Uv0s5WQfPdtksK4Faj8VQtJTeNbSKasRmAFhHZI+tRDvBWZBorBN+tiyl/OPU4A5yT8mnRIG4WvhpwL2IWGWegbscsaPZA9Fk2VV0xexnfRFYbfSG4EazrBPOinssxa+Fb7EIenTOndplDgUyfRkrylh/zmS84aUvNPKtQhCvmcWRaNGOjmnCH1+L7CCj4RQNeA8lxBmHwT2Ye6So142Dx9fC58LnYCshD7pLOj5zMTK2emLXQr5WeAm176RNMOAg4+FkHL3Wpukn9szbztk5cSV83PmLxM+1LoUQtWH9yHmL4X5S6GuF6JA/pjsKudvZI/1Gau1zwyYgTtk4Lk/wIWC0046Hx5Z6jenIvRaHaSFsUDyER8T3jA3QhC+E4axpCM+anphJQRhzry07AQerJAgDEmfWloF1gJv2P1UUuHvpEfhB+EnwrcTNYvkz+tYWhhzjvgOnb3melfUD9LZ4xvhY4FYFMYkjDmT79D+dRl7DMJKoGcUWmGObJRE3di14AMy6hXbAAAgAElEQVT8FAkpmfsv4+fS4ylNitzcb6p8rQDgvthWSfi5V4j9jZDvmSB9X+WWJjyU8/9xlXuotkxdHNgjhzopcL8UNskZkq8+T64ZW+fsNVZX+5jjC+GrBHT4Y8Ys5CD7P7hGNWo6YRB+OZrxtvNYz7cr7DEDZuCuGHiOD3C/KxhsCp034Y8F3szmvDktlPeJsCt6lCr7LKsYNQh7ZZQpsTRm6J1yeBijjn5zhDw+6Kg5JmslMHMr7KvkOFHcyL9LNeTwoTKWu0z1g9asJ9eDDX/EAMIcpdR2FX5tbqVl4AwpErW2Se+0NsIqIbnfWIiPySBnqALHZuuVn+ui9H1VP2a2KW+T1nKPIB8PM6fIoORD9+gpvchtDhQQg9tWGIQgZEHnTDuhFZbCewL+PtlaXgvnBoMQCj9qyUkVOmiy5zHplLAWyB0E7EY4dc85e6ntQWHP/UjGRr78hzPCO4FrHEZys4t5WqERhiovVLZNM2AGXhADz/EBjje1LKWOrxNaoc8J1RoLey3950LdI6c0UsjJspCySgZ6rkPPsiz0MbXMJb4VeBg7VToVbATWQ8J++yIhSM8z1LPktFWRg29IQKdXrgs4JMyBDrLQo0nGoBWe16+jrz5El4VdzhIKPyqxIfnQc11emyKfvJzbF37qSrsIPdwvK6GcoSkTRnTy+cYs/wQYRnJqV52T52dfYuuEXJf9dV3ZtxmpYbZDUp5zr8Rso08JOUMRjJXOnmXfXcrvtC4Tcgm57LUW6jp8UxKqAD0WyVfHxnrsKudWdiP0Y8mVj9pl8uU9Z5RNpjSKDCNRZgpC3ouUVmhQkhALhc08vy1s4j8I+Mu8ImWW2imrmZXpJDNgBp6EgR89ya6P23Sn8r8S+L9e4v9+pvy/YBoKf71LK8c3Am9wfyHshX+VkqLWRuBND2GPfxDo/ZfCHwnUdALCvquk/5nWvxWi0AtBaIT8BoqvFd4TyKUve2chrxPKcxThUXVI+fX//U4rP/twRvbphXJOfOzHWf5OGNuTszInefQZhLwP618K5FBLP4RefyXgz3v/KsVY0EOKsSK90ApB+BMBTtCZl73Re4HaRmA/5toI5OT++INAD/b+WyEL32oxL9ILQWiFfyuQSw/23Qp/LSBR4Dz0HRNmo+c/Ez4Wfiy0KfHvkx60AnJ3KdZrjQIccQ72YA1JH7T+oxCTj/mY7S+SP2htBHoi9P0HgZqVQF9yOyEIrZDPSd5aoJYYfQcBf67tk1/LW8K58qz0GIR/I/yl0Al/IuAHS6EXmAsOO4G9mY+8ncD+xH4l/LWARKET8LdCSCAXfSXQH70XmIn96UsNZ5kj7ElPQL+/K4qC9FYoeSOPvZqUN2jlbNhR4EzMQC985Ywy35KNPFH4C6EX6I8EoRX4xo0YPXcCe9MTXxCyYFPbC38qLAVmQaiDm7xG6YfmbBV/Twiphr7o/yAMgsUMmAEzcHUGGu3Am92Y1H9BeiznKXzMbHk+DGw16tg9NjyfI3hSM2AGzIAZeO4MPMefUGvOGzlAFv70eO/CQ0BIGO59WM/3FgNcu1J4oBveyrLDDJgBM2AGzMCVGPjRlfresi0fnkshCDwY/VO1Of5WqH8WqdJuav6ZdmMuforhJxLL82HgVxp1JfCT1F8J3H/Ya6G+957PqTypGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANm4AYMvIT/iOEGNN3tFvwF+v9M4N+O8l+gv9vL5MHMgBkwA2bADFyWgef2z4i0Ov5O4N902wjYgP/6dJ98Wh6EeMjGDVf2vcW/OcfZODf/AOevb3i+sa1aOZnnmgKvXON4zU2eUe+lZi3/b5ee0ege9UYMtNrHr5kbke1tzIAZOM7ASiljD0gL+XmgQdB3Qkx2XprKvpY5Nt+l9+rUEC44a7x084l+S/lBKezfj/irtIuY7BMv0un5N+Hab5//MXyCKzPQq3+88h5ubwbMwBMw8MHMPfnQ/lrYCUNRw//H5X5mj0ulTe2HP3+goTNzLSs5utr5TO2guTkn6G90Bjgdqr3YP95of2/zBwa414HFDJgBM2AG3kEG5v6EGsTNL4UfBH6y4/+AHfDhfQ8SZgyxVs5iRt7cFHpdsl/e9xo9556pzAtVEXOd+pNd3eOcOZ66Zu51XmrQudduipdDe03VjPFzSu5Y/ZRv7vmm6i/lPzTHNc5+jZ6Zi0PX/Bhfh3g4VjsWv/d+YzPbZwbeWQbmPsBBUFtgSPo9EBc0BEC6tDZa90JM9korCEKbkEJvLVGerUB9k6Ih2b1WdPxLIQrkHnrjI5derYBEoU945XlVv5HBjMQ74VDPVvEgNELWmaP86RY/+0YBYd0JrVDus36I/kEaqa0QBPIAs5DHQ3sjtEIUkEYo98G3FFohCNSj0wNpBPI3Aj2j0AnkzRF6x4RWKzaCTt+dEB48r/bAxz5jEuTcCr3A/l2ytTzMix2FHMOHBGFT+Fvp7MMahV3SqdsKawFZCtQFoU3AvxDIIQ46IQoI+ejktEkPWjuh/qm+kQ8Qz7rUB6EP+UFYJfRa6XtIiFNLzTLprEgU+gR0ctYC+YeE3J3QCtRgd0KTbHygF0qJMui9TGtTBHOfIN825eQweXuBHPQo9AJ9Dgn5IKSVvlmiFGzia4FerEgj7IT2lflQT259vYhTF1N8oRUJySaf/q3wLQEJORsBfxS65NPyIK2wFmJac8+H4IQE+TuBWZqkkxqFPoGencDeU36FXr9umI9+1AQCkijshFZohE6gL0J+PmNyeTEDZuDSDEQ1BFnQf1bY11bZjze2NmHQiq+WvvK3svHNkaCkvbAokpuk86b0feHnDagrbNSxN+q2yInS+8LeSG8KG70t7DG1lzNWgXrfOoc3yZ2Qz8Va1tCPmiy1TQxfLaWfnuxRCvtyxiytlL6wY2UXoTdUauhVSnmeVoFynyC7eTP9LSvKw/VcCsye+28LnSL6rlEkdWyf/Hmhx05YCMsEYkPyoSO9QLwVqMmCHhPWhT8kX3bV164rclGxY+EbpDeFvZFe9i9Cr9VOWqwCvWzOhkTht0J4ZT78b30fFqHXaittJ+Q+rHshFsl9YRMv80nDDil/0LpMOit2Kb2MTeFopG/fTHnLon8puT73XxRBYm1ho5c2ofp6le8jdX3Oj1LYZ4VDQl6TdBb0NtmsWU+uB45iNibWXn7OlKXcI8rJ9WUGcnJe9ofky/NtZRPLQt2usIlh1/3ouy7yrJoBM3CEgfePxMfCvFD7IrCX/puxxCv7WvUH8Qr7DOrZC5wV4c0mC28+f1PYnD8U9jkqP03SN8sgJRb2pVRmzaAneilrGX3hYCZ8p0ij5PIs1G6F+ufX/pSmRW4986BYnrGT3hS5UTq+YxKUwMz0ZtaF8JnQC1nwr5JBnNwsH0rBl4VYxk46iIUv5/VS6MnaCV8KnyabmkFoBf6AhB8b/5i0cm6rADb+LIMUkIUZF4Vdq8Q+F/o6IJu5s/wgZahyYmWPmewPEFZ47JNdL2s5hpSXY9ghGY1WbIReHyW9XPJe+AaB8x2TXyiBeyEIm5TMLJ1Q9iv1lHZw2Sn6RZURRyr6tM82xQ69VzAX+aUcmysqeSkwT5ZBSihs3t/pQ06Z9zvZ5OLbCkGoXzfUgZWQJfvKfuibIseqGTADRxg49QGOFyEvvlJ44XVH9rlmeFBzcGnp1JA3S4RzY2dZSOHNBv9jhV4Ia0zAXj94byvMMBRb7qXvThyh7lGWL0/sNSe9nI/Ze6FJhcw/R76rkvKcrDGBlHXK22htkk6cvx96bC/yENaMXnonsK6EPxe+Eb4VkEGIAj9d4/+tEIQxwV/PgL0YS57p4/xTEopAve9UzSX8UU0yuA67omkrvRFOPfN/pZr/rcBXqWfUysMLf2Dj2zJsJKT1MQucwS9nWAlzZs45rDGBGdb8j+TQA3BKGV24r2KBnfRuNPNN51DlhAM1nNViBszABRn44MRevNFsT6y5Rfpw4iYx5fcH6jjnRqjfeOAgg/JY9FhI3xf2HJV8/iSL9HMKrpjDLGFm/1Z5oBZ6xMoJL8iu8l/CjGqyLRp10vkmizn6wn+KmusGFYEx6eRsUmA1llD5etmHvs3iDOQg6wR8g9A8eP/wc9nYfuSFlJcXbHg4V3YHCocDsWuEOMdC6EeaB/k2QhTq81JT++oW/0oOUEuUo03OoLUXOmFIvlOWUCW3svE1yb9MNvMiYzPjO/ZeQZ9d6jFnoWcQ+jnJR3KGA/FTZjrQxiEzYAYyA++fSEVQfv3GwhvO6sQ+j0lnv3Okr4rmvKF0qvlaKGsb2WVtefZltceUGavARnbtW08Vn+CfO09uyRzlefA3E/sNE/5OfvZdFHF6/nwi/1R32Rc9CF3RZCudb6yYYV/4T1WZl7lLWScj9+5kgznSK4kPX2qzBClN8sU/uB/uL2ZfCOUMXfIXqa/VVlqsAtTiz0K/U4QZvhHKGegRhO0pjS6Q26kHe5dniLKXAvMxK0CwsxA/V9ZF4SB9l+yN1qZqGiqb/NJXzxEVz/3KUvLq3DLO3tSWkuf8aqQ2VLm1yQy/FGIRWEhv6sQZ9qAceq2qXkF2f6SeM+dfPI6kOmwGzAAM/OhEGnjz+DthKOr+UjpvIN2Jvc5Jb1X0Xwp/IvAm86cCb0C1NHLwdzF4U/hBGIS9wKz/vvCeMFYn9xsyyPqXAufO8g9S/lrgZ4f/VNgKK4F5fiXwc0wUgvAbgX0HgRxq/lzg//aKNytq/gehF6KwFJjvJ0InTP3fY7WKcT54+GfCTkA4K/v8XqAH/cn7N0IQ+GaKPf5IIBf7LwTyyKHPPwqc7/8Rflz4pD6chRg56IPQCnmW/1M6/PRCI3wowBFztgLSCHwT9WfCvxXYGy6Yg7m+E6YkKMDe1P5z4QsBvgehFHr2I/4q7YET9l4JmQPOhfxK+EshCJyD67YV8jX5W+l/JTQCD/nUcy2jUPOce/6dYmuBc8JJEPBFgXp8/6EA7/THJkY+fs67Ecgtrx37DgLyFwLzkvurBPyt8F8I1HKdolBeh53sMenlXAlB+Eig718LXIco1NeuTX72+Z+FvVBLlIO9g8BDLTn1vbiWr76v/l4+/LxGuK/gpRf+QeBa/F4Iwg8C9xZccK6/FOjFTMSCUM8t11vSyMNs+ZqwD/04+yDAA6+Tj4Ug/JPQC8ggMBNxZkWwmYHXWi80Qo7TeyXQm3OWfHCWQUB6IQpLAR7K9wpi9GAPXh/sCwf/QmDPQRiTX8nJWf5E4Bpzr3dCFEqe4A4+Sj819MaP0IsZgjB2v3AuZufa5X5SH+b8QugwLGbADFyeAV54FjNw7ww0VxwwqHc30n8j33rEb9e7wUCrYwKLGTADZuAmDLx/4i67E/OdbgZuxUCjjULaLH8bcI292WOsf3+NzdzTDJgBM2AGzMAYA6f+hDrWwz4zcA8MtBri/xD+pdBdcaBBvfm5h2+j+QkrCPzkhP2vBMu7x0CjI/MTbRT4Sdh/0H337gGf2AyYATNgBh7BQHhE7Tml7MeDm8UMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM/AMGPjRM5jRI5oBM2AGzMArBjZa/kz4IwH9L4R/ElZJ//tXaf5fM2AGzIAZMANmwAyYgXthoCkG2UlfJnuhNd7LkJ7DDJgBM2AGzIAZMANm4BUDPKQBhHWf9GznWOG2agbMgBkwA2bADJgBM3AvDKw0yPZehvEcZsAM3J6B92+/pXc0A2bADJiBRzIQVd8/sofLzYAZMANmwAyYATNwMwbCzXa63412Gm15v+N5spkM+GfvmUQ57W0GPnjbZY8ZMANm4O4YCJroM+HPBfR4dxPeZiDO/knCh1rrvwt3mym8y2MZiOna0adJKP9O42P7u94MmAEzYAbMwF0wENIUUWt/FxN5CDNwPgPcwyGVt1qBxQycxID/DtxJdDnZDJiBJ2JgeKJ939Vt/fPseVc+qGzOz6Jr5eV72t+insf1O1/lB7h3/hYwAWbADJiBNxjYmI+zGeBnUPg79hC3SzuQtxS6s3d0oRkwA2bADJiBZ8BA1Iz9M5jzuY7Iw0fzXIe/k7mD5tjOmIWHt0449rA3o5VTzIAZMANmwAzcNwNR4/X3PeKznQ5u5zx4PNsD3nDwRnutD+zHQ1sZXx3IdcgMjDLg/wp1lBY7zYAZMAMPDDTC14mLH7QOSZ+z8CHNfzFayscyhiPFe8X5L0yR747k1uFPK8fPZR96kCjTOxmxbmj74RsyuGmF3Uw+yO8Txmq2inGtfpb6/WRmX6eZATNgBsyAGXiWDERN3d94cj5sfy/8Vghn7L1STZd6bGbUx5TLnu2M/DolyMFD215g5kWdMGI38nUj/nfdBXfwwsp1XJ5ASJNqTyhxqhkwA2bADJiBl8lA1LH6Gx+ND+9B4IHq20fszYc/feY8ULVpP/Y85aGhHC8/fDSlc0LfPWKfiZYvwl1fq3DiqQbl1z1ObOF0M2AGzIAZMAPPlwE+BPNPTt8n/dwHm3NYiCriYercb8XynszczByAhyr247yPeQjYHNmPmYYjOZcIc4b1mY2C6lZn1j5lWfeIMz/l3N7bDJgBM/AkDDRPsuvL3fSUh45rscCHf6zAXLeUVpvlhzhmOVfCzELy9gJ7/mJmzVga3B2SVsHNRAIcd8JWaEZyVvL1wlo4tg97HMsZ2eK1K0prDiVcMMac+Wyt9Lw3Z/j8hH0a5fYn5DvVDJgBM/BOM9C806e//OHhc3f5ts+yY6+pL/Gt2NzD8xCRHxq5DteQXk0P9W4VJ2dMgpzrsUDl4xzNjLxjKTxA8XB1K4H7ZbXZVjZzzBFmpYfFDJgBM2AGZjDQzMhxihk4h4Ggor3w2G/FTtm7S/vxHyTUDxOn9JnK5SxxKih/I3w/EV9P+Gs3Dz2XkKgm7SUazejBXlzrWlo5TnkoG3sIrHvaNgNmwAy88wwEMRDfeRZefUsBF5bLM7BSy/yt2NwHmMdMwbc4u7Tnr7ViX0pC6nuoX5zIgYflocIUY49uRt7cFLi4hbTaZDuyUTfhH0l9cPVCMxW03wycy4D/r7TOZc5198YAHyagFYKwFi75QXer83KGRmDlw6M8Qyc7/2me830rZGml4AOdENOq5bUEab2Qv3Fhj1bYCOzTCPh6IQgIay+U3zh0ySa3EeoauV60bHU6/n01hH/Ha3nl0+7VvxF+l/ZqL7hfmNFrSDllLvcL2M2oz/fHWCrcrQVy2tRzLK/0wUc4lnSBeFSPvurTyA4C6ylCjcUMmAEzYAYqBjrZTfLxQYDw4dIn/VZLq43mIEwMxMz8VMWKcJZt0vMySGmS0aY1ai3zetn4poSHscwTOTuhLZLRS5tQ+QCH3QsblCStVvAuCbzBy6W/FZvicJ32q6/fVP4cf0w9j+Xmh/6cxyyLY0Up3mqNE7nlfUteeV9OlDzcd1P9pmrO8XPmJu3FeZn1nH171bWCxQxclIEPLtrNzczA7RngjRXpqq33soPAhwx6m+J5TeasJSgrCt2R7PZI/FiYOb8SWBFW5i9lkAGQNq1R65B0FvSl0Be+Wt0WDvbZ1Qkz7EP9y/L/RMZHM/o9Rcr/pU1/c+bGjep6Aa75Ju6LM/vMLdsoMQqfCV8LXLNhbvGBvB8OxHKInHxPsbI3900tQY5FiucYvimJCnwufCd0wlAkhsrOobF9i7IHjuo/cJRxdM4z1M7CjtL5xrNLvj6tX2rNenI9vM8M2RhZ6/yRFLvMwOkM+AHudM5ccV8MNBoHIHxwlG/uPDSUNh86UxJToK8Sgmz6NkJXxa5hDmraCqzsO0c6JW2LxCC9nVNY5NQfissZ9XXNVMl/oMCfTgWf2P/vHrE/99Na4GHqvUf0OaW0UTL7zr035vQ+9rBDj6HYM0rfTDTmvtlWsTCRixv+/kbohPr/9os+9DtFMi+PvR5Rm/bVxjvZPKjX0smxEvZ1oLDzXAdSHDIDpzHgB7jT+HL2/THwiUbijRWJAm/6SCN8k3SWKGySvdSaa4qUUXUY9Y4723H3W95OnrG+fAjQg/mQKOBDgjAkfWzp5GxSgHUYSzrBtzgh91jq/6IE8FKFb/CaGx6O/xqV+2K40J5hRh/24r5cC91EPn3y65H8LFOvNfp1Cej8FE3/INCHB0v8U/UKvSV7efq3vKc7okq2VRmz8K1cloUU5uQPiqx8izgmp55hrId9ZuAtBvwA9xYldjwzBn6peYMwFHPzhtkIq8LHm20QiEWhKWKXUttHNmK2bdGDmbNwlo1Q+nIsJKV7nW3lFgxwvdZCvMVmaY9eK/fB7oZ7stUg8FNnJ+yrvbkn2xTjYTameJfWQWtIel5WUkruyCn/wPVj2VMPRFExOLimfKrmzFdKkJF5R+eb5T8XmPO9MrHS4edcYYZO2J/bwHVmwAyYgXtlIGgw3sw/E3iza5JdvmlG+fjT/UoopZHB32kB/AwGsk2slL6yr2Eu1ZR9OAuIwk5ohZDW32tlzkYoZZBBLis5vxCCUAr2zwTinBO7FX4rkB8TvtX6vdAIQShrZL5VQx41cIw+Jd1U4AL+oB7tBfrMbcG1GgTWW0mnjdoLbxbUj/vhmPDa6SeStvIvhCBkPspcfF1VS34jfJrQasWXpZMyxW1f5F1abdTwa6F8jeQ94GAnMOc6OVut+A9Jr2DOP5Q3Fts/onasn31mwAyYgbtjgDf7ZmKqVn7eZPPKG3AtUQ4wJf1U4Ar+cGLPrfLrM3Fe/Pck/cgwzMnDIw+MfGhm8EDIg2MYqRlzRTnbIsD9wPnxEWsEeq4LmzixUwWueVhlj1sJc3dX2gzO45HenBmMSZ+cq7QGrV2VmHMq9+T13aXEek84h4unEvaPxea9dGYMBwbaVzUHUkdDcdRrpxkwA2bghTCQPzzGjtPLyZtsFFqhEWqJcoAxobYfC9yJb2q2Kf9TjV3OA6dbIV83Ym0ajFgjDALfDsbkP7SQ0xYJ6KGw2Wdf2KiNECvfHLNPtXNyL5HTqMnuEo0menCefB0mUg66uY5cM3qwbtJaFjUpXvoO6X3qUc9F73Co8MaxXZqzmdgXPnhAPleoj+cWu84MmAEz8BwYqN/oy5mbwiAvjBwoygdqwdcWCHXCHdiciQ+2T9MsMdnLC83Gh8glpC+adNLLa8aHXH0N2XcQ+LbrmEQltEVSUxXAz7by1ftV4VGzk5dej5H1CcVL5e4EuDhX2O9QPefpzm2eeq+00od1SogfmqOsiyO9lvKtp5o/kZ/zMuuUEO+ngjP8zYwcp5gBM2AGnjUDcz8Ypg4ZFADPWfiAiwLrJYUPoUtIXzQprxc6D3Bjc7cpdmz/qARykZCQzIdlJ6xLh/RTz0V9V/U41aQ+ziwKyhuEMV5mtnio7Y4kk8M+j5Gg4mNzLpRTX4O5ez6mdu4e18jbqGl7jcbuaQbMgBkwA2bgGAOrYwkz4/1EXpR/fyA25ycoerQTPfjwn3pAnCh5y93IsxPodY4EFfF38IaZxezDN49xZv5YWisnP0HPuX6D8sJYk5m+x9TO3OJZpj2W12d5aA99GwY+uM023sUMmIFnyECjmYPAw9VS4AHmGhLVtJ9oTIx/KuYxQg/+/a5z5+fsXwpr4ZMTBqFuIawEdOSrtB5btkpgXh4880/jx2rYi32CwJ7YnJtex6RVAmiOJU7Ehwn/u+zmGnANzc27fBf47GbADJiBGzPQab8g8CGEbJKdzLOWfqIK/3oixgdgTDHWQViO5BJrR/y4mH07ETvmDkrgWywepC4B+h2T7kJ7MS9nnyuDEhdzk513lIFeGWP36tFCJ5iBOQz4G7g5LDnHDLxbDPAwxQPPIOQPoL30kHyXZIMHBr5hakaa4usTCKMzxw7jBInK7U7IL1PZ7ydn1o6VDWPOyreR3c3Im5NyCleNGrI3q+VxDMAh3J/C/+N2dLUZMANmwAy88wy0YiAmFlZaF8L2Aqz0Iz0a+ab8bZUfZOc5ltKxs0QpbWFnldn5Jop8y3EGOqVwzS3nMxBU2gvcexYzcDUG3r9aZzc2A2bguTLQafAo/FT4UGiFRriG8LDQpcZ84H0mbIRBaJM/L1FKLzRCEFrhmEQlPObvvx3r/9LijQ4EZ37gPe/Kcg+3Avf1/rwWrjIDZsAMmAEz8DgG+BC6pPRVs1Y2347xQUeMB7coTEmnwC+EsYcL6tqisJH+pfC9wN9hQ8dnmcfAGMfzKt/trKDj8xBnMQNXZ+C9q+/gDcyAGXiuDPAAt73g8L16xUf026V6ZmI2PiiH1I++oE32Uy1BG/MtYvkhztzn/pe09NsnzDkT+TElsi+wmAEz8AIZ8E+oL/Ci+khm4EIM3NOHf9CZBqF8mFle6JzntGEe/uMLkOdokv4brR8L/JMh3wlfCL8WzpGNihYnFHbKzaDWYgbMwAtlwA9wL/TC+lhm4AIMDBfocakWPLi1qdlaaxS2l2p+Yp+8P79gAGZDWJkpCLtXroefhlcCeTH55iwLJfFw+Inw0ZyCkf79iG9mK6eZATNw7wx8cO8Dej4zYAZeDAPDI/RovFYAABEdSURBVE7Cw1F+KKJP3Qs7P0g9YpuHUh6eWiGmRvTdChthnfQhxcqFHGQpZD3Hh6zMXNmT//iCb/AA0ggh6fXSJ0dec5xZal9da9sMmAEzYAbMgBkwA8+egVU6QaM1VKfpZOefTuufUHPqTsqiquuTjX+oYlPmWoFmKjjij/J1hb+VDixmwAy8QAb8DdwLvKg+khkwA49iIH97xrdXXdVpL5ufQ6ckKMADHHlZeBADCH7icyQqibqFQF0jTP2cyrd05AQhC3XbwrZqBszAC2LAD3Av6GL6KGbADFyUgTDSbZAvYyT88Pfd8kNTlB6EXtil5GXSyUPfCPsUq5dFikWt9OzqhBGbXrkuSO9HcuwyA2bADJgBM2AGzMCLZICHq2biZDx08U+F5J9QQ8pj5UELtAI9almn+EorD1qHJCjYHEoYidGT3nP6j5TbZQbMgBkwA2bADJiBl8sAD2cxYexBberk2xTotAbh2EPcVB/7zYAZMANmwAyYATNgBm7EAN+MIVHI+o229jZmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZuBHpuDiDER1/Fz4WNil7q3WT4Xvku3lvhloNN5nwg/C/s5GjZqnvr/ubMSLjdOoE9fhPWG4WNfxRgu5/2k89Nobpb0r3B+hwmEzYAbMwOkM8EYLLi1bNQSXkEZN+qIR8/7+Eo2P9FjdaJ8jY7yI8E6niMVJLnl/0PYx93Gj+r6Y7R7VS71GuQ7tlQ+YX5+8fkoJI/s28vUjfrvMgBkwAzdl4IOb7naZzZapTX+Zdq+7tNL2F+o5VH0u1ffYeL0SfnIsyfFZDNTXbD2ran7SY+7jYf42T5IZ0679BXbfXqDHsRZca1439bxc8/q6D8eaOW4GzIAZuAUD799ikwvv0Vy4X263kzJcqfet2vJBdIsPvFud55724d645P2xuqfDXXiWeOF+t2jH64bXT5YgZXmLjb2HGTADZuAcBp7TN3ALHfBLgb+Dws+R+e+UBem/TYf/ndZeiMKHAjX8vTNqkJ8LfHDi74Xdg/dVPv34O09d8rGEFBu0UrMtYjH5sn9fxI6payX8WlgKvxQGoRG+F/j7Pr1QS5SDMyFj5/xG/laAm69epT38bxDYhzkHoReOySolcKaPBHpnIYY/CNukE2sF9iY34Ehrl1ZmCAI29VmYK/fER89DkvPJ2Qn0xUddI+Tzo+PfCEgQcu4gvX/w/kHIjcL+TfeDhf/Q/UHtTuiFIOT77SvpUSBO3z7pP9UKuJ70/S7FtEwKs4fJ6KtYTHH22iadGmqz5DmwmWsl4ENyDXoQ4oP3VTzHWtmZY2qzlPEv5eS+mXM2ZoipCXqXGx5YG8XyzKxLYSf0qSbPRb9tkdtKZ/ZvBGqId0IjMC/+QaCeMxBnRbiWpRCLQhDYY0jBRiu9uKZZ2GsjTNUUqVbNgBkwAy+XAd6AY3W8Vvau8PFGyZtqSD5W6ngTzdJJaQp7Lb0vbN506UEvZJVQ68QHIecRj0KPUgj7Z8k1TeGjfyeUfYrwg7oWdoWT3F5YJl/QWu6Du0sxlo1Aj0NCTjlDXyQTy3vh7gqbmp3QCkFAyAerZEet5GQJUnqBWmQttEmfWqjP+Vvp9F4mH/5BaJKNjg/8TMiS67IdpOAjD6HfIMRX5sP/NkJf2FF6V9jU40OCwHVoXpkP/7sT6JullxIL+5DaKdgUCRvpfWHTl/2zEF8loyv8QXqb7IVWZsKH0IO6rE/1C0rgbPSnB0If6rP0UmJhH1LZM/dZS8cupZUBshBvkpHzqc/7d4WOHzukfOwh+dD3RWwnPaY8llboCzurUQq5q+QIWoccTOtWay8sk91q3QiHalKqFzNgBszAy2Vg7AFuoePuBVaENb9ZJtdbDzZBAWpYkSj0SWfZCWWPVnaT4qybpLP0Ar4sUUpf2Kj1g1UjX1fksFeevyp9bRIvZybQVMljD4q5b1RuX+XXJnHysmQOGjm6wo8ahW8LXy+9LWxqd4WNWs7XVfn5fFXJazNK64tgK70rbFTieeYcClLKOYiTl2UrpSls1F6IhQ8dX5ZBSixsdPpkqe/TXoFyLuz4h/RJjZpydhKp64sK9LL3sogP0hdFLjGkEzaFn/ps99Kn+lFSn62VD2TppcTCPqSSm2dizvp10soHspTxKGf+9p14I3Q5Ma1R67bw9dKbKgcTfyz8bfIVrgeVnH3lHJu55HZOTb2PbTNgBszAQQbePxh9PkHeUHmTbtLIUWt/ZPxBcerCRN4n8u+KWCu9SzYrNj+DRYEPniCcIsy7SrXUBYF5Dkl9zkbJ9JmSfL6lEtgrTiUW/rV0en4vfC0MKRYLPbke+MFfCr4s7A+mJKY4K2DOsr6uG+QIlXMsv96TOno3wirpZZvPZJAzVxZK/EhgjQlB6yCU0lc2M5wqUQV1n7rHp3IEgVzAXJmDTjoPOb8W+BZyEBBm6ZPOshXWyT7UL5fsitrHqDEVN1qzfqjfDwqGIuG7Qqd+qIqxub6l1DlV+Kg55+yZ/9xsTs3RjZ1gBsyAGcgMvP/MqYjF/Bvp+QNoIb1+A506KrmnSqOCTvhG6IW5e5X7ULMVmHmZ+pTxKX2jQFMED+3N2fqUy15ZL8rfUgd5qPtCyH/XDntMxvyH5ql7kDsIfYFYJxU2uZyjFfhQDkInHJPMw6DEuTwc6pnPuFNSn9BpXR8qOhCLB2KE5nBaztKrZpV6brT+sfCV8LHwi+Q/tkz1y3VzZiI3HNmoUzwKrNsjuYThuBV4yGySjh9ZZKVYw4jvVBc9LtHn1H2dbwbMgBmYZOA5PsD9UJxmWeh84PChwht8P3niPwSCVN7wqRuT38hZ9icn2xvprcB+pTSVfcykDzX0nZqj7kEe+1I31MHKXssmvx/Ja0Z8uLKfGuo7gfmwWUvB/q7ynWL2So5VQb1H3W+QoxX4jz8aAS6OyUYJfQK5oShgP3otCt8c9dD9Mae+zIkHCnaKHYpTOjZ/5rFRHI62wkrgYQ4Z65trDvVL5bOXcCCTedhzk3IWRW6epS7H3wjcd6ycI8tWSl1Hz3Pu0aHoG6SXsxUhq2bADJiBp2HgOT7A8Yad30yHiraN7JVQ+3NaU+STuy5y6zdocoln/7LQ94Ueih5SH4SaD7NRrKHy7WTT61TZqIDZ+onCPDO9+akvy1JKEIgPf3C/oa1kkZeF3J3QJUeO4W+EdU5Mayhs9JIHapC8ttLpF165H/y5f3K9tTTyfFng8yqDHrl/Dg2VL+eElNBqbXKy1jxTLHzUlGchfy3kvXINJdlXz1G0e+A/JMeuDFR6J5tc+mdppJSzMEcr5P2CdIAQK+fZFv4oPfclJxSxtqjDn2Mp5Y15iOU9iPdCTIm5fzLfWPayyjpqfki+MFFPPj8F53vgp0WPrqohtxHWyc+Cr9yzCL1xpl2RRz42gn7stV3vMaeG/uu0hxczYAbMwFEG3juacX8JvBmuhEHoq/GC7Ch0lR/z98LHwlKgRy8MAhIT0AehQ5EEgb14c0X6tJYz7OUbhCbFicWUx9ImZFcnZciG1rWwKew5Knsshb5Kbgu7kz4IjcCMYCdEIQhTe9KX/ggrNUOyWZpkE+sF+iJtWln6pMfC10lvKnuQTZ+VkPVtkVOrSzkaIecE6VHIPVrpWQYpXWE30vOs1LMndTkn9xhSTdAK+mTHtA5au6TXNeQGoUlxlk6IQki+XiugthGGZOfZZI5KzqVuJ2AjbVrHZiG0FIJAf3L6pGt5PQP9iG1xJpnq1xY5nfQohOQbtOKjthGwe4G9pyQqEFJOngNfJ6xSTMtDH+Jt0unJPlEgbynkfRrpgxCEbeFvpWcZpHTJaLSGpPdaARIF9mDfIdn4snRSmsqOskPyDVoBviydlKayB9lRYAUWM2AGzMA7x0Bz4MQ8wN2jrO9xqDudqZ+Yiw9Yy8tnoNUR48gxNxP+kVS7zIAZMAMvg4H3X8AxWp0B8CflKcmxQzlTtdfw79U0CEthe40NXmjPXudqqrPB4e6FntfHepOBsevPa9r3gO8UM2AG3jkG3nsBJ+bNOwo8FHUj5wnyNYWfnGEk75auqM2Yuxf88HEa83AXBD64ueZIl1YvL5+BoCNGgeuPcA9s05pcXsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZMANmwAyYATNgBsyAGTADZsAMmAEzYAbMgBkwA2bADJgBM2AGzIAZuFcGXsJ/xJC5DVLyP1rLv5Kf/4L7Ie6p+SwlfDOz5lA/x8yAGTADZsAMmAEzYAZOYCAqtxf4997Qj0mjBB7ylsIg8F+yWcyAGTADZsAMmAEzcPcM/P/tnbGO00AQhoXgAdJS4evoSEFBh6mgjEQBpTtK0tHmDTBvYN7gSrozPRJp6ZYnIHSU/L80e1pZ2HFyB5cT30i/dnZndnbzVSPHuTvm78Dln++f2ofrdSFrrrVK7KSt5ObNIwYBCEAAAhCAAAROnsC9A27YKNdPrKyVtAn/gBInler/Z+jPYluf1M24DAQgAAEIQAACEJggcHciVoaWmvySPkkp/Ccaj31q5XqvpBfSQvom2ey/k2rJ7+e5wXojOe97zDVcmvOfS68l+/elWvL7bEn6k1Va9D/AriPouc9KMS/v5iWfm20lx/c+k5Lkuz2U/M4dBgEIQAACEIAABE6KQK3bfJXcJNncyNThHzpstMGNmfe7XiddSDbPN5LfY+ukVqqKNTdX2Wo5P6R1LPhOrrvvHTif4b35DPuVZDuXeqkKdRp9N++x+fwk9VIb2nee92EQgAAEIAABCEDgRgh0OtUNUyO5WcrmeV/Mp9xaQTc86yKpirUm1nLOtsjJa5tYc0Plu7jhKs3xuQ2V85yfLe91k1ZabvTyWi/Ha5XkXN/BPgYBCEAAAhCAAAT+CYFD3oHLzVKrm/lpXC/tJK9vZ962jjw3QE8He6rBvJ+o6cZpIc09d6LUZagOb1jzp9bznxop66SYrOYUJwcCEIAABCAAAQhcF4G5v0J1k1JJjeTmyU3NRrLtpGHTE6HR4Y4ipZ5p3g6yXXfMFmOBK6yP1fQ9xmJXOI6tEIAABCAAAQhA4DgCc5/ANSq/iiNS+OdHHLkt9vSF7wZpKZVrU+VTBKuppANjrvlIck372R7I+VzMcSEAAQhAAAIQgMCNEpj7BK7XLd1gZXPDlZsxr78vYlOum76P0lvJNbK1cnbFfJ/rsz9I/mqzimTXy191lrX31crxRo6fLK6LDfaHa3PrkQcBCEAAAhCAAAT+CgF/jTnXcmOTG60uNi411lI7t5DymtiTYk+n0b5rlXW2mjtWrnlu2VahpNFaxNyxteT9Qxue4XgdSd7fSM6xJamL0XPfI8c8HzvDMQwCEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQhAAAIQgAAEIAABCEAAAhCAAAQgAAEIQAACEIAABCAAAQjcXgL+FepL6fHt/QjcHAIQgAAEIAABCPxXBL78BtHZSABFkTqYAAAAAElFTkSuQmCC"
    }
   },
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proximal Policy Optimization\n",
    "\n",
    "\n",
    "![image.png](attachment:image.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "PPO is very similar to VPG,but with few differences <br>\n",
    "\n",
    "1. PPO use gradient clip to clip the gradient\n",
    "2. PPO limit the KL different between current and new policy\n",
    "\n",
    "In order word, PPO limit the allowed difference between current policy and updated policy to ensure the training stability.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch.optim import Adam\n",
    "import gym\n",
    "import time\n",
    "import spinup.algos.pytorch.ppo.core as core\n",
    "from spinup.utils.logx import EpochLogger\n",
    "from spinup.utils.mpi_pytorch import setup_pytorch_for_mpi, sync_params, mpi_avg_grads\n",
    "from spinup.utils.mpi_tools import mpi_fork, mpi_avg, proc_id, mpi_statistics_scalar, num_procs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PPO_Agent(VPG_Agent):\n",
    "    def __init__(self, env, model=MLPActorCritic, optimizer=Adam, train_pi_iters=80, clip_ratio=0.2, target_kl=0.01,\n",
    "                 *args, **kwargs):\n",
    "        super().__init__(env, model, optimizer, *args, **kwargs)\n",
    "        \n",
    "        self.train_pi_iters=train_pi_iters\n",
    "        self.clip_ratio=clip_ratio\n",
    "        self.target_kl=target_kl\n",
    "        \n",
    "        \n",
    "    # Set up function for computing PPO policy loss\n",
    "    def compute_loss_pi(self,data):\n",
    "        \"\"\"\n",
    "        compute policy loss (i.e. - log pi A), \n",
    "        \"\"\"\n",
    "        obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data['logp']\n",
    "\n",
    "        # Policy loss\n",
    "        ######### Insert Code Here #########\n",
    "        ### compute ratio: exp(logp - logp_old)\n",
    "        ### compute the clipped_adv: torch.clamp(ratio, 1-clip_ratio, 1+clip_ratio) * adv\n",
    "        ### compute loss_pi: -(torch.min(ratio*adv, clipped_adv)).mean()\n",
    "\n",
    "        pi, logp = self.ac.pi(obs, act)\n",
    "        ratio = torch.exp(logp - logp_old)\n",
    "        clip_adv = torch.clamp(ratio, 1-self.clip_ratio, 1+self.clip_ratio) * adv\n",
    "        loss_pi = -(torch.min(ratio * adv, clip_adv)).mean()\n",
    "        \n",
    "        ####################################\n",
    "        \n",
    "        # Useful extra info\n",
    "        approx_kl = (logp_old - logp).mean().item()\n",
    "        ent = pi.entropy().mean().item()\n",
    "        clipped = ratio.gt(1+self.clip_ratio) | ratio.lt(1-self.clip_ratio)\n",
    "        clipfrac = torch.as_tensor(clipped, dtype=torch.float32).mean().item()\n",
    "        pi_info = dict(kl=approx_kl, ent=ent, cf=clipfrac)\n",
    "\n",
    "        return loss_pi, pi_info\n",
    "\n",
    "    # Set up function for computing value loss\n",
    "    def compute_loss_v(self, data):\n",
    "        \"\"\"\n",
    "        compute value loss\n",
    "        e.g. loss_v is the average of (v - r)^2\n",
    "        \"\"\"\n",
    "        obs, ret = data['obs'], data['ret']\n",
    "        \n",
    "        ######### Insert Code Here #########\n",
    "        loss_v = ((self.ac.v(obs) - ret)**2).mean()\n",
    "        ####################################\n",
    "        \n",
    "        return loss_v\n",
    "    \n",
    "    def update_actor(self, data):\n",
    "        \"\"\"\n",
    "        compute policy loss and then perform gradient descent multiple times\n",
    "        \"\"\"\n",
    "        for i in range(self.train_pi_iters): \n",
    "        ######### Insert Code Here #########\n",
    "            self.pi_optimizer.zero_grad()\n",
    "            loss_pi, pi_info = self.compute_loss_pi(data)\n",
    "\n",
    "            kl = mpi_avg(pi_info['kl'])\n",
    "            if kl > 1.5 * self.target_kl:\n",
    "                self.logger.log('Early stopping at step %d due to reaching max kl.'%i)\n",
    "                break\n",
    "\n",
    "            loss_pi.backward()\n",
    "            mpi_avg_grads(self.ac.pi)    # average grads across MPI processes\n",
    "            self.pi_optimizer.step()\n",
    "        ####################################\n",
    "\n",
    "        self.logger.store(StopIter=i)\n",
    "        \n",
    "        return loss_pi, pi_info\n",
    "    \n",
    "    def update_critic(self, data):\n",
    "        \"\"\"\n",
    "        compute critic loss and then perform a gradient descent\n",
    "        \"\"\"\n",
    "        self.vf_optimizer.zero_grad()\n",
    "        loss_v = self.compute_loss_v(data)\n",
    "        loss_v.backward()\n",
    "        mpi_avg_grads(self.ac.v)    # average grads across MPI processes\n",
    "        self.vf_optimizer.step()\n",
    "        return loss_v\n",
    "\n",
    "    def update(self):\n",
    "        \"\"\"\n",
    "        compute policy loss and value loss then update actor and critic \n",
    "        \"\"\"\n",
    "        data = self.buf.get()\n",
    "\n",
    "        # Get loss and info values before update\n",
    "        pi_l_old, pi_info_old = self.compute_loss_pi(data)\n",
    "        pi_l_old = pi_l_old.item()\n",
    "        v_l_old = self.compute_loss_v(data).item()\n",
    "\n",
    "        # Train policy with multiple step of gradient descent\n",
    "        loss_pi, pi_info=self.update_actor(data) \n",
    "\n",
    "        \n",
    "        # Value function learning\n",
    "        for i in range(self.train_v_iters):\n",
    "            loss_v=self.update_critic(data)\n",
    "\n",
    "        # Log changes from update\n",
    "        kl, ent, cf = pi_info['kl'], pi_info_old['ent'], pi_info['cf']\n",
    "        self.logger.store(LossPi=pi_l_old, LossV=v_l_old,\n",
    "                     KL=kl, Entropy=ent, ClipFrac=cf,\n",
    "                     DeltaLossPi=(loss_pi.item() - pi_l_old),\n",
    "                     DeltaLossV=(loss_v.item() - v_l_old))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32;1mLogging data to /tmp/experiments/1646381990/progress.txt\u001b[0m\n",
      "\u001b[36;1mSaving config:\n",
      "\u001b[0m\n",
      "{\n",
      "    \"act_dim\":\t[],\n",
      "    \"batch_size\":\t5000,\n",
      "    \"env\":\t{\n",
      "        \"<TimeLimit<CartPoleEnv<CartPole-v0>>>\":\t{\n",
      "            \"_elapsed_steps\":\t0,\n",
      "            \"_max_episode_steps\":\t200,\n",
      "            \"action_space\":\t{\n",
      "                \"Discrete(2)\":\t{\n",
      "                    \"dtype\":\t\"int64\",\n",
      "                    \"n\":\t2,\n",
      "                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                    \"shape\":\t[]\n",
      "                }\n",
      "            },\n",
      "            \"env\":\t{\n",
      "                \"<CartPoleEnv<CartPole-v0>>\":\t{\n",
      "                    \"_pole_geom\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.FilledPolygon object at 0x7f5c493e7208>\":\t{\n",
      "                            \"_color\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7358>\":\t{\n",
      "                                    \"vec4\":\t[\n",
      "                                        0.8,\n",
      "                                        0.6,\n",
      "                                        0.4,\n",
      "                                        1\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"attrs\":\t[\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7358>\":\t{\n",
      "                                        \"vec4\":\t[\n",
      "                                            0.8,\n",
      "                                            0.6,\n",
      "                                            0.4,\n",
      "                                            1\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                        \"rotation\":\t-0.047655701937088506,\n",
      "                                        \"scale\":\t[\n",
      "                                            1.0,\n",
      "                                            1.0\n",
      "                                        ],\n",
      "                                        \"translation\":\t[\n",
      "                                            0.0,\n",
      "                                            7.5\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                        \"rotation\":\t0.0,\n",
      "                                        \"scale\":\t[\n",
      "                                            1.0,\n",
      "                                            1.0\n",
      "                                        ],\n",
      "                                        \"translation\":\t[\n",
      "                                            414.0354050993562,\n",
      "                                            100.0\n",
      "                                        ]\n",
      "                                    }\n",
      "                                }\n",
      "                            ],\n",
      "                            \"v\":\t[\n",
      "                                [\n",
      "                                    -5.0,\n",
      "                                    -5.0\n",
      "                                ],\n",
      "                                [\n",
      "                                    -5.0,\n",
      "                                    120.0\n",
      "                                ],\n",
      "                                [\n",
      "                                    5.0,\n",
      "                                    120.0\n",
      "                                ],\n",
      "                                [\n",
      "                                    5.0,\n",
      "                                    -5.0\n",
      "                                ]\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"action_space\":\t{\n",
      "                        \"Discrete(2)\":\t{\n",
      "                            \"dtype\":\t\"int64\",\n",
      "                            \"n\":\t2,\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"shape\":\t[]\n",
      "                        }\n",
      "                    },\n",
      "                    \"axle\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.FilledPolygon object at 0x7f5c493e70b8>\":\t{\n",
      "                            \"_color\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7048>\":\t{\n",
      "                                    \"vec4\":\t[\n",
      "                                        0.5,\n",
      "                                        0.5,\n",
      "                                        0.8,\n",
      "                                        1\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"attrs\":\t[\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7048>\":\t{\n",
      "                                        \"vec4\":\t[\n",
      "                                            0.5,\n",
      "                                            0.5,\n",
      "                                            0.8,\n",
      "                                            1\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                        \"rotation\":\t-0.047655701937088506,\n",
      "                                        \"scale\":\t[\n",
      "                                            1.0,\n",
      "                                            1.0\n",
      "                                        ],\n",
      "                                        \"translation\":\t[\n",
      "                                            0.0,\n",
      "                                            7.5\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                        \"rotation\":\t0.0,\n",
      "                                        \"scale\":\t[\n",
      "                                            1.0,\n",
      "                                            1.0\n",
      "                                        ],\n",
      "                                        \"translation\":\t[\n",
      "                                            414.0354050993562,\n",
      "                                            100.0\n",
      "                                        ]\n",
      "                                    }\n",
      "                                }\n",
      "                            ],\n",
      "                            \"v\":\t[\n",
      "                                [\n",
      "                                    5.0,\n",
      "                                    0.0\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.8907380036690284,\n",
      "                                    1.0395584540887965\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.567727288213004,\n",
      "                                    2.0336832153790008\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.045084971874737,\n",
      "                                    2.938926261462366\n",
      "                                ],\n",
      "                                [\n",
      "                                    3.345653031794291,\n",
      "                                    3.7157241273869706\n",
      "                                ],\n",
      "                                [\n",
      "                                    2.5000000000000004,\n",
      "                                    4.330127018922193\n",
      "                                ],\n",
      "                                [\n",
      "                                    1.5450849718747373,\n",
      "                                    4.755282581475767\n",
      "                                ],\n",
      "                                [\n",
      "                                    0.5226423163382673,\n",
      "                                    4.972609476841367\n",
      "                                ],\n",
      "                                [\n",
      "                                    -0.5226423163382666,\n",
      "                                    4.972609476841367\n",
      "                                ],\n",
      "                                [\n",
      "                                    -1.5450849718747368,\n",
      "                                    4.755282581475768\n",
      "                                ],\n",
      "                                [\n",
      "                                    -2.499999999999999,\n",
      "                                    4.330127018922194\n",
      "                                ],\n",
      "                                [\n",
      "                                    -3.3456530317942894,\n",
      "                                    3.7157241273869723\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.045084971874736,\n",
      "                                    2.9389262614623664\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.567727288213005,\n",
      "                                    2.0336832153790003\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.8907380036690284,\n",
      "                                    1.0395584540887965\n",
      "                                ],\n",
      "                                [\n",
      "                                    -5.0,\n",
      "                                    2.8327694488239896e-15\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.8907380036690284,\n",
      "                                    -1.0395584540887954\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.567727288213004,\n",
      "                                    -2.033683215379001\n",
      "                                ],\n",
      "                                [\n",
      "                                    -4.045084971874738,\n",
      "                                    -2.938926261462365\n",
      "                                ],\n",
      "                                [\n",
      "                                    -3.345653031794292,\n",
      "                                    -3.71572412738697\n",
      "                                ],\n",
      "                                [\n",
      "                                    -2.500000000000002,\n",
      "                                    -4.330127018922192\n",
      "                                ],\n",
      "                                [\n",
      "                                    -1.5450849718747377,\n",
      "                                    -4.755282581475767\n",
      "                                ],\n",
      "                                [\n",
      "                                    -0.5226423163382712,\n",
      "                                    -4.972609476841367\n",
      "                                ],\n",
      "                                [\n",
      "                                    0.522642316338265,\n",
      "                                    -4.972609476841367\n",
      "                                ],\n",
      "                                [\n",
      "                                    1.5450849718747361,\n",
      "                                    -4.755282581475768\n",
      "                                ],\n",
      "                                [\n",
      "                                    2.5000000000000004,\n",
      "                                    -4.330127018922193\n",
      "                                ],\n",
      "                                [\n",
      "                                    3.345653031794292,\n",
      "                                    -3.71572412738697\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.045084971874736,\n",
      "                                    -2.938926261462367\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.567727288213005,\n",
      "                                    -2.0336832153790008\n",
      "                                ],\n",
      "                                [\n",
      "                                    4.8907380036690284,\n",
      "                                    -1.039558454088795\n",
      "                                ]\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"carttrans\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                            \"rotation\":\t0.0,\n",
      "                            \"scale\":\t[\n",
      "                                1.0,\n",
      "                                1.0\n",
      "                            ],\n",
      "                            \"translation\":\t[\n",
      "                                414.0354050993562,\n",
      "                                100.0\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"force_mag\":\t10.0,\n",
      "                    \"gravity\":\t9.8,\n",
      "                    \"kinematics_integrator\":\t\"euler\",\n",
      "                    \"length\":\t0.5,\n",
      "                    \"masscart\":\t1.0,\n",
      "                    \"masspole\":\t0.1,\n",
      "                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                    \"observation_space\":\t{\n",
      "                        \"Box(4,)\":\t{\n",
      "                            \"bounded_above\":\t\"[ True  True  True  True]\",\n",
      "                            \"bounded_below\":\t\"[ True  True  True  True]\",\n",
      "                            \"dtype\":\t\"float32\",\n",
      "                            \"high\":\t\"[4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\",\n",
      "                            \"low\":\t\"[-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\",\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"shape\":\t[\n",
      "                                4\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"polemass_length\":\t0.05,\n",
      "                    \"poletrans\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                            \"rotation\":\t-0.047655701937088506,\n",
      "                            \"scale\":\t[\n",
      "                                1.0,\n",
      "                                1.0\n",
      "                            ],\n",
      "                            \"translation\":\t[\n",
      "                                0.0,\n",
      "                                7.5\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"spec\":\t{\n",
      "                        \"EnvSpec(CartPole-v0)\":\t{\n",
      "                            \"_env_name\":\t\"CartPole\",\n",
      "                            \"_kwargs\":\t{},\n",
      "                            \"entry_point\":\t\"gym.envs.classic_control:CartPoleEnv\",\n",
      "                            \"id\":\t\"CartPole-v0\",\n",
      "                            \"max_episode_steps\":\t200,\n",
      "                            \"nondeterministic\":\tfalse,\n",
      "                            \"reward_threshold\":\t195.0,\n",
      "                            \"tags\":\t{\n",
      "                                \"wrapper_config.TimeLimit.max_episode_steps\":\t200\n",
      "                            }\n",
      "                        }\n",
      "                    },\n",
      "                    \"state\":\t\"[-0.03874528 -0.02872086 -0.00417958 -0.02820998]\",\n",
      "                    \"steps_beyond_done\":\tnull,\n",
      "                    \"tau\":\t0.02,\n",
      "                    \"theta_threshold_radians\":\t0.20943951023931953,\n",
      "                    \"total_mass\":\t1.1,\n",
      "                    \"track\":\t{\n",
      "                        \"<gym.envs.classic_control.rendering.Line object at 0x7f5c493e7080>\":\t{\n",
      "                            \"_color\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Color object at 0x7f5c49406ba8>\":\t{\n",
      "                                    \"vec4\":\t[\n",
      "                                        0,\n",
      "                                        0,\n",
      "                                        0,\n",
      "                                        1\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"attrs\":\t[\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.Color object at 0x7f5c49406ba8>\":\t{\n",
      "                                        \"vec4\":\t[\n",
      "                                            0,\n",
      "                                            0,\n",
      "                                            0,\n",
      "                                            1\n",
      "                                        ]\n",
      "                                    }\n",
      "                                },\n",
      "                                {\n",
      "                                    \"<gym.envs.classic_control.rendering.LineWidth object at 0x7f5c49406fd0>\":\t{\n",
      "                                        \"stroke\":\t1\n",
      "                                    }\n",
      "                                }\n",
      "                            ],\n",
      "                            \"end\":\t[\n",
      "                                600,\n",
      "                                100\n",
      "                            ],\n",
      "                            \"linewidth\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.LineWidth object at 0x7f5c49406fd0>\":\t{\n",
      "                                    \"stroke\":\t1\n",
      "                                }\n",
      "                            },\n",
      "                            \"start\":\t[\n",
      "                                0,\n",
      "                                100\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"viewer\":\tnull,\n",
      "                    \"x_threshold\":\t2.4\n",
      "                }\n",
      "            },\n",
      "            \"metadata\":\t{\n",
      "                \"render.modes\":\t[\n",
      "                    \"human\",\n",
      "                    \"rgb_array\"\n",
      "                ],\n",
      "                \"video.frames_per_second\":\t50\n",
      "            },\n",
      "            \"observation_space\":\t{\n",
      "                \"Box(4,)\":\t{\n",
      "                    \"bounded_above\":\t\"[ True  True  True  True]\",\n",
      "                    \"bounded_below\":\t\"[ True  True  True  True]\",\n",
      "                    \"dtype\":\t\"float32\",\n",
      "                    \"high\":\t\"[4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\",\n",
      "                    \"low\":\t\"[-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\",\n",
      "                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                    \"shape\":\t[\n",
      "                        4\n",
      "                    ]\n",
      "                }\n",
      "            },\n",
      "            \"reward_range\":\t[\n",
      "                -Infinity,\n",
      "                Infinity\n",
      "            ]\n",
      "        }\n",
      "    },\n",
      "    \"epochs\":\t50,\n",
      "    \"gamma\":\t0.99,\n",
      "    \"lam\":\t0.97,\n",
      "    \"logger_kwargs\":\t{},\n",
      "    \"max_ep_len\":\t1000,\n",
      "    \"model\":\t\"MLPActorCritic\",\n",
      "    \"obs_dim\":\t[\n",
      "        4\n",
      "    ],\n",
      "    \"optimizer\":\t\"Adam\",\n",
      "    \"pi_lr\":\t0.0003,\n",
      "    \"render\":\tfalse,\n",
      "    \"save_freq\":\t10,\n",
      "    \"self\":\t{\n",
      "        \"<__main__.PPO_Agent object at 0x7f5c4940d9e8>\":\t{\n",
      "            \"ac\":\t{\n",
      "                \"MLPActorCritic(\\n  (pi): MLPCategoricalActor(\\n    (logits_net): Sequential(\\n      (0): Linear(in_features=4, out_features=64, bias=True)\\n      (1): Tanh()\\n      (2): Linear(in_features=64, out_features=64, bias=True)\\n      (3): Tanh()\\n      (4): Linear(in_features=64, out_features=2, bias=True)\\n      (5): Identity()\\n    )\\n  )\\n  (v): MLPCritic(\\n    (v_net): Sequential(\\n      (0): Linear(in_features=4, out_features=64, bias=True)\\n      (1): Tanh()\\n      (2): Linear(in_features=64, out_features=64, bias=True)\\n      (3): Tanh()\\n      (4): Linear(in_features=64, out_features=1, bias=True)\\n      (5): Identity()\\n    )\\n  )\\n)\":\t{\n",
      "                    \"_backward_hooks\":\t{},\n",
      "                    \"_buffers\":\t{},\n",
      "                    \"_forward_hooks\":\t{},\n",
      "                    \"_forward_pre_hooks\":\t{},\n",
      "                    \"_load_state_dict_pre_hooks\":\t{},\n",
      "                    \"_modules\":\t{\n",
      "                        \"pi\":\t{\n",
      "                            \"MLPCategoricalActor(\\n  (logits_net): Sequential(\\n    (0): Linear(in_features=4, out_features=64, bias=True)\\n    (1): Tanh()\\n    (2): Linear(in_features=64, out_features=64, bias=True)\\n    (3): Tanh()\\n    (4): Linear(in_features=64, out_features=2, bias=True)\\n    (5): Identity()\\n  )\\n)\":\t{\n",
      "                                \"_backward_hooks\":\t{},\n",
      "                                \"_buffers\":\t{},\n",
      "                                \"_forward_hooks\":\t{},\n",
      "                                \"_forward_pre_hooks\":\t{},\n",
      "                                \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                \"_modules\":\t{\n",
      "                                    \"logits_net\":\t{\n",
      "                                        \"Sequential(\\n  (0): Linear(in_features=4, out_features=64, bias=True)\\n  (1): Tanh()\\n  (2): Linear(in_features=64, out_features=64, bias=True)\\n  (3): Tanh()\\n  (4): Linear(in_features=64, out_features=2, bias=True)\\n  (5): Identity()\\n)\":\t{\n",
      "                                            \"_backward_hooks\":\t{},\n",
      "                                            \"_buffers\":\t{},\n",
      "                                            \"_forward_hooks\":\t{},\n",
      "                                            \"_forward_pre_hooks\":\t{},\n",
      "                                            \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                            \"_modules\":\t{\n",
      "                                                \"0\":\t{\n",
      "                                                    \"Linear(in_features=4, out_features=64, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([ 0.2180, -0.4633,  0.0878, -0.0481,  0.3598,  0.4560,  0.2959,  0.2989,\\n        -0.3152,  0.1381,  0.0249, -0.4466,  0.4558, -0.0654, -0.3887, -0.2491,\\n        -0.1787,  0.4533,  0.0646, -0.4187,  0.0103,  0.3599,  0.0383,  0.4436,\\n        -0.3418, -0.4376,  0.3456, -0.3948, -0.1364,  0.0949, -0.4863,  0.3808,\\n        -0.0020,  0.1381,  0.3255, -0.3443, -0.0305,  0.0478, -0.0130, -0.4808,\\n        -0.0986, -0.3300,  0.3806,  0.1614,  0.3973,  0.1239, -0.2894, -0.4722,\\n         0.4688, -0.1839,  0.3504, -0.1437, -0.1743,  0.3258, -0.1071, -0.3790,\\n        -0.2558,  0.4896,  0.2300,  0.0223,  0.3480, -0.0201,  0.3922,  0.4385],\\n       requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[ 0.2182,  0.1502,  0.2190,  0.1570],\\n        [-0.0975,  0.3038, -0.2508,  0.4292],\\n        [ 0.0213,  0.3857,  0.2167, -0.3548],\\n        [ 0.4498, -0.1420,  0.1133, -0.3394],\\n        [-0.2016,  0.4640, -0.3121,  0.1686],\\n        [ 0.0695, -0.3104,  0.2271, -0.4669],\\n        [-0.2765,  0.1354,  0.3019, -0.4493],\\n        [ 0.0465, -0.0685, -0.4886,  0.3870],\\n        [-0.0507,  0.0108,  0.3714,  0.0787],\\n        [ 0.4949, -0.0564,  0.4851, -0.0596],\\n        [ 0.2470, -0.0243, -0.0239, -0.4281],\\n        [-0.3040,  0.2113, -0.0694,  0.2426],\\n        [-0.3174,  0.1246,  0.4821, -0.3982],\\n        [-0.4412,  0.3479,  0.0373,  0.4002],\\n        [-0.3844, -0.4317,  0.3026,  0.0091],\\n        [ 0.0321,  0.4794,  0.2691,  0.2008],\\n        [ 0.1334,  0.1296,  0.0871, -0.2280],\\n        [-0.3896,  0.4061,  0.4264, -0.1785],\\n        [-0.4562,  0.2675, -0.1465,  0.1955],\\n        [ 0.0865,  0.0945,  0.4313, -0.2399],\\n        [-0.2657, -0.4643,  0.2161, -0.3022],\\n        [-0.4918,  0.1329, -0.1593,  0.2530],\\n        [ 0.0292,  0.3271, -0.1450,  0.3476],\\n        [ 0.0218, -0.1429, -0.0315,  0.1586],\\n        [ 0.3225, -0.1911, -0.3070, -0.1333],\\n        [-0.1857,  0.0964,  0.2458, -0.3479],\\n        [ 0.1991, -0.1589,  0.4866,  0.2840],\\n        [-0.1273,  0.1852, -0.4674, -0.0240],\\n        [-0.4967, -0.1928, -0.4274,  0.2571],\\n        [-0.2628,  0.1266, -0.1986,  0.4604],\\n        [-0.1418,  0.2583,  0.2092, -0.0363],\\n        [ 0.0971, -0.2821,  0.1842,  0.3372],\\n        [-0.4749, -0.3698, -0.4326, -0.4827],\\n        [-0.4694,  0.4066,  0.3726, -0.0246],\\n        [-0.4485, -0.3710,  0.1005,  0.0321],\\n        [ 0.1190, -0.0534, -0.4345, -0.2616],\\n        [ 0.0889, -0.4557,  0.4455,  0.2464],\\n        [-0.4203,  0.4131, -0.3354, -0.3805],\\n        [ 0.1120,  0.3966,  0.1377, -0.1412],\\n        [-0.1419, -0.2713,  0.3843,  0.2909],\\n        [ 0.1210,  0.1178, -0.4366, -0.3741],\\n        [ 0.0029,  0.0955,  0.4989, -0.2486],\\n        [-0.3300,  0.0374, -0.0474, -0.4097],\\n        [-0.0271, -0.4657,  0.2483, -0.1775],\\n        [-0.1120,  0.3283, -0.3712,  0.1058],\\n        [-0.4768,  0.4389,  0.1024, -0.4137],\\n        [ 0.4964, -0.4126, -0.4245, -0.4758],\\n        [ 0.4206,  0.0826,  0.3870,  0.0466],\\n        [ 0.2256,  0.1224, -0.0396,  0.0215],\\n        [ 0.1606, -0.4854,  0.1921,  0.1478],\\n        [-0.1145,  0.0776, -0.2222,  0.0800],\\n        [ 0.1333,  0.0464,  0.0523,  0.4922],\\n        [-0.1892, -0.1255,  0.4747, -0.4142],\\n        [ 0.3065,  0.3613,  0.1482,  0.0951],\\n        [ 0.3348, -0.2107,  0.3079, -0.3979],\\n        [ 0.3106, -0.2848, -0.1182, -0.3050],\\n        [-0.0157,  0.4854,  0.2167,  0.4795],\\n        [ 0.1900,  0.1209, -0.1876,  0.1083],\\n        [-0.2559,  0.4848, -0.2010, -0.0699],\\n        [ 0.4577, -0.3192, -0.3336, -0.0359],\\n        [-0.1720,  0.2303, -0.0319, -0.4413],\\n        [-0.0078,  0.4234,  0.2529, -0.2104],\\n        [-0.3272, -0.4298,  0.2011, -0.1399],\\n        [ 0.3519,  0.3402, -0.3283, -0.4003]], requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t4,\n",
      "                                                        \"out_features\":\t64,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"1\":\t{\n",
      "                                                    \"Tanh()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"2\":\t{\n",
      "                                                    \"Linear(in_features=64, out_features=64, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([-0.1225, -0.0617, -0.0645, -0.1166,  0.0576, -0.0961, -0.0439,  0.0347,\\n        -0.1191,  0.0641, -0.0507,  0.0841,  0.1247, -0.0112,  0.0304,  0.0361,\\n        -0.0727,  0.0968, -0.1124,  0.0002, -0.0104, -0.0163, -0.0205, -0.0365,\\n        -0.0691,  0.0682, -0.0239,  0.0198,  0.0705,  0.1005,  0.0704,  0.0365,\\n        -0.0361,  0.0932, -0.0621, -0.1168,  0.0968, -0.0380,  0.1074, -0.0406,\\n         0.1181, -0.0117,  0.0873,  0.0036, -0.1201,  0.0726, -0.0205,  0.0735,\\n         0.1182,  0.0905, -0.0347, -0.0859,  0.0030,  0.1154, -0.0922,  0.0364,\\n        -0.0104,  0.0075, -0.0739,  0.0239,  0.1103, -0.0884,  0.0597,  0.0150],\\n       requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[ 0.0406, -0.1061,  0.1227,  ..., -0.0614,  0.0167,  0.0881],\\n        [ 0.0994,  0.0748,  0.1234,  ..., -0.0019, -0.1098,  0.0462],\\n        [-0.0694, -0.0946,  0.0164,  ..., -0.0399,  0.1116,  0.0072],\\n        ...,\\n        [-0.0482, -0.1249, -0.1149,  ...,  0.1223,  0.0101, -0.0382],\\n        [ 0.0416, -0.0935, -0.0401,  ..., -0.0620, -0.0250, -0.0935],\\n        [ 0.0438,  0.0508,  0.0576,  ..., -0.0366, -0.0214,  0.0408]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t64,\n",
      "                                                        \"out_features\":\t64,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"3\":\t{\n",
      "                                                    \"Tanh()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"4\":\t{\n",
      "                                                    \"Linear(in_features=64, out_features=2, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([-0.0315,  0.0003], requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[ 0.0119, -0.0910, -0.0359, -0.0299, -0.0802, -0.0171, -0.0461,  0.0998,\\n          0.0136, -0.1118, -0.0996,  0.0528,  0.0598, -0.1069, -0.1150,  0.0086,\\n         -0.0524,  0.0871, -0.0861,  0.1161, -0.0421,  0.0187,  0.0336, -0.1065,\\n         -0.1221,  0.0843,  0.0429, -0.0311,  0.0816, -0.0881,  0.0718,  0.0618,\\n          0.0783, -0.0265, -0.0182, -0.1131, -0.0907,  0.0460, -0.1202, -0.0170,\\n         -0.0782, -0.0421, -0.0980,  0.0364,  0.0589,  0.1225, -0.0884,  0.0413,\\n         -0.0683,  0.0673, -0.0350, -0.0703, -0.0687,  0.0937,  0.0537, -0.0411,\\n          0.0511,  0.0389,  0.0052, -0.1069,  0.0869, -0.0129,  0.0618, -0.1218],\\n        [ 0.0109, -0.1020,  0.0132,  0.0520, -0.0236,  0.0461,  0.0561,  0.0005,\\n         -0.0724,  0.0471,  0.0990,  0.0688,  0.0800, -0.0468, -0.1008, -0.1060,\\n          0.0472,  0.0666, -0.1002,  0.0651,  0.0943,  0.1084, -0.0049, -0.0104,\\n          0.0671,  0.0366, -0.0139,  0.0387,  0.1050, -0.0339,  0.1229, -0.0305,\\n         -0.0515, -0.0809,  0.1056,  0.0919,  0.1249, -0.0960,  0.0352, -0.0445,\\n          0.0529,  0.0162, -0.0857,  0.1205,  0.1249, -0.0210,  0.0957,  0.1077,\\n         -0.1247, -0.1006, -0.0902, -0.0767,  0.1098, -0.0349,  0.0644, -0.0983,\\n          0.0940, -0.0513, -0.0419,  0.0535,  0.0670,  0.0406,  0.0186,  0.0802]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t64,\n",
      "                                                        \"out_features\":\t2,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"5\":\t{\n",
      "                                                    \"Identity()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                }\n",
      "                                            },\n",
      "                                            \"_parameters\":\t{},\n",
      "                                            \"_state_dict_hooks\":\t{},\n",
      "                                            \"training\":\ttrue\n",
      "                                        }\n",
      "                                    }\n",
      "                                },\n",
      "                                \"_parameters\":\t{},\n",
      "                                \"_state_dict_hooks\":\t{},\n",
      "                                \"training\":\ttrue\n",
      "                            }\n",
      "                        },\n",
      "                        \"v\":\t{\n",
      "                            \"MLPCritic(\\n  (v_net): Sequential(\\n    (0): Linear(in_features=4, out_features=64, bias=True)\\n    (1): Tanh()\\n    (2): Linear(in_features=64, out_features=64, bias=True)\\n    (3): Tanh()\\n    (4): Linear(in_features=64, out_features=1, bias=True)\\n    (5): Identity()\\n  )\\n)\":\t{\n",
      "                                \"_backward_hooks\":\t{},\n",
      "                                \"_buffers\":\t{},\n",
      "                                \"_forward_hooks\":\t{},\n",
      "                                \"_forward_pre_hooks\":\t{},\n",
      "                                \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                \"_modules\":\t{\n",
      "                                    \"v_net\":\t{\n",
      "                                        \"Sequential(\\n  (0): Linear(in_features=4, out_features=64, bias=True)\\n  (1): Tanh()\\n  (2): Linear(in_features=64, out_features=64, bias=True)\\n  (3): Tanh()\\n  (4): Linear(in_features=64, out_features=1, bias=True)\\n  (5): Identity()\\n)\":\t{\n",
      "                                            \"_backward_hooks\":\t{},\n",
      "                                            \"_buffers\":\t{},\n",
      "                                            \"_forward_hooks\":\t{},\n",
      "                                            \"_forward_pre_hooks\":\t{},\n",
      "                                            \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                            \"_modules\":\t{\n",
      "                                                \"0\":\t{\n",
      "                                                    \"Linear(in_features=4, out_features=64, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([ 0.2301, -0.1723,  0.2125,  0.0330, -0.0833, -0.0644, -0.2025, -0.2111,\\n         0.3983,  0.1534,  0.2977, -0.0365,  0.3636,  0.2069, -0.3122, -0.4690,\\n        -0.0993, -0.4728,  0.2546,  0.1855,  0.4169, -0.0657, -0.3313, -0.3148,\\n         0.4304,  0.3751,  0.0659,  0.2636, -0.4452, -0.3182,  0.2270,  0.4189,\\n         0.4857,  0.0636, -0.0642,  0.0602, -0.4869,  0.1228, -0.3612, -0.1643,\\n         0.3477,  0.4739,  0.4626, -0.3879,  0.4825,  0.4246,  0.2210,  0.4413,\\n         0.0482, -0.4683, -0.0701,  0.4940,  0.1916, -0.0415,  0.1077,  0.1163,\\n         0.3196,  0.2687,  0.4177, -0.0242, -0.2623, -0.3845,  0.4229, -0.4380],\\n       requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-0.3046, -0.1918,  0.0523,  0.3271],\\n        [-0.1415, -0.4875, -0.0637, -0.3759],\\n        [-0.4630, -0.2976, -0.3494, -0.1698],\\n        [-0.2379, -0.2897,  0.1577, -0.3999],\\n        [-0.2167, -0.0742,  0.2768,  0.2149],\\n        [-0.4259, -0.1657, -0.0945, -0.1095],\\n        [ 0.2064,  0.4486, -0.1387, -0.4114],\\n        [-0.0912, -0.0291, -0.3227, -0.1379],\\n        [ 0.1627, -0.3350,  0.4531,  0.2289],\\n        [ 0.1905,  0.0685, -0.1709, -0.2482],\\n        [-0.1533, -0.2316, -0.4476,  0.3658],\\n        [-0.4819,  0.4891,  0.0079,  0.1391],\\n        [-0.4716,  0.1511,  0.0481,  0.1660],\\n        [-0.0415,  0.2473,  0.3640,  0.0599],\\n        [-0.4459, -0.0894, -0.4943, -0.0590],\\n        [ 0.3453,  0.2564, -0.4064,  0.0843],\\n        [-0.4408, -0.4239, -0.3204, -0.3726],\\n        [-0.3426, -0.1770, -0.1594,  0.1186],\\n        [ 0.2905, -0.4371, -0.4643, -0.2343],\\n        [-0.4657, -0.2208, -0.1578,  0.3718],\\n        [ 0.2675, -0.4389,  0.1728, -0.0609],\\n        [ 0.0612, -0.4578, -0.0491,  0.0720],\\n        [-0.1718, -0.4770,  0.1738,  0.4588],\\n        [-0.2767,  0.2978, -0.4757,  0.2273],\\n        [-0.0092,  0.4414, -0.0186, -0.4925],\\n        [-0.4736,  0.1219,  0.3363,  0.0415],\\n        [-0.3901, -0.2408,  0.3166,  0.2090],\\n        [-0.0096, -0.3663, -0.3221,  0.3964],\\n        [-0.3949, -0.2505,  0.0146,  0.3000],\\n        [-0.4308, -0.2378, -0.1647, -0.1867],\\n        [-0.1784, -0.1501,  0.2326, -0.2588],\\n        [-0.0702, -0.0813,  0.0289,  0.0428],\\n        [ 0.0849, -0.0435,  0.1627,  0.3724],\\n        [ 0.1842,  0.4236,  0.1706,  0.0365],\\n        [ 0.4441, -0.0103,  0.0107, -0.2948],\\n        [-0.0369,  0.3381, -0.2350, -0.3527],\\n        [-0.2363, -0.3043,  0.4807,  0.2524],\\n        [-0.1136,  0.3661, -0.1946,  0.4539],\\n        [ 0.2163,  0.4419,  0.1724,  0.4253],\\n        [-0.4434, -0.1263,  0.4149,  0.1217],\\n        [-0.1798,  0.4687, -0.1688,  0.1493],\\n        [ 0.3912,  0.0923,  0.2357,  0.0719],\\n        [-0.2328, -0.1624, -0.2533, -0.1228],\\n        [-0.0172, -0.0785, -0.0834, -0.3193],\\n        [ 0.1099, -0.0978,  0.3939, -0.2596],\\n        [ 0.1560, -0.3776,  0.3001, -0.1504],\\n        [-0.2786, -0.2878, -0.3424, -0.3911],\\n        [ 0.1133,  0.0829,  0.2896, -0.2677],\\n        [-0.1146, -0.3145,  0.0093,  0.3290],\\n        [ 0.2363,  0.0456,  0.2930, -0.3394],\\n        [ 0.2647, -0.0007, -0.2162, -0.1204],\\n        [ 0.4828,  0.1533, -0.2487, -0.3262],\\n        [ 0.0922,  0.0774,  0.0341, -0.2173],\\n        [-0.3901, -0.4961,  0.3223, -0.2756],\\n        [-0.1942, -0.4203,  0.1836, -0.1201],\\n        [-0.3856,  0.2215, -0.4257, -0.4785],\\n        [ 0.0197, -0.4667, -0.1265,  0.4570],\\n        [ 0.0467,  0.2426, -0.4953, -0.4371],\\n        [-0.3449, -0.2379,  0.0808, -0.2834],\\n        [-0.2662, -0.3645,  0.0479, -0.1848],\\n        [-0.1354,  0.1759,  0.0200, -0.4311],\\n        [ 0.0451,  0.1949,  0.2435,  0.3584],\\n        [-0.4296,  0.2431, -0.2296, -0.3621],\\n        [-0.4085,  0.2420,  0.0142,  0.3426]], requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t4,\n",
      "                                                        \"out_features\":\t64,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"1\":\t{\n",
      "                                                    \"Tanh()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"2\":\t{\n",
      "                                                    \"Linear(in_features=64, out_features=64, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([-0.0874, -0.0215, -0.0205,  0.0957, -0.1089,  0.1017, -0.0802,  0.1077,\\n        -0.0370, -0.0731, -0.0715,  0.0465,  0.0521,  0.0769,  0.1241, -0.0209,\\n         0.0433, -0.0176,  0.0071, -0.0418, -0.1172, -0.1009,  0.0621, -0.0339,\\n        -0.0633, -0.0716,  0.0639,  0.0775,  0.0496,  0.0224, -0.0961, -0.0269,\\n        -0.1183, -0.0148,  0.0597,  0.0857,  0.0229,  0.0402, -0.0143, -0.1061,\\n        -0.0525,  0.0328,  0.0683,  0.0164,  0.0513,  0.0359,  0.0015, -0.0269,\\n         0.0636, -0.0164, -0.1185, -0.0451, -0.0472, -0.0164, -0.0735,  0.1169,\\n         0.0244,  0.1184,  0.0783,  0.0016,  0.0994, -0.0495,  0.1106, -0.0335],\\n       requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-0.0073,  0.0402, -0.0251,  ..., -0.0138, -0.0635,  0.0215],\\n        [ 0.0494,  0.0854, -0.0403,  ...,  0.0784, -0.1188,  0.0727],\\n        [-0.0558,  0.0055,  0.0922,  ...,  0.0349,  0.0725, -0.1028],\\n        ...,\\n        [ 0.0195, -0.0718, -0.0365,  ...,  0.0369,  0.0174, -0.0408],\\n        [ 0.0390, -0.0564,  0.0868,  ..., -0.0269,  0.1140, -0.0686],\\n        [ 0.0545, -0.1144, -0.0796,  ..., -0.0106,  0.0728,  0.0141]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t64,\n",
      "                                                        \"out_features\":\t64,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"3\":\t{\n",
      "                                                    \"Tanh()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"4\":\t{\n",
      "                                                    \"Linear(in_features=64, out_features=1, bias=True)\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{\n",
      "                                                            \"bias\":\t\"Parameter containing:\\ntensor([0.0775], requires_grad=True)\",\n",
      "                                                            \"weight\":\t\"Parameter containing:\\ntensor([[-0.0645, -0.0927,  0.0723, -0.0304,  0.0263, -0.0037, -0.0324, -0.1015,\\n          0.1027,  0.0147, -0.0703,  0.0511,  0.0867, -0.1199, -0.0923, -0.0811,\\n          0.0481,  0.0487, -0.0832,  0.0528, -0.1204,  0.0258,  0.0945, -0.0956,\\n          0.0254, -0.1043,  0.0649,  0.1168, -0.0513,  0.0184, -0.0009, -0.1168,\\n          0.0770,  0.0619,  0.0639,  0.0416, -0.0984,  0.0049,  0.0571,  0.0556,\\n          0.0948,  0.0830,  0.0604,  0.0819, -0.0367,  0.0424, -0.0619,  0.0499,\\n         -0.0676, -0.0704, -0.1057, -0.0082, -0.0049, -0.0838, -0.0141, -0.0631,\\n         -0.0845,  0.0386, -0.0969,  0.0832,  0.0489,  0.0146,  0.0960,  0.0233]],\\n       requires_grad=True)\"\n",
      "                                                        },\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"in_features\":\t64,\n",
      "                                                        \"out_features\":\t1,\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                },\n",
      "                                                \"5\":\t{\n",
      "                                                    \"Identity()\":\t{\n",
      "                                                        \"_backward_hooks\":\t{},\n",
      "                                                        \"_buffers\":\t{},\n",
      "                                                        \"_forward_hooks\":\t{},\n",
      "                                                        \"_forward_pre_hooks\":\t{},\n",
      "                                                        \"_load_state_dict_pre_hooks\":\t{},\n",
      "                                                        \"_modules\":\t{},\n",
      "                                                        \"_parameters\":\t{},\n",
      "                                                        \"_state_dict_hooks\":\t{},\n",
      "                                                        \"training\":\ttrue\n",
      "                                                    }\n",
      "                                                }\n",
      "                                            },\n",
      "                                            \"_parameters\":\t{},\n",
      "                                            \"_state_dict_hooks\":\t{},\n",
      "                                            \"training\":\ttrue\n",
      "                                        }\n",
      "                                    }\n",
      "                                },\n",
      "                                \"_parameters\":\t{},\n",
      "                                \"_state_dict_hooks\":\t{},\n",
      "                                \"training\":\ttrue\n",
      "                            }\n",
      "                        }\n",
      "                    },\n",
      "                    \"_parameters\":\t{},\n",
      "                    \"_state_dict_hooks\":\t{},\n",
      "                    \"training\":\ttrue\n",
      "                }\n",
      "            },\n",
      "            \"buf\":\t{\n",
      "                \"<__main__.Buffer object at 0x7f5c250db0b8>\":\t{\n",
      "                    \"act_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"adv_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"gamma\":\t0.99,\n",
      "                    \"lam\":\t0.97,\n",
      "                    \"logp_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"max_size\":\t4000,\n",
      "                    \"obs_buf\":\t\"[[0. 0. 0. 0.]\\n [0. 0. 0. 0.]\\n [0. 0. 0. 0.]\\n ...\\n [0. 0. 0. 0.]\\n [0. 0. 0. 0.]\\n [0. 0. 0. 0.]]\",\n",
      "                    \"path_start_idx\":\t0,\n",
      "                    \"ptr\":\t0,\n",
      "                    \"ret_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"rew_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\",\n",
      "                    \"val_buf\":\t\"[0. 0. 0. ... 0. 0. 0.]\"\n",
      "                }\n",
      "            },\n",
      "            \"env\":\t{\n",
      "                \"<TimeLimit<CartPoleEnv<CartPole-v0>>>\":\t{\n",
      "                    \"_elapsed_steps\":\t0,\n",
      "                    \"_max_episode_steps\":\t200,\n",
      "                    \"action_space\":\t{\n",
      "                        \"Discrete(2)\":\t{\n",
      "                            \"dtype\":\t\"int64\",\n",
      "                            \"n\":\t2,\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"shape\":\t[]\n",
      "                        }\n",
      "                    },\n",
      "                    \"env\":\t{\n",
      "                        \"<CartPoleEnv<CartPole-v0>>\":\t{\n",
      "                            \"_pole_geom\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.FilledPolygon object at 0x7f5c493e7208>\":\t{\n",
      "                                    \"_color\":\t{\n",
      "                                        \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7358>\":\t{\n",
      "                                            \"vec4\":\t[\n",
      "                                                0.8,\n",
      "                                                0.6,\n",
      "                                                0.4,\n",
      "                                                1\n",
      "                                            ]\n",
      "                                        }\n",
      "                                    },\n",
      "                                    \"attrs\":\t[\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7358>\":\t{\n",
      "                                                \"vec4\":\t[\n",
      "                                                    0.8,\n",
      "                                                    0.6,\n",
      "                                                    0.4,\n",
      "                                                    1\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                                \"rotation\":\t-0.047655701937088506,\n",
      "                                                \"scale\":\t[\n",
      "                                                    1.0,\n",
      "                                                    1.0\n",
      "                                                ],\n",
      "                                                \"translation\":\t[\n",
      "                                                    0.0,\n",
      "                                                    7.5\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                                \"rotation\":\t0.0,\n",
      "                                                \"scale\":\t[\n",
      "                                                    1.0,\n",
      "                                                    1.0\n",
      "                                                ],\n",
      "                                                \"translation\":\t[\n",
      "                                                    414.0354050993562,\n",
      "                                                    100.0\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        }\n",
      "                                    ],\n",
      "                                    \"v\":\t[\n",
      "                                        [\n",
      "                                            -5.0,\n",
      "                                            -5.0\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -5.0,\n",
      "                                            120.0\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            5.0,\n",
      "                                            120.0\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            5.0,\n",
      "                                            -5.0\n",
      "                                        ]\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"action_space\":\t{\n",
      "                                \"Discrete(2)\":\t{\n",
      "                                    \"dtype\":\t\"int64\",\n",
      "                                    \"n\":\t2,\n",
      "                                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                                    \"shape\":\t[]\n",
      "                                }\n",
      "                            },\n",
      "                            \"axle\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.FilledPolygon object at 0x7f5c493e70b8>\":\t{\n",
      "                                    \"_color\":\t{\n",
      "                                        \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7048>\":\t{\n",
      "                                            \"vec4\":\t[\n",
      "                                                0.5,\n",
      "                                                0.5,\n",
      "                                                0.8,\n",
      "                                                1\n",
      "                                            ]\n",
      "                                        }\n",
      "                                    },\n",
      "                                    \"attrs\":\t[\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Color object at 0x7f5c493e7048>\":\t{\n",
      "                                                \"vec4\":\t[\n",
      "                                                    0.5,\n",
      "                                                    0.5,\n",
      "                                                    0.8,\n",
      "                                                    1\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                                \"rotation\":\t-0.047655701937088506,\n",
      "                                                \"scale\":\t[\n",
      "                                                    1.0,\n",
      "                                                    1.0\n",
      "                                                ],\n",
      "                                                \"translation\":\t[\n",
      "                                                    0.0,\n",
      "                                                    7.5\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                                \"rotation\":\t0.0,\n",
      "                                                \"scale\":\t[\n",
      "                                                    1.0,\n",
      "                                                    1.0\n",
      "                                                ],\n",
      "                                                \"translation\":\t[\n",
      "                                                    414.0354050993562,\n",
      "                                                    100.0\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        }\n",
      "                                    ],\n",
      "                                    \"v\":\t[\n",
      "                                        [\n",
      "                                            5.0,\n",
      "                                            0.0\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.8907380036690284,\n",
      "                                            1.0395584540887965\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.567727288213004,\n",
      "                                            2.0336832153790008\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.045084971874737,\n",
      "                                            2.938926261462366\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            3.345653031794291,\n",
      "                                            3.7157241273869706\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            2.5000000000000004,\n",
      "                                            4.330127018922193\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            1.5450849718747373,\n",
      "                                            4.755282581475767\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            0.5226423163382673,\n",
      "                                            4.972609476841367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -0.5226423163382666,\n",
      "                                            4.972609476841367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -1.5450849718747368,\n",
      "                                            4.755282581475768\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -2.499999999999999,\n",
      "                                            4.330127018922194\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -3.3456530317942894,\n",
      "                                            3.7157241273869723\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.045084971874736,\n",
      "                                            2.9389262614623664\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.567727288213005,\n",
      "                                            2.0336832153790003\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.8907380036690284,\n",
      "                                            1.0395584540887965\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -5.0,\n",
      "                                            2.8327694488239896e-15\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.8907380036690284,\n",
      "                                            -1.0395584540887954\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.567727288213004,\n",
      "                                            -2.033683215379001\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -4.045084971874738,\n",
      "                                            -2.938926261462365\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -3.345653031794292,\n",
      "                                            -3.71572412738697\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -2.500000000000002,\n",
      "                                            -4.330127018922192\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -1.5450849718747377,\n",
      "                                            -4.755282581475767\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            -0.5226423163382712,\n",
      "                                            -4.972609476841367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            0.522642316338265,\n",
      "                                            -4.972609476841367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            1.5450849718747361,\n",
      "                                            -4.755282581475768\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            2.5000000000000004,\n",
      "                                            -4.330127018922193\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            3.345653031794292,\n",
      "                                            -3.71572412738697\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.045084971874736,\n",
      "                                            -2.938926261462367\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.567727288213005,\n",
      "                                            -2.0336832153790008\n",
      "                                        ],\n",
      "                                        [\n",
      "                                            4.8907380036690284,\n",
      "                                            -1.039558454088795\n",
      "                                        ]\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"carttrans\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e70f0>\":\t{\n",
      "                                    \"rotation\":\t0.0,\n",
      "                                    \"scale\":\t[\n",
      "                                        1.0,\n",
      "                                        1.0\n",
      "                                    ],\n",
      "                                    \"translation\":\t[\n",
      "                                        414.0354050993562,\n",
      "                                        100.0\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"force_mag\":\t10.0,\n",
      "                            \"gravity\":\t9.8,\n",
      "                            \"kinematics_integrator\":\t\"euler\",\n",
      "                            \"length\":\t0.5,\n",
      "                            \"masscart\":\t1.0,\n",
      "                            \"masspole\":\t0.1,\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"observation_space\":\t{\n",
      "                                \"Box(4,)\":\t{\n",
      "                                    \"bounded_above\":\t\"[ True  True  True  True]\",\n",
      "                                    \"bounded_below\":\t\"[ True  True  True  True]\",\n",
      "                                    \"dtype\":\t\"float32\",\n",
      "                                    \"high\":\t\"[4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\",\n",
      "                                    \"low\":\t\"[-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\",\n",
      "                                    \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                                    \"shape\":\t[\n",
      "                                        4\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"polemass_length\":\t0.05,\n",
      "                            \"poletrans\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Transform object at 0x7f5c493e7240>\":\t{\n",
      "                                    \"rotation\":\t-0.047655701937088506,\n",
      "                                    \"scale\":\t[\n",
      "                                        1.0,\n",
      "                                        1.0\n",
      "                                    ],\n",
      "                                    \"translation\":\t[\n",
      "                                        0.0,\n",
      "                                        7.5\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"spec\":\t{\n",
      "                                \"EnvSpec(CartPole-v0)\":\t{\n",
      "                                    \"_env_name\":\t\"CartPole\",\n",
      "                                    \"_kwargs\":\t{},\n",
      "                                    \"entry_point\":\t\"gym.envs.classic_control:CartPoleEnv\",\n",
      "                                    \"id\":\t\"CartPole-v0\",\n",
      "                                    \"max_episode_steps\":\t200,\n",
      "                                    \"nondeterministic\":\tfalse,\n",
      "                                    \"reward_threshold\":\t195.0,\n",
      "                                    \"tags\":\t{\n",
      "                                        \"wrapper_config.TimeLimit.max_episode_steps\":\t200\n",
      "                                    }\n",
      "                                }\n",
      "                            },\n",
      "                            \"state\":\t\"[-0.03874528 -0.02872086 -0.00417958 -0.02820998]\",\n",
      "                            \"steps_beyond_done\":\tnull,\n",
      "                            \"tau\":\t0.02,\n",
      "                            \"theta_threshold_radians\":\t0.20943951023931953,\n",
      "                            \"total_mass\":\t1.1,\n",
      "                            \"track\":\t{\n",
      "                                \"<gym.envs.classic_control.rendering.Line object at 0x7f5c493e7080>\":\t{\n",
      "                                    \"_color\":\t{\n",
      "                                        \"<gym.envs.classic_control.rendering.Color object at 0x7f5c49406ba8>\":\t{\n",
      "                                            \"vec4\":\t[\n",
      "                                                0,\n",
      "                                                0,\n",
      "                                                0,\n",
      "                                                1\n",
      "                                            ]\n",
      "                                        }\n",
      "                                    },\n",
      "                                    \"attrs\":\t[\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.Color object at 0x7f5c49406ba8>\":\t{\n",
      "                                                \"vec4\":\t[\n",
      "                                                    0,\n",
      "                                                    0,\n",
      "                                                    0,\n",
      "                                                    1\n",
      "                                                ]\n",
      "                                            }\n",
      "                                        },\n",
      "                                        {\n",
      "                                            \"<gym.envs.classic_control.rendering.LineWidth object at 0x7f5c49406fd0>\":\t{\n",
      "                                                \"stroke\":\t1\n",
      "                                            }\n",
      "                                        }\n",
      "                                    ],\n",
      "                                    \"end\":\t[\n",
      "                                        600,\n",
      "                                        100\n",
      "                                    ],\n",
      "                                    \"linewidth\":\t{\n",
      "                                        \"<gym.envs.classic_control.rendering.LineWidth object at 0x7f5c49406fd0>\":\t{\n",
      "                                            \"stroke\":\t1\n",
      "                                        }\n",
      "                                    },\n",
      "                                    \"start\":\t[\n",
      "                                        0,\n",
      "                                        100\n",
      "                                    ]\n",
      "                                }\n",
      "                            },\n",
      "                            \"viewer\":\tnull,\n",
      "                            \"x_threshold\":\t2.4\n",
      "                        }\n",
      "                    },\n",
      "                    \"metadata\":\t{\n",
      "                        \"render.modes\":\t[\n",
      "                            \"human\",\n",
      "                            \"rgb_array\"\n",
      "                        ],\n",
      "                        \"video.frames_per_second\":\t50\n",
      "                    },\n",
      "                    \"observation_space\":\t{\n",
      "                        \"Box(4,)\":\t{\n",
      "                            \"bounded_above\":\t\"[ True  True  True  True]\",\n",
      "                            \"bounded_below\":\t\"[ True  True  True  True]\",\n",
      "                            \"dtype\":\t\"float32\",\n",
      "                            \"high\":\t\"[4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38]\",\n",
      "                            \"low\":\t\"[-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38]\",\n",
      "                            \"np_random\":\t\"RandomState(MT19937)\",\n",
      "                            \"shape\":\t[\n",
      "                                4\n",
      "                            ]\n",
      "                        }\n",
      "                    },\n",
      "                    \"reward_range\":\t[\n",
      "                        -Infinity,\n",
      "                        Infinity\n",
      "                    ]\n",
      "                }\n",
      "            },\n",
      "            \"epochs\":\t50,\n",
      "            \"local_steps_per_epoch\":\t4000,\n",
      "            \"logger\":\t{\n",
      "                \"<spinup.utils.logx.EpochLogger object at 0x7f5c250db978>\":\t{\n",
      "                    \"epoch_dict\":\t{},\n",
      "                    \"exp_name\":\tnull,\n",
      "                    \"first_row\":\ttrue,\n",
      "                    \"log_current_row\":\t{},\n",
      "                    \"log_headers\":\t[],\n",
      "                    \"output_dir\":\t\"/tmp/experiments/1646381990\",\n",
      "                    \"output_file\":\t{\n",
      "                        \"<_io.TextIOWrapper name='/tmp/experiments/1646381990/progress.txt' mode='w' encoding='UTF-8'>\":\t{\n",
      "                            \"mode\":\t\"w\"\n",
      "                        }\n",
      "                    }\n",
      "                }\n",
      "            },\n",
      "            \"max_ep_len\":\t1000,\n",
      "            \"pi_optimizer\":\t{\n",
      "                \"Adam (\\nParameter Group 0\\n    amsgrad: False\\n    betas: (0.9, 0.999)\\n    eps: 1e-08\\n    lr: 0.0003\\n    weight_decay: 0\\n)\":\t{\n",
      "                    \"defaults\":\t{\n",
      "                        \"amsgrad\":\tfalse,\n",
      "                        \"betas\":\t[\n",
      "                            0.9,\n",
      "                            0.999\n",
      "                        ],\n",
      "                        \"eps\":\t1e-08,\n",
      "                        \"lr\":\t0.0003,\n",
      "                        \"weight_decay\":\t0\n",
      "                    },\n",
      "                    \"param_groups\":\t[\n",
      "                        {\n",
      "                            \"amsgrad\":\tfalse,\n",
      "                            \"betas\":\t[\n",
      "                                0.9,\n",
      "                                0.999\n",
      "                            ],\n",
      "                            \"eps\":\t1e-08,\n",
      "                            \"lr\":\t0.0003,\n",
      "                            \"params\":\t[\n",
      "                                \"Parameter containing:\\ntensor([[ 0.2182,  0.1502,  0.2190,  0.1570],\\n        [-0.0975,  0.3038, -0.2508,  0.4292],\\n        [ 0.0213,  0.3857,  0.2167, -0.3548],\\n        [ 0.4498, -0.1420,  0.1133, -0.3394],\\n        [-0.2016,  0.4640, -0.3121,  0.1686],\\n        [ 0.0695, -0.3104,  0.2271, -0.4669],\\n        [-0.2765,  0.1354,  0.3019, -0.4493],\\n        [ 0.0465, -0.0685, -0.4886,  0.3870],\\n        [-0.0507,  0.0108,  0.3714,  0.0787],\\n        [ 0.4949, -0.0564,  0.4851, -0.0596],\\n        [ 0.2470, -0.0243, -0.0239, -0.4281],\\n        [-0.3040,  0.2113, -0.0694,  0.2426],\\n        [-0.3174,  0.1246,  0.4821, -0.3982],\\n        [-0.4412,  0.3479,  0.0373,  0.4002],\\n        [-0.3844, -0.4317,  0.3026,  0.0091],\\n        [ 0.0321,  0.4794,  0.2691,  0.2008],\\n        [ 0.1334,  0.1296,  0.0871, -0.2280],\\n        [-0.3896,  0.4061,  0.4264, -0.1785],\\n        [-0.4562,  0.2675, -0.1465,  0.1955],\\n        [ 0.0865,  0.0945,  0.4313, -0.2399],\\n        [-0.2657, -0.4643,  0.2161, -0.3022],\\n        [-0.4918,  0.1329, -0.1593,  0.2530],\\n        [ 0.0292,  0.3271, -0.1450,  0.3476],\\n        [ 0.0218, -0.1429, -0.0315,  0.1586],\\n        [ 0.3225, -0.1911, -0.3070, -0.1333],\\n        [-0.1857,  0.0964,  0.2458, -0.3479],\\n        [ 0.1991, -0.1589,  0.4866,  0.2840],\\n        [-0.1273,  0.1852, -0.4674, -0.0240],\\n        [-0.4967, -0.1928, -0.4274,  0.2571],\\n        [-0.2628,  0.1266, -0.1986,  0.4604],\\n        [-0.1418,  0.2583,  0.2092, -0.0363],\\n        [ 0.0971, -0.2821,  0.1842,  0.3372],\\n        [-0.4749, -0.3698, -0.4326, -0.4827],\\n        [-0.4694,  0.4066,  0.3726, -0.0246],\\n        [-0.4485, -0.3710,  0.1005,  0.0321],\\n        [ 0.1190, -0.0534, -0.4345, -0.2616],\\n        [ 0.0889, -0.4557,  0.4455,  0.2464],\\n        [-0.4203,  0.4131, -0.3354, -0.3805],\\n        [ 0.1120,  0.3966,  0.1377, -0.1412],\\n        [-0.1419, -0.2713,  0.3843,  0.2909],\\n        [ 0.1210,  0.1178, -0.4366, -0.3741],\\n        [ 0.0029,  0.0955,  0.4989, -0.2486],\\n        [-0.3300,  0.0374, -0.0474, -0.4097],\\n        [-0.0271, -0.4657,  0.2483, -0.1775],\\n        [-0.1120,  0.3283, -0.3712,  0.1058],\\n        [-0.4768,  0.4389,  0.1024, -0.4137],\\n        [ 0.4964, -0.4126, -0.4245, -0.4758],\\n        [ 0.4206,  0.0826,  0.3870,  0.0466],\\n        [ 0.2256,  0.1224, -0.0396,  0.0215],\\n        [ 0.1606, -0.4854,  0.1921,  0.1478],\\n        [-0.1145,  0.0776, -0.2222,  0.0800],\\n        [ 0.1333,  0.0464,  0.0523,  0.4922],\\n        [-0.1892, -0.1255,  0.4747, -0.4142],\\n        [ 0.3065,  0.3613,  0.1482,  0.0951],\\n        [ 0.3348, -0.2107,  0.3079, -0.3979],\\n        [ 0.3106, -0.2848, -0.1182, -0.3050],\\n        [-0.0157,  0.4854,  0.2167,  0.4795],\\n        [ 0.1900,  0.1209, -0.1876,  0.1083],\\n        [-0.2559,  0.4848, -0.2010, -0.0699],\\n        [ 0.4577, -0.3192, -0.3336, -0.0359],\\n        [-0.1720,  0.2303, -0.0319, -0.4413],\\n        [-0.0078,  0.4234,  0.2529, -0.2104],\\n        [-0.3272, -0.4298,  0.2011, -0.1399],\\n        [ 0.3519,  0.3402, -0.3283, -0.4003]], requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([ 0.2180, -0.4633,  0.0878, -0.0481,  0.3598,  0.4560,  0.2959,  0.2989,\\n        -0.3152,  0.1381,  0.0249, -0.4466,  0.4558, -0.0654, -0.3887, -0.2491,\\n        -0.1787,  0.4533,  0.0646, -0.4187,  0.0103,  0.3599,  0.0383,  0.4436,\\n        -0.3418, -0.4376,  0.3456, -0.3948, -0.1364,  0.0949, -0.4863,  0.3808,\\n        -0.0020,  0.1381,  0.3255, -0.3443, -0.0305,  0.0478, -0.0130, -0.4808,\\n        -0.0986, -0.3300,  0.3806,  0.1614,  0.3973,  0.1239, -0.2894, -0.4722,\\n         0.4688, -0.1839,  0.3504, -0.1437, -0.1743,  0.3258, -0.1071, -0.3790,\\n        -0.2558,  0.4896,  0.2300,  0.0223,  0.3480, -0.0201,  0.3922,  0.4385],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([[ 0.0406, -0.1061,  0.1227,  ..., -0.0614,  0.0167,  0.0881],\\n        [ 0.0994,  0.0748,  0.1234,  ..., -0.0019, -0.1098,  0.0462],\\n        [-0.0694, -0.0946,  0.0164,  ..., -0.0399,  0.1116,  0.0072],\\n        ...,\\n        [-0.0482, -0.1249, -0.1149,  ...,  0.1223,  0.0101, -0.0382],\\n        [ 0.0416, -0.0935, -0.0401,  ..., -0.0620, -0.0250, -0.0935],\\n        [ 0.0438,  0.0508,  0.0576,  ..., -0.0366, -0.0214,  0.0408]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([-0.1225, -0.0617, -0.0645, -0.1166,  0.0576, -0.0961, -0.0439,  0.0347,\\n        -0.1191,  0.0641, -0.0507,  0.0841,  0.1247, -0.0112,  0.0304,  0.0361,\\n        -0.0727,  0.0968, -0.1124,  0.0002, -0.0104, -0.0163, -0.0205, -0.0365,\\n        -0.0691,  0.0682, -0.0239,  0.0198,  0.0705,  0.1005,  0.0704,  0.0365,\\n        -0.0361,  0.0932, -0.0621, -0.1168,  0.0968, -0.0380,  0.1074, -0.0406,\\n         0.1181, -0.0117,  0.0873,  0.0036, -0.1201,  0.0726, -0.0205,  0.0735,\\n         0.1182,  0.0905, -0.0347, -0.0859,  0.0030,  0.1154, -0.0922,  0.0364,\\n        -0.0104,  0.0075, -0.0739,  0.0239,  0.1103, -0.0884,  0.0597,  0.0150],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([[ 0.0119, -0.0910, -0.0359, -0.0299, -0.0802, -0.0171, -0.0461,  0.0998,\\n          0.0136, -0.1118, -0.0996,  0.0528,  0.0598, -0.1069, -0.1150,  0.0086,\\n         -0.0524,  0.0871, -0.0861,  0.1161, -0.0421,  0.0187,  0.0336, -0.1065,\\n         -0.1221,  0.0843,  0.0429, -0.0311,  0.0816, -0.0881,  0.0718,  0.0618,\\n          0.0783, -0.0265, -0.0182, -0.1131, -0.0907,  0.0460, -0.1202, -0.0170,\\n         -0.0782, -0.0421, -0.0980,  0.0364,  0.0589,  0.1225, -0.0884,  0.0413,\\n         -0.0683,  0.0673, -0.0350, -0.0703, -0.0687,  0.0937,  0.0537, -0.0411,\\n          0.0511,  0.0389,  0.0052, -0.1069,  0.0869, -0.0129,  0.0618, -0.1218],\\n        [ 0.0109, -0.1020,  0.0132,  0.0520, -0.0236,  0.0461,  0.0561,  0.0005,\\n         -0.0724,  0.0471,  0.0990,  0.0688,  0.0800, -0.0468, -0.1008, -0.1060,\\n          0.0472,  0.0666, -0.1002,  0.0651,  0.0943,  0.1084, -0.0049, -0.0104,\\n          0.0671,  0.0366, -0.0139,  0.0387,  0.1050, -0.0339,  0.1229, -0.0305,\\n         -0.0515, -0.0809,  0.1056,  0.0919,  0.1249, -0.0960,  0.0352, -0.0445,\\n          0.0529,  0.0162, -0.0857,  0.1205,  0.1249, -0.0210,  0.0957,  0.1077,\\n         -0.1247, -0.1006, -0.0902, -0.0767,  0.1098, -0.0349,  0.0644, -0.0983,\\n          0.0940, -0.0513, -0.0419,  0.0535,  0.0670,  0.0406,  0.0186,  0.0802]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([-0.0315,  0.0003], requires_grad=True)\"\n",
      "                            ],\n",
      "                            \"weight_decay\":\t0\n",
      "                        }\n",
      "                    ],\n",
      "                    \"state\":\t{}\n",
      "                }\n",
      "            },\n",
      "            \"render\":\tfalse,\n",
      "            \"save_freq\":\t10,\n",
      "            \"steps_per_epoch\":\t4000,\n",
      "            \"train_v_iters\":\t80,\n",
      "            \"vf_optimizer\":\t{\n",
      "                \"Adam (\\nParameter Group 0\\n    amsgrad: False\\n    betas: (0.9, 0.999)\\n    eps: 1e-08\\n    lr: 0.001\\n    weight_decay: 0\\n)\":\t{\n",
      "                    \"defaults\":\t{\n",
      "                        \"amsgrad\":\tfalse,\n",
      "                        \"betas\":\t[\n",
      "                            0.9,\n",
      "                            0.999\n",
      "                        ],\n",
      "                        \"eps\":\t1e-08,\n",
      "                        \"lr\":\t0.001,\n",
      "                        \"weight_decay\":\t0\n",
      "                    },\n",
      "                    \"param_groups\":\t[\n",
      "                        {\n",
      "                            \"amsgrad\":\tfalse,\n",
      "                            \"betas\":\t[\n",
      "                                0.9,\n",
      "                                0.999\n",
      "                            ],\n",
      "                            \"eps\":\t1e-08,\n",
      "                            \"lr\":\t0.001,\n",
      "                            \"params\":\t[\n",
      "                                \"Parameter containing:\\ntensor([[-0.3046, -0.1918,  0.0523,  0.3271],\\n        [-0.1415, -0.4875, -0.0637, -0.3759],\\n        [-0.4630, -0.2976, -0.3494, -0.1698],\\n        [-0.2379, -0.2897,  0.1577, -0.3999],\\n        [-0.2167, -0.0742,  0.2768,  0.2149],\\n        [-0.4259, -0.1657, -0.0945, -0.1095],\\n        [ 0.2064,  0.4486, -0.1387, -0.4114],\\n        [-0.0912, -0.0291, -0.3227, -0.1379],\\n        [ 0.1627, -0.3350,  0.4531,  0.2289],\\n        [ 0.1905,  0.0685, -0.1709, -0.2482],\\n        [-0.1533, -0.2316, -0.4476,  0.3658],\\n        [-0.4819,  0.4891,  0.0079,  0.1391],\\n        [-0.4716,  0.1511,  0.0481,  0.1660],\\n        [-0.0415,  0.2473,  0.3640,  0.0599],\\n        [-0.4459, -0.0894, -0.4943, -0.0590],\\n        [ 0.3453,  0.2564, -0.4064,  0.0843],\\n        [-0.4408, -0.4239, -0.3204, -0.3726],\\n        [-0.3426, -0.1770, -0.1594,  0.1186],\\n        [ 0.2905, -0.4371, -0.4643, -0.2343],\\n        [-0.4657, -0.2208, -0.1578,  0.3718],\\n        [ 0.2675, -0.4389,  0.1728, -0.0609],\\n        [ 0.0612, -0.4578, -0.0491,  0.0720],\\n        [-0.1718, -0.4770,  0.1738,  0.4588],\\n        [-0.2767,  0.2978, -0.4757,  0.2273],\\n        [-0.0092,  0.4414, -0.0186, -0.4925],\\n        [-0.4736,  0.1219,  0.3363,  0.0415],\\n        [-0.3901, -0.2408,  0.3166,  0.2090],\\n        [-0.0096, -0.3663, -0.3221,  0.3964],\\n        [-0.3949, -0.2505,  0.0146,  0.3000],\\n        [-0.4308, -0.2378, -0.1647, -0.1867],\\n        [-0.1784, -0.1501,  0.2326, -0.2588],\\n        [-0.0702, -0.0813,  0.0289,  0.0428],\\n        [ 0.0849, -0.0435,  0.1627,  0.3724],\\n        [ 0.1842,  0.4236,  0.1706,  0.0365],\\n        [ 0.4441, -0.0103,  0.0107, -0.2948],\\n        [-0.0369,  0.3381, -0.2350, -0.3527],\\n        [-0.2363, -0.3043,  0.4807,  0.2524],\\n        [-0.1136,  0.3661, -0.1946,  0.4539],\\n        [ 0.2163,  0.4419,  0.1724,  0.4253],\\n        [-0.4434, -0.1263,  0.4149,  0.1217],\\n        [-0.1798,  0.4687, -0.1688,  0.1493],\\n        [ 0.3912,  0.0923,  0.2357,  0.0719],\\n        [-0.2328, -0.1624, -0.2533, -0.1228],\\n        [-0.0172, -0.0785, -0.0834, -0.3193],\\n        [ 0.1099, -0.0978,  0.3939, -0.2596],\\n        [ 0.1560, -0.3776,  0.3001, -0.1504],\\n        [-0.2786, -0.2878, -0.3424, -0.3911],\\n        [ 0.1133,  0.0829,  0.2896, -0.2677],\\n        [-0.1146, -0.3145,  0.0093,  0.3290],\\n        [ 0.2363,  0.0456,  0.2930, -0.3394],\\n        [ 0.2647, -0.0007, -0.2162, -0.1204],\\n        [ 0.4828,  0.1533, -0.2487, -0.3262],\\n        [ 0.0922,  0.0774,  0.0341, -0.2173],\\n        [-0.3901, -0.4961,  0.3223, -0.2756],\\n        [-0.1942, -0.4203,  0.1836, -0.1201],\\n        [-0.3856,  0.2215, -0.4257, -0.4785],\\n        [ 0.0197, -0.4667, -0.1265,  0.4570],\\n        [ 0.0467,  0.2426, -0.4953, -0.4371],\\n        [-0.3449, -0.2379,  0.0808, -0.2834],\\n        [-0.2662, -0.3645,  0.0479, -0.1848],\\n        [-0.1354,  0.1759,  0.0200, -0.4311],\\n        [ 0.0451,  0.1949,  0.2435,  0.3584],\\n        [-0.4296,  0.2431, -0.2296, -0.3621],\\n        [-0.4085,  0.2420,  0.0142,  0.3426]], requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([ 0.2301, -0.1723,  0.2125,  0.0330, -0.0833, -0.0644, -0.2025, -0.2111,\\n         0.3983,  0.1534,  0.2977, -0.0365,  0.3636,  0.2069, -0.3122, -0.4690,\\n        -0.0993, -0.4728,  0.2546,  0.1855,  0.4169, -0.0657, -0.3313, -0.3148,\\n         0.4304,  0.3751,  0.0659,  0.2636, -0.4452, -0.3182,  0.2270,  0.4189,\\n         0.4857,  0.0636, -0.0642,  0.0602, -0.4869,  0.1228, -0.3612, -0.1643,\\n         0.3477,  0.4739,  0.4626, -0.3879,  0.4825,  0.4246,  0.2210,  0.4413,\\n         0.0482, -0.4683, -0.0701,  0.4940,  0.1916, -0.0415,  0.1077,  0.1163,\\n         0.3196,  0.2687,  0.4177, -0.0242, -0.2623, -0.3845,  0.4229, -0.4380],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([[-0.0073,  0.0402, -0.0251,  ..., -0.0138, -0.0635,  0.0215],\\n        [ 0.0494,  0.0854, -0.0403,  ...,  0.0784, -0.1188,  0.0727],\\n        [-0.0558,  0.0055,  0.0922,  ...,  0.0349,  0.0725, -0.1028],\\n        ...,\\n        [ 0.0195, -0.0718, -0.0365,  ...,  0.0369,  0.0174, -0.0408],\\n        [ 0.0390, -0.0564,  0.0868,  ..., -0.0269,  0.1140, -0.0686],\\n        [ 0.0545, -0.1144, -0.0796,  ..., -0.0106,  0.0728,  0.0141]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([-0.0874, -0.0215, -0.0205,  0.0957, -0.1089,  0.1017, -0.0802,  0.1077,\\n        -0.0370, -0.0731, -0.0715,  0.0465,  0.0521,  0.0769,  0.1241, -0.0209,\\n         0.0433, -0.0176,  0.0071, -0.0418, -0.1172, -0.1009,  0.0621, -0.0339,\\n        -0.0633, -0.0716,  0.0639,  0.0775,  0.0496,  0.0224, -0.0961, -0.0269,\\n        -0.1183, -0.0148,  0.0597,  0.0857,  0.0229,  0.0402, -0.0143, -0.1061,\\n        -0.0525,  0.0328,  0.0683,  0.0164,  0.0513,  0.0359,  0.0015, -0.0269,\\n         0.0636, -0.0164, -0.1185, -0.0451, -0.0472, -0.0164, -0.0735,  0.1169,\\n         0.0244,  0.1184,  0.0783,  0.0016,  0.0994, -0.0495,  0.1106, -0.0335],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([[-0.0645, -0.0927,  0.0723, -0.0304,  0.0263, -0.0037, -0.0324, -0.1015,\\n          0.1027,  0.0147, -0.0703,  0.0511,  0.0867, -0.1199, -0.0923, -0.0811,\\n          0.0481,  0.0487, -0.0832,  0.0528, -0.1204,  0.0258,  0.0945, -0.0956,\\n          0.0254, -0.1043,  0.0649,  0.1168, -0.0513,  0.0184, -0.0009, -0.1168,\\n          0.0770,  0.0619,  0.0639,  0.0416, -0.0984,  0.0049,  0.0571,  0.0556,\\n          0.0948,  0.0830,  0.0604,  0.0819, -0.0367,  0.0424, -0.0619,  0.0499,\\n         -0.0676, -0.0704, -0.1057, -0.0082, -0.0049, -0.0838, -0.0141, -0.0631,\\n         -0.0845,  0.0386, -0.0969,  0.0832,  0.0489,  0.0146,  0.0960,  0.0233]],\\n       requires_grad=True)\",\n",
      "                                \"Parameter containing:\\ntensor([0.0775], requires_grad=True)\"\n",
      "                            ],\n",
      "                            \"weight_decay\":\t0\n",
      "                        }\n",
      "                    ],\n",
      "                    \"state\":\t{}\n",
      "                }\n",
      "            }\n",
      "        }\n",
      "    },\n",
      "    \"steps_per_epoch\":\t4000,\n",
      "    \"train_v_iters\":\t80,\n",
      "    \"vf_lr\":\t0.001\n",
      "}\n",
      "\u001b[32;1m\n",
      "Number of parameters: \t pi: 4610, \t v: 4545\n",
      "\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: trajectory cut off by epoch at 3 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               0 |\n",
      "|      AverageEpRet |              21 |\n",
      "|          StdEpRet |             9.8 |\n",
      "|          MaxEpRet |              61 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |              21 |\n",
      "|      AverageVVals |          0.0494 |\n",
      "|          StdVVals |          0.0311 |\n",
      "|          MaxVVals |           0.121 |\n",
      "|          MinVVals |         -0.0333 |\n",
      "| TotalEnvInteracts |           4e+03 |\n",
      "|            LossPi |        1.29e-08 |\n",
      "|             LossV |             214 |\n",
      "|       DeltaLossPi |         -0.0182 |\n",
      "|        DeltaLossV |            -130 |\n",
      "|           Entropy |           0.691 |\n",
      "|                KL |         0.00842 |\n",
      "|              Time |            1.71 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 15 steps.\n",
      "\u001b[32;1mEarly stopping at step 17 due to reaching max kl.\u001b[0m\n",
      "---------------------------------------\n",
      "|             Epoch |               1 |\n",
      "|      AverageEpRet |            28.3 |\n",
      "|          StdEpRet |            16.8 |\n",
      "|          MaxEpRet |             106 |\n",
      "|          MinEpRet |               9 |\n",
      "|             EpLen |            28.3 |\n",
      "|      AverageVVals |               8 |\n",
      "|          StdVVals |           0.473 |\n",
      "|          MaxVVals |             8.5 |\n",
      "|          MinVVals |            4.83 |\n",
      "| TotalEnvInteracts |           8e+03 |\n",
      "|            LossPi |       -1.25e-08 |\n",
      "|             LossV |             225 |\n",
      "|       DeltaLossPi |         -0.0165 |\n",
      "|        DeltaLossV |           -75.9 |\n",
      "|           Entropy |           0.689 |\n",
      "|                KL |          0.0152 |\n",
      "|              Time |            3.22 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |               2 |\n",
      "|      AverageEpRet |            36.7 |\n",
      "|          StdEpRet |            22.7 |\n",
      "|          MaxEpRet |             124 |\n",
      "|          MinEpRet |              10 |\n",
      "|             EpLen |            36.7 |\n",
      "|      AverageVVals |            13.8 |\n",
      "|          StdVVals |            1.83 |\n",
      "|          MaxVVals |            14.7 |\n",
      "|          MinVVals |           -8.64 |\n",
      "| TotalEnvInteracts |         1.2e+04 |\n",
      "|            LossPi |        2.03e-08 |\n",
      "|             LossV |             275 |\n",
      "|       DeltaLossPi |         -0.0151 |\n",
      "|        DeltaLossV |           -55.5 |\n",
      "|           Entropy |           0.671 |\n",
      "|                KL |         0.00521 |\n",
      "|              Time |            4.85 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |               3 |\n",
      "|      AverageEpRet |            48.8 |\n",
      "|          StdEpRet |            33.7 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              13 |\n",
      "|             EpLen |            48.8 |\n",
      "|      AverageVVals |            18.9 |\n",
      "|          StdVVals |            2.01 |\n",
      "|          MaxVVals |            19.7 |\n",
      "|          MinVVals |           -8.36 |\n",
      "| TotalEnvInteracts |         1.6e+04 |\n",
      "|            LossPi |       -3.49e-08 |\n",
      "|             LossV |             435 |\n",
      "|       DeltaLossPi |         -0.0162 |\n",
      "|        DeltaLossV |             -71 |\n",
      "|           Entropy |           0.652 |\n",
      "|                KL |          0.0081 |\n",
      "|              Time |            6.46 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 20 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               4 |\n",
      "|      AverageEpRet |              78 |\n",
      "|          StdEpRet |            54.2 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              14 |\n",
      "|             EpLen |              78 |\n",
      "|      AverageVVals |            24.3 |\n",
      "|          StdVVals |            3.25 |\n",
      "|          MaxVVals |            25.2 |\n",
      "|          MinVVals |           -17.1 |\n",
      "| TotalEnvInteracts |           2e+04 |\n",
      "|            LossPi |       -1.73e-08 |\n",
      "|             LossV |             748 |\n",
      "|       DeltaLossPi |         -0.0112 |\n",
      "|        DeltaLossV |            -174 |\n",
      "|           Entropy |           0.639 |\n",
      "|                KL |         0.00274 |\n",
      "|              Time |            8.14 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 137 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               5 |\n",
      "|      AverageEpRet |             110 |\n",
      "|          StdEpRet |              58 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              13 |\n",
      "|             EpLen |             110 |\n",
      "|      AverageVVals |            32.3 |\n",
      "|          StdVVals |            2.57 |\n",
      "|          MaxVVals |            33.1 |\n",
      "|          MinVVals |           -3.83 |\n",
      "| TotalEnvInteracts |         2.4e+04 |\n",
      "|            LossPi |       -2.54e-08 |\n",
      "|             LossV |             723 |\n",
      "|       DeltaLossPi |        -0.00906 |\n",
      "|        DeltaLossV |            -159 |\n",
      "|           Entropy |           0.626 |\n",
      "|                KL |         0.00347 |\n",
      "|              Time |            9.82 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 13 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               6 |\n",
      "|      AverageEpRet |             142 |\n",
      "|          StdEpRet |            45.9 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              24 |\n",
      "|             EpLen |             142 |\n",
      "|      AverageVVals |            38.1 |\n",
      "|          StdVVals |             5.8 |\n",
      "|          MaxVVals |            40.2 |\n",
      "|          MinVVals |           -14.3 |\n",
      "| TotalEnvInteracts |         2.8e+04 |\n",
      "|            LossPi |       -9.54e-10 |\n",
      "|             LossV |             592 |\n",
      "|       DeltaLossPi |        -0.00805 |\n",
      "|        DeltaLossV |            -173 |\n",
      "|           Entropy |           0.607 |\n",
      "|                KL |         0.00549 |\n",
      "|              Time |            11.5 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 20 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               7 |\n",
      "|      AverageEpRet |             181 |\n",
      "|          StdEpRet |            29.2 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             102 |\n",
      "|             EpLen |             181 |\n",
      "|      AverageVVals |            41.9 |\n",
      "|          StdVVals |            9.36 |\n",
      "|          MaxVVals |            46.4 |\n",
      "|          MinVVals |            6.22 |\n",
      "| TotalEnvInteracts |         3.2e+04 |\n",
      "|            LossPi |          -2e-08 |\n",
      "|             LossV |             574 |\n",
      "|       DeltaLossPi |        -0.00533 |\n",
      "|        DeltaLossV |            -129 |\n",
      "|           Entropy |           0.609 |\n",
      "|                KL |         0.00446 |\n",
      "|              Time |            13.2 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 100 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               8 |\n",
      "|      AverageEpRet |             170 |\n",
      "|          StdEpRet |            50.3 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              42 |\n",
      "|             EpLen |             170 |\n",
      "|      AverageVVals |            49.1 |\n",
      "|          StdVVals |            6.89 |\n",
      "|          MaxVVals |            52.3 |\n",
      "|          MinVVals |              10 |\n",
      "| TotalEnvInteracts |         3.6e+04 |\n",
      "|            LossPi |        2.03e-08 |\n",
      "|             LossV |             535 |\n",
      "|       DeltaLossPi |        -0.00335 |\n",
      "|        DeltaLossV |           -49.2 |\n",
      "|           Entropy |           0.597 |\n",
      "|                KL |         0.00328 |\n",
      "|              Time |            14.9 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 88 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |               9 |\n",
      "|      AverageEpRet |             196 |\n",
      "|          StdEpRet |            11.1 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             163 |\n",
      "|             EpLen |             196 |\n",
      "|      AverageVVals |            50.2 |\n",
      "|          StdVVals |            9.89 |\n",
      "|          MaxVVals |            55.6 |\n",
      "|          MinVVals |               7 |\n",
      "| TotalEnvInteracts |           4e+04 |\n",
      "|            LossPi |        9.54e-10 |\n",
      "|             LossV |             505 |\n",
      "|       DeltaLossPi |        -0.00141 |\n",
      "|        DeltaLossV |           -69.1 |\n",
      "|           Entropy |           0.593 |\n",
      "|                KL |         0.00453 |\n",
      "|              Time |            16.6 |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: trajectory cut off by epoch at 5 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              10 |\n",
      "|      AverageEpRet |             174 |\n",
      "|          StdEpRet |            46.5 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              39 |\n",
      "|             EpLen |             174 |\n",
      "|      AverageVVals |            49.8 |\n",
      "|          StdVVals |            11.4 |\n",
      "|          MaxVVals |            58.9 |\n",
      "|          MinVVals |            5.76 |\n",
      "| TotalEnvInteracts |         4.4e+04 |\n",
      "|            LossPi |       -2.86e-09 |\n",
      "|             LossV |             422 |\n",
      "|       DeltaLossPi |        -0.00386 |\n",
      "|        DeltaLossV |           -56.7 |\n",
      "|           Entropy |           0.597 |\n",
      "|                KL |         0.00287 |\n",
      "|              Time |            18.2 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 120 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              11 |\n",
      "|      AverageEpRet |             194 |\n",
      "|          StdEpRet |            17.1 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             126 |\n",
      "|             EpLen |             194 |\n",
      "|      AverageVVals |            54.8 |\n",
      "|          StdVVals |            10.8 |\n",
      "|          MaxVVals |            61.4 |\n",
      "|          MinVVals |            4.03 |\n",
      "| TotalEnvInteracts |         4.8e+04 |\n",
      "|            LossPi |               0 |\n",
      "|             LossV |             383 |\n",
      "|       DeltaLossPi |        -0.00357 |\n",
      "|        DeltaLossV |           -59.1 |\n",
      "|           Entropy |           0.583 |\n",
      "|                KL |         0.00716 |\n",
      "|              Time |            19.9 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              12 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            55.3 |\n",
      "|          StdVVals |            12.1 |\n",
      "|          MaxVVals |            63.8 |\n",
      "|          MinVVals |              10 |\n",
      "| TotalEnvInteracts |         5.2e+04 |\n",
      "|            LossPi |       -8.58e-09 |\n",
      "|             LossV |             389 |\n",
      "|       DeltaLossPi |        -0.00314 |\n",
      "|        DeltaLossV |           -30.6 |\n",
      "|           Entropy |            0.58 |\n",
      "|                KL |         0.00455 |\n",
      "|              Time |            21.6 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              13 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            59.7 |\n",
      "|          StdVVals |            8.16 |\n",
      "|          MaxVVals |            66.1 |\n",
      "|          MinVVals |            25.2 |\n",
      "| TotalEnvInteracts |         5.6e+04 |\n",
      "|            LossPi |        5.72e-09 |\n",
      "|             LossV |             445 |\n",
      "|       DeltaLossPi |        -0.00229 |\n",
      "|        DeltaLossV |           -55.7 |\n",
      "|           Entropy |           0.576 |\n",
      "|                KL |        0.000896 |\n",
      "|              Time |            23.3 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 121 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              14 |\n",
      "|      AverageEpRet |             194 |\n",
      "|          StdEpRet |            26.4 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              79 |\n",
      "|             EpLen |             194 |\n",
      "|      AverageVVals |            55.8 |\n",
      "|          StdVVals |            12.9 |\n",
      "|          MaxVVals |            67.6 |\n",
      "|          MinVVals |            21.1 |\n",
      "| TotalEnvInteracts |           6e+04 |\n",
      "|            LossPi |       -8.11e-09 |\n",
      "|             LossV |             385 |\n",
      "|       DeltaLossPi |        -0.00372 |\n",
      "|        DeltaLossV |           -42.9 |\n",
      "|           Entropy |           0.585 |\n",
      "|                KL |         0.00542 |\n",
      "|              Time |              25 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 7 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              15 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |            1.53 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             193 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            61.6 |\n",
      "|          StdVVals |            8.15 |\n",
      "|          MaxVVals |            69.2 |\n",
      "|          MinVVals |            35.5 |\n",
      "| TotalEnvInteracts |         6.4e+04 |\n",
      "|            LossPi |       -2.86e-09 |\n",
      "|             LossV |             524 |\n",
      "|       DeltaLossPi |        -0.00354 |\n",
      "|        DeltaLossV |           -70.3 |\n",
      "|           Entropy |           0.581 |\n",
      "|                KL |         0.00151 |\n",
      "|              Time |            26.9 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 19 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              16 |\n",
      "|      AverageEpRet |             181 |\n",
      "|          StdEpRet |            38.9 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              76 |\n",
      "|             EpLen |             181 |\n",
      "|      AverageVVals |            57.9 |\n",
      "|          StdVVals |            7.99 |\n",
      "|          MaxVVals |            68.6 |\n",
      "|          MinVVals |            36.7 |\n",
      "| TotalEnvInteracts |         6.8e+04 |\n",
      "|            LossPi |       -1.25e-08 |\n",
      "|             LossV |             548 |\n",
      "|       DeltaLossPi |        -0.00239 |\n",
      "|        DeltaLossV |           -62.9 |\n",
      "|           Entropy |           0.582 |\n",
      "|                KL |         0.00265 |\n",
      "|              Time |            28.6 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 124 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              17 |\n",
      "|      AverageEpRet |             194 |\n",
      "|          StdEpRet |            23.2 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |              94 |\n",
      "|             EpLen |             194 |\n",
      "|      AverageVVals |            55.7 |\n",
      "|          StdVVals |            8.79 |\n",
      "|          MaxVVals |            68.1 |\n",
      "|          MinVVals |              27 |\n",
      "| TotalEnvInteracts |         7.2e+04 |\n",
      "|            LossPi |        7.39e-09 |\n",
      "|             LossV |             447 |\n",
      "|       DeltaLossPi |        -0.00257 |\n",
      "|        DeltaLossV |           -30.5 |\n",
      "|           Entropy |           0.573 |\n",
      "|                KL |         0.00162 |\n",
      "|              Time |            30.6 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              18 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            57.3 |\n",
      "|          StdVVals |            9.53 |\n",
      "|          MaxVVals |            69.1 |\n",
      "|          MinVVals |              33 |\n",
      "| TotalEnvInteracts |         7.6e+04 |\n",
      "|            LossPi |       -3.81e-09 |\n",
      "|             LossV |             466 |\n",
      "|       DeltaLossPi |        -0.00111 |\n",
      "|        DeltaLossV |           -16.8 |\n",
      "|           Entropy |           0.565 |\n",
      "|                KL |         0.00145 |\n",
      "|              Time |            32.3 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 8 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              19 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |            1.74 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             192 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            56.6 |\n",
      "|          StdVVals |            9.48 |\n",
      "|          MaxVVals |              70 |\n",
      "|          MinVVals |            26.5 |\n",
      "| TotalEnvInteracts |           8e+04 |\n",
      "|            LossPi |               0 |\n",
      "|             LossV |             421 |\n",
      "|       DeltaLossPi |        -0.00163 |\n",
      "|        DeltaLossV |           -28.2 |\n",
      "|           Entropy |           0.556 |\n",
      "|                KL |         0.00557 |\n",
      "|              Time |              34 |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: trajectory cut off by epoch at 24 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              20 |\n",
      "|      AverageEpRet |             199 |\n",
      "|          StdEpRet |            5.23 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             176 |\n",
      "|             EpLen |             199 |\n",
      "|      AverageVVals |            64.2 |\n",
      "|          StdVVals |            7.73 |\n",
      "|          MaxVVals |            72.1 |\n",
      "|          MinVVals |            40.8 |\n",
      "| TotalEnvInteracts |         8.4e+04 |\n",
      "|            LossPi |        1.91e-09 |\n",
      "|             LossV |             518 |\n",
      "|       DeltaLossPi |        -0.00221 |\n",
      "|        DeltaLossV |            -104 |\n",
      "|           Entropy |           0.557 |\n",
      "|                KL |        0.000911 |\n",
      "|              Time |            35.7 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              21 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            58.1 |\n",
      "|          StdVVals |            8.67 |\n",
      "|          MaxVVals |            69.7 |\n",
      "|          MinVVals |              35 |\n",
      "| TotalEnvInteracts |         8.8e+04 |\n",
      "|            LossPi |       -7.63e-09 |\n",
      "|             LossV |             617 |\n",
      "|       DeltaLossPi |        -0.00218 |\n",
      "|        DeltaLossV |            -114 |\n",
      "|           Entropy |           0.566 |\n",
      "|                KL |         0.00221 |\n",
      "|              Time |            37.4 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 59 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              22 |\n",
      "|      AverageEpRet |             197 |\n",
      "|          StdEpRet |            8.96 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             166 |\n",
      "|             EpLen |             197 |\n",
      "|      AverageVVals |            59.7 |\n",
      "|          StdVVals |            8.44 |\n",
      "|          MaxVVals |            71.7 |\n",
      "|          MinVVals |            27.5 |\n",
      "| TotalEnvInteracts |         9.2e+04 |\n",
      "|            LossPi |        1.14e-08 |\n",
      "|             LossV |             534 |\n",
      "|       DeltaLossPi |          -0.003 |\n",
      "|        DeltaLossV |           -31.4 |\n",
      "|           Entropy |           0.562 |\n",
      "|                KL |        0.000911 |\n",
      "|              Time |            39.1 |\n",
      "---------------------------------------\n",
      "Warning: trajectory cut off by epoch at 3 steps.\n",
      "---------------------------------------\n",
      "|             Epoch |              23 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |           0.654 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             197 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            56.8 |\n",
      "|          StdVVals |            7.37 |\n",
      "|          MaxVVals |            66.3 |\n",
      "|          MinVVals |            31.2 |\n",
      "| TotalEnvInteracts |         9.6e+04 |\n",
      "|            LossPi |        9.54e-10 |\n",
      "|             LossV |             526 |\n",
      "|       DeltaLossPi |        -0.00294 |\n",
      "|        DeltaLossV |           -53.9 |\n",
      "|           Entropy |           0.547 |\n",
      "|                KL |         0.00311 |\n",
      "|              Time |            40.8 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              24 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            57.4 |\n",
      "|          StdVVals |            7.84 |\n",
      "|          MaxVVals |            66.9 |\n",
      "|          MinVVals |              30 |\n",
      "| TotalEnvInteracts |           1e+05 |\n",
      "|            LossPi |        4.77e-09 |\n",
      "|             LossV |             508 |\n",
      "|       DeltaLossPi |        -0.00206 |\n",
      "|        DeltaLossV |           -24.8 |\n",
      "|           Entropy |           0.541 |\n",
      "|                KL |         0.00191 |\n",
      "|              Time |            42.6 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              25 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            58.5 |\n",
      "|          StdVVals |            8.01 |\n",
      "|          MaxVVals |            68.2 |\n",
      "|          MinVVals |            33.6 |\n",
      "| TotalEnvInteracts |        1.04e+05 |\n",
      "|            LossPi |        6.68e-09 |\n",
      "|             LossV |             570 |\n",
      "|       DeltaLossPi |        -0.00305 |\n",
      "|        DeltaLossV |           -49.6 |\n",
      "|           Entropy |           0.528 |\n",
      "|                KL |         0.00678 |\n",
      "|              Time |            44.3 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              26 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            54.6 |\n",
      "|          StdVVals |            8.49 |\n",
      "|          MaxVVals |            70.1 |\n",
      "|          MinVVals |              33 |\n",
      "| TotalEnvInteracts |        1.08e+05 |\n",
      "|            LossPi |        1.14e-08 |\n",
      "|             LossV |             456 |\n",
      "|       DeltaLossPi |        -0.00274 |\n",
      "|        DeltaLossV |           -58.1 |\n",
      "|           Entropy |           0.536 |\n",
      "|                KL |         0.00232 |\n",
      "|              Time |            46.1 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              27 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            63.3 |\n",
      "|          StdVVals |            10.2 |\n",
      "|          MaxVVals |            73.4 |\n",
      "|          MinVVals |            28.4 |\n",
      "| TotalEnvInteracts |        1.12e+05 |\n",
      "|            LossPi |       -2.86e-09 |\n",
      "|             LossV |             510 |\n",
      "|       DeltaLossPi |         -0.0019 |\n",
      "|        DeltaLossV |           -36.2 |\n",
      "|           Entropy |           0.533 |\n",
      "|                KL |         0.00265 |\n",
      "|              Time |            47.8 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              28 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            60.8 |\n",
      "|          StdVVals |            6.55 |\n",
      "|          MaxVVals |            71.2 |\n",
      "|          MinVVals |            40.3 |\n",
      "| TotalEnvInteracts |        1.16e+05 |\n",
      "|            LossPi |       -5.72e-09 |\n",
      "|             LossV |             551 |\n",
      "|       DeltaLossPi |         -0.0046 |\n",
      "|        DeltaLossV |             -37 |\n",
      "|           Entropy |           0.529 |\n",
      "|                KL |         0.00569 |\n",
      "|              Time |            49.5 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              29 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            55.9 |\n",
      "|          StdVVals |            11.1 |\n",
      "|          MaxVVals |            70.3 |\n",
      "|          MinVVals |            25.4 |\n",
      "| TotalEnvInteracts |         1.2e+05 |\n",
      "|            LossPi |        2.86e-09 |\n",
      "|             LossV |             524 |\n",
      "|       DeltaLossPi |         -0.0023 |\n",
      "|        DeltaLossV |           -67.1 |\n",
      "|           Entropy |           0.526 |\n",
      "|                KL |        0.000205 |\n",
      "|              Time |            51.3 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              30 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            61.1 |\n",
      "|          StdVVals |            8.34 |\n",
      "|          MaxVVals |              71 |\n",
      "|          MinVVals |            33.2 |\n",
      "| TotalEnvInteracts |        1.24e+05 |\n",
      "|            LossPi |        2.86e-09 |\n",
      "|             LossV |             527 |\n",
      "|       DeltaLossPi |        -0.00206 |\n",
      "|        DeltaLossV |           -32.5 |\n",
      "|           Entropy |           0.523 |\n",
      "|                KL |         0.00395 |\n",
      "|              Time |              53 |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------\n",
      "|             Epoch |              31 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            59.4 |\n",
      "|          StdVVals |            6.87 |\n",
      "|          MaxVVals |            70.5 |\n",
      "|          MinVVals |            42.3 |\n",
      "| TotalEnvInteracts |        1.28e+05 |\n",
      "|            LossPi |       -1.34e-08 |\n",
      "|             LossV |             583 |\n",
      "|       DeltaLossPi |        -0.00468 |\n",
      "|        DeltaLossV |           -67.1 |\n",
      "|           Entropy |           0.517 |\n",
      "|                KL |         0.00482 |\n",
      "|              Time |            54.7 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              32 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            58.3 |\n",
      "|          StdVVals |            7.67 |\n",
      "|          MaxVVals |            68.8 |\n",
      "|          MinVVals |            29.6 |\n",
      "| TotalEnvInteracts |        1.32e+05 |\n",
      "|            LossPi |       -6.68e-09 |\n",
      "|             LossV |             522 |\n",
      "|       DeltaLossPi |        -0.00103 |\n",
      "|        DeltaLossV |           -56.9 |\n",
      "|           Entropy |           0.519 |\n",
      "|                KL |         0.00142 |\n",
      "|              Time |            56.4 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              33 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            52.7 |\n",
      "|          StdVVals |            13.9 |\n",
      "|          MaxVVals |              71 |\n",
      "|          MinVVals |              19 |\n",
      "| TotalEnvInteracts |        1.36e+05 |\n",
      "|            LossPi |       -3.62e-08 |\n",
      "|             LossV |             398 |\n",
      "|       DeltaLossPi |        -0.00191 |\n",
      "|        DeltaLossV |           -42.3 |\n",
      "|           Entropy |           0.515 |\n",
      "|                KL |         0.00361 |\n",
      "|              Time |            58.2 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              34 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            65.3 |\n",
      "|          StdVVals |            8.07 |\n",
      "|          MaxVVals |            72.9 |\n",
      "|          MinVVals |            31.9 |\n",
      "| TotalEnvInteracts |         1.4e+05 |\n",
      "|            LossPi |        7.63e-09 |\n",
      "|             LossV |             556 |\n",
      "|       DeltaLossPi |        -0.00136 |\n",
      "|        DeltaLossV |           -78.9 |\n",
      "|           Entropy |           0.524 |\n",
      "|                KL |         0.00117 |\n",
      "|              Time |            59.9 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              35 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            59.7 |\n",
      "|          StdVVals |            6.47 |\n",
      "|          MaxVVals |            68.3 |\n",
      "|          MinVVals |            18.1 |\n",
      "| TotalEnvInteracts |        1.44e+05 |\n",
      "|            LossPi |       -3.81e-09 |\n",
      "|             LossV |             585 |\n",
      "|       DeltaLossPi |       -0.000822 |\n",
      "|        DeltaLossV |             -51 |\n",
      "|           Entropy |           0.509 |\n",
      "|                KL |         0.00423 |\n",
      "|              Time |            61.6 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              36 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |              59 |\n",
      "|          StdVVals |             6.5 |\n",
      "|          MaxVVals |            67.9 |\n",
      "|          MinVVals |            27.1 |\n",
      "| TotalEnvInteracts |        1.48e+05 |\n",
      "|            LossPi |        5.72e-09 |\n",
      "|             LossV |             577 |\n",
      "|       DeltaLossPi |        -0.00248 |\n",
      "|        DeltaLossV |           -82.5 |\n",
      "|           Entropy |            0.51 |\n",
      "|                KL |         0.00426 |\n",
      "|              Time |            63.2 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              37 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |              53 |\n",
      "|          StdVVals |             9.9 |\n",
      "|          MaxVVals |            69.8 |\n",
      "|          MinVVals |            29.6 |\n",
      "| TotalEnvInteracts |        1.52e+05 |\n",
      "|            LossPi |       -2.86e-08 |\n",
      "|             LossV |             456 |\n",
      "|       DeltaLossPi |        -0.00236 |\n",
      "|        DeltaLossV |           -54.9 |\n",
      "|           Entropy |           0.519 |\n",
      "|                KL |         0.00249 |\n",
      "|              Time |            64.9 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              38 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            64.5 |\n",
      "|          StdVVals |            10.9 |\n",
      "|          MaxVVals |            74.1 |\n",
      "|          MinVVals |            30.2 |\n",
      "| TotalEnvInteracts |        1.56e+05 |\n",
      "|            LossPi |        6.68e-09 |\n",
      "|             LossV |             524 |\n",
      "|       DeltaLossPi |        -0.00177 |\n",
      "|        DeltaLossV |           -40.3 |\n",
      "|           Entropy |           0.526 |\n",
      "|                KL |         0.00348 |\n",
      "|              Time |            66.7 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              39 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            53.5 |\n",
      "|          StdVVals |            11.6 |\n",
      "|          MaxVVals |            70.2 |\n",
      "|          MinVVals |            27.3 |\n",
      "| TotalEnvInteracts |         1.6e+05 |\n",
      "|            LossPi |       -2.86e-09 |\n",
      "|             LossV |             410 |\n",
      "|       DeltaLossPi |        -0.00271 |\n",
      "|        DeltaLossV |           -37.3 |\n",
      "|           Entropy |            0.53 |\n",
      "|                KL |         0.00603 |\n",
      "|              Time |            68.4 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              40 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            65.4 |\n",
      "|          StdVVals |            9.05 |\n",
      "|          MaxVVals |            75.1 |\n",
      "|          MinVVals |            31.5 |\n",
      "| TotalEnvInteracts |        1.64e+05 |\n",
      "|            LossPi |       -4.77e-09 |\n",
      "|             LossV |             601 |\n",
      "|       DeltaLossPi |        -0.00262 |\n",
      "|        DeltaLossV |           -93.5 |\n",
      "|           Entropy |           0.527 |\n",
      "|                KL |         0.00229 |\n",
      "|              Time |              70 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              41 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            59.2 |\n",
      "|          StdVVals |            5.23 |\n",
      "|          MaxVVals |            65.8 |\n",
      "|          MinVVals |            39.2 |\n",
      "| TotalEnvInteracts |        1.68e+05 |\n",
      "|            LossPi |        9.54e-09 |\n",
      "|             LossV |             520 |\n",
      "|       DeltaLossPi |        -0.00232 |\n",
      "|        DeltaLossV |           -33.3 |\n",
      "|           Entropy |           0.522 |\n",
      "|                KL |         0.00426 |\n",
      "|              Time |            71.7 |\n",
      "---------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------------------------------\n",
      "|             Epoch |              42 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            50.9 |\n",
      "|          StdVVals |            16.4 |\n",
      "|          MaxVVals |            67.9 |\n",
      "|          MinVVals |            9.15 |\n",
      "| TotalEnvInteracts |        1.72e+05 |\n",
      "|            LossPi |        3.81e-09 |\n",
      "|             LossV |             526 |\n",
      "|       DeltaLossPi |       -0.000605 |\n",
      "|        DeltaLossV |            -113 |\n",
      "|           Entropy |           0.525 |\n",
      "|                KL |         0.00574 |\n",
      "|              Time |            73.4 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              43 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            56.8 |\n",
      "|          StdVVals |            14.8 |\n",
      "|          MaxVVals |            68.1 |\n",
      "|          MinVVals |              13 |\n",
      "| TotalEnvInteracts |        1.76e+05 |\n",
      "|            LossPi |       -9.54e-09 |\n",
      "|             LossV |             461 |\n",
      "|       DeltaLossPi |        -0.00191 |\n",
      "|        DeltaLossV |           -67.1 |\n",
      "|           Entropy |           0.511 |\n",
      "|                KL |         0.00358 |\n",
      "|              Time |            75.5 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              44 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            55.7 |\n",
      "|          StdVVals |            11.8 |\n",
      "|          MaxVVals |            68.8 |\n",
      "|          MinVVals |            28.9 |\n",
      "| TotalEnvInteracts |         1.8e+05 |\n",
      "|            LossPi |        2.19e-08 |\n",
      "|             LossV |             404 |\n",
      "|       DeltaLossPi |        -0.00217 |\n",
      "|        DeltaLossV |           -58.7 |\n",
      "|           Entropy |            0.51 |\n",
      "|                KL |         0.00236 |\n",
      "|              Time |            77.2 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              45 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            61.3 |\n",
      "|          StdVVals |            10.9 |\n",
      "|          MaxVVals |            71.9 |\n",
      "|          MinVVals |            26.4 |\n",
      "| TotalEnvInteracts |        1.84e+05 |\n",
      "|            LossPi |       -1.34e-08 |\n",
      "|             LossV |             379 |\n",
      "|       DeltaLossPi |       -0.000689 |\n",
      "|        DeltaLossV |           -58.4 |\n",
      "|           Entropy |           0.495 |\n",
      "|                KL |         0.00061 |\n",
      "|              Time |              79 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              46 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            54.7 |\n",
      "|          StdVVals |            15.9 |\n",
      "|          MaxVVals |            73.3 |\n",
      "|          MinVVals |            12.7 |\n",
      "| TotalEnvInteracts |        1.88e+05 |\n",
      "|            LossPi |        7.63e-09 |\n",
      "|             LossV |             300 |\n",
      "|       DeltaLossPi |        -0.00361 |\n",
      "|        DeltaLossV |           -41.9 |\n",
      "|           Entropy |           0.492 |\n",
      "|                KL |         0.00641 |\n",
      "|              Time |            80.8 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              47 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            63.7 |\n",
      "|          StdVVals |            14.9 |\n",
      "|          MaxVVals |            77.1 |\n",
      "|          MinVVals |            19.8 |\n",
      "| TotalEnvInteracts |        1.92e+05 |\n",
      "|            LossPi |       -1.91e-09 |\n",
      "|             LossV |             503 |\n",
      "|       DeltaLossPi |        -0.00175 |\n",
      "|        DeltaLossV |           -82.9 |\n",
      "|           Entropy |           0.494 |\n",
      "|                KL |         0.00567 |\n",
      "|              Time |            82.5 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              48 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            60.8 |\n",
      "|          StdVVals |            10.4 |\n",
      "|          MaxVVals |            75.3 |\n",
      "|          MinVVals |            39.6 |\n",
      "| TotalEnvInteracts |        1.96e+05 |\n",
      "|            LossPi |        1.14e-08 |\n",
      "|             LossV |             582 |\n",
      "|       DeltaLossPi |         -0.0021 |\n",
      "|        DeltaLossV |           -96.7 |\n",
      "|           Entropy |           0.492 |\n",
      "|                KL |         0.00436 |\n",
      "|              Time |            84.3 |\n",
      "---------------------------------------\n",
      "---------------------------------------\n",
      "|             Epoch |              49 |\n",
      "|      AverageEpRet |             200 |\n",
      "|          StdEpRet |               0 |\n",
      "|          MaxEpRet |             200 |\n",
      "|          MinEpRet |             200 |\n",
      "|             EpLen |             200 |\n",
      "|      AverageVVals |            59.1 |\n",
      "|          StdVVals |            5.32 |\n",
      "|          MaxVVals |            66.6 |\n",
      "|          MinVVals |            42.3 |\n",
      "| TotalEnvInteracts |           2e+05 |\n",
      "|            LossPi |        1.91e-09 |\n",
      "|             LossV |             507 |\n",
      "|       DeltaLossPi |        -0.00165 |\n",
      "|        DeltaLossV |           -44.9 |\n",
      "|           Entropy |           0.502 |\n",
      "|                KL |         0.00317 |\n",
      "|              Time |              86 |\n",
      "---------------------------------------\n"
     ]
    }
   ],
   "source": [
    "agent=PPO_Agent(env)\n",
    "agent.train()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "PPO_Tutorial.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}