Successive Halving Visualization.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "SH Render.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "WcgcJCCQZLNw"
      },
      "source": [
        "#Please run the first 3 cells to have all things to work\r\n",
        "#The fourth is as substitution you won't need it for now, but you may \r\n",
        "#need if you want to delete log of tensorboard\r\n",
        "\r\n",
        "#Then the rest is the same just changing the name of environment\r\n",
        "\r\n",
        "\r\n",
        "!pip install stable-baselines\r\n",
        "#!pip install --upgrade pip\r\n",
        "!pip install gym[box2d]\r\n",
        "!pip install -c conda-forge pyglet\r\n",
        "#remove \" > /dev/null 2>&1\" to see what is going on under the hood\r\n",
        "!apt-get install swig cmake libopenmpi-dev zlib1g-dev xvfb x11-utils ffmpeg -qq #remove -qq for full output\r\n",
        "!pip install stable-baselines[mpi] box2d box2d-kengz pyvirtualdisplay pyglet==1.3.1 --quiet #remove --quiet for full output \r\n",
        "# Stable Baselines only supports tensorflow 1.x for now\r\n",
        "#%tensorflow_version 1.x\r\n",
        "\r\n",
        "import imageio\r\n",
        "import base64\r\n",
        "import IPython\r\n",
        "import PIL.Image\r\n",
        "import pyvirtualdisplay\r\n",
        "#Video stuff \r\n",
        "from pathlib import Path\r\n",
        "from IPython import display as ipythondisplay"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jCUNeA5xPoPi"
      },
      "source": [
        "#!sudo apt-get update && sudo apt-get install cmake libopenmpi-dev zlib1g-dev\r\n",
        "!pip install stable-baselines[mpi]\r\n",
        "\r\n",
        "%tensorflow_version 1.x\r\n",
        "#import tensorflow as tf\r\n",
        "\r\n",
        "#!apt-get install ffmpeg freeglut3-dev xvfb  # For visualization\r\n",
        "!pip install stable-baselines[mpi]==2.10.0\r\n",
        "\r\n",
        "#!pip install gym\r\n",
        "#!pip install gym[atari]\r\n",
        "\r\n",
        "#import os\r\n",
        "#os.kill(os.getpid(), 9)\r\n",
        "\r\n",
        "import stable_baselines\r\n",
        "stable_baselines.__version__\r\n",
        "\r\n",
        "import gym\r\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CxmREapgY-wF"
      },
      "source": [
        "#Load the TensorBoard notebook extension\r\n",
        "%load_ext tensorboard\r\n",
        "import tensorflow as tf\r\n",
        "import datetime, os"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ZrqiqQM5qfu"
      },
      "source": [
        "# Rendering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NJgm3VD85pcy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c270e064-afd3-4e1d-b789-adb01db56aba"
      },
      "source": [
        "# Stable Baselines only supports tensorflow 1.x for now\r\n",
        "%tensorflow_version 1.x\r\n",
        "!apt-get update\r\n",
        "!apt-get install swig cmake libopenmpi-dev zlib1g-dev ffmpeg freeglut3-dev xvfb\r\n",
        "##!pip install stable-baselines[mpi] --upgrade\r\n",
        "!pip install stable-baselines\r\n",
        "!pip install pybullet\r\n",
        "!pip install box2d box2d-kengz pyyaml pytablewriter optuna scikit-optimize"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\r0% [Working]\r            \rHit:1 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n",
            "\r0% [Waiting for headers] [Connecting to security.ubuntu.com (91.189.88.142)] [C\r                                                                               \rHit:2 http://archive.ubuntu.com/ubuntu bionic InRelease\n",
            "\r0% [Waiting for headers] [Waiting for headers] [Waiting for headers] [Waiting f\r0% [1 InRelease gpgv 15.9 kB] [Waiting for headers] [Waiting for headers] [Wait\r                                                                               \rGet:3 http://security.ubuntu.com/ubuntu bionic-security InRelease [88.7 kB]\n",
            "\r0% [1 InRelease gpgv 15.9 kB] [Waiting for headers] [3 InRelease 14.2 kB/88.7 k\r                                                                               \rGet:4 http://archive.ubuntu.com/ubuntu bionic-updates InRelease [88.7 kB]\n",
            "\r0% [1 InRelease gpgv 15.9 kB] [4 InRelease 14.2 kB/88.7 kB 16%] [3 InRelease 14\r                                                                               \rHit:5 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n",
            "\r0% [1 InRelease gpgv 15.9 kB] [4 InRelease 15.6 kB/88.7 kB 18%] [3 InRelease 14\r                                                                               \rHit:6 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n",
            "Get:7 http://archive.ubuntu.com/ubuntu bionic-backports InRelease [74.6 kB]\n",
            "Ign:8 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  InRelease\n",
            "Ign:9 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  InRelease\n",
            "Hit:10 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64  Release\n",
            "Hit:11 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64  Release\n",
            "Fetched 252 kB in 2s (156 kB/s)\n",
            "Reading package lists... Done\n",
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "zlib1g-dev is already the newest version (1:1.2.11.dfsg-0ubuntu2).\n",
            "freeglut3-dev is already the newest version (2.8.1-3).\n",
            "libopenmpi-dev is already the newest version (2.1.1-8).\n",
            "swig is already the newest version (3.0.12-1).\n",
            "cmake is already the newest version (3.10.2-1ubuntu2.18.04.1).\n",
            "ffmpeg is already the newest version (7:3.4.8-0ubuntu0.2).\n",
            "xvfb is already the newest version (2:1.19.6-1ubuntu4.8).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 36 not upgraded.\n",
            "Requirement already satisfied: stable-baselines in /usr/local/lib/python3.6/dist-packages (2.10.0)\n",
            "Requirement already satisfied: gym[atari,classic_control]>=0.11 in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (0.17.3)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (1.18.5)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (1.1.5)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (3.2.2)\n",
            "Requirement already satisfied: cloudpickle>=0.5.5 in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (1.3.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (0.17.0)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (4.1.2.30)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from stable-baselines) (1.4.1)\n",
            "Requirement already satisfied: pyglet<=1.5.0,>=1.4.0 in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines) (1.5.0)\n",
            "Requirement already satisfied: Pillow; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines) (7.0.0)\n",
            "Requirement already satisfied: atari-py~=0.2.0; extra == \"atari\" in /usr/local/lib/python3.6/dist-packages (from gym[atari,classic_control]>=0.11->stable-baselines) (0.2.6)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->stable-baselines) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas->stable-baselines) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib->stable-baselines) (1.3.1)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from pyglet<=1.5.0,>=1.4.0->gym[atari,classic_control]>=0.11->stable-baselines) (0.16.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from atari-py~=0.2.0; extra == \"atari\"->gym[atari,classic_control]>=0.11->stable-baselines) (1.15.0)\n",
            "Requirement already satisfied: pybullet in /usr/local/lib/python3.6/dist-packages (3.0.7)\n",
            "Requirement already satisfied: box2d in /usr/local/lib/python3.6/dist-packages (2.3.10)\n",
            "Requirement already satisfied: box2d-kengz in /usr/local/lib/python3.6/dist-packages (2.3.3)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (3.13)\n",
            "Requirement already satisfied: pytablewriter in /usr/local/lib/python3.6/dist-packages (0.58.0)\n",
            "Requirement already satisfied: optuna in /usr/local/lib/python3.6/dist-packages (2.3.0)\n",
            "Requirement already satisfied: scikit-optimize in /usr/local/lib/python3.6/dist-packages (0.8.1)\n",
            "Requirement already satisfied: tabledata<2,>=1.1.3 in /usr/local/lib/python3.6/dist-packages (from pytablewriter) (1.1.3)\n",
            "Requirement already satisfied: mbstrdecoder<2,>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from pytablewriter) (1.0.0)\n",
            "Requirement already satisfied: typepy[datetime]<2,>=1.1.1 in /usr/local/lib/python3.6/dist-packages (from pytablewriter) (1.1.2)\n",
            "Requirement already satisfied: msgfy<1,>=0.1.0 in /usr/local/lib/python3.6/dist-packages (from pytablewriter) (0.1.0)\n",
            "Requirement already satisfied: tcolorpy<1,>=0.0.5 in /usr/local/lib/python3.6/dist-packages (from pytablewriter) (0.0.8)\n",
            "Requirement already satisfied: pathvalidate<3,>=2.3.0 in /usr/local/lib/python3.6/dist-packages (from pytablewriter) (2.3.1)\n",
            "Requirement already satisfied: setuptools>=38.3.0 in /usr/local/lib/python3.6/dist-packages (from pytablewriter) (50.3.2)\n",
            "Requirement already satisfied: DataProperty<2,>=0.50.0 in /usr/local/lib/python3.6/dist-packages (from pytablewriter) (0.50.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.6/dist-packages (from optuna) (20.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from optuna) (0.17.0)\n",
            "Requirement already satisfied: cmaes>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from optuna) (0.7.0)\n",
            "Requirement already satisfied: sqlalchemy>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from optuna) (1.3.20)\n",
            "Requirement already satisfied: cliff in /usr/local/lib/python3.6/dist-packages (from optuna) (3.5.0)\n",
            "Requirement already satisfied: colorlog in /usr/local/lib/python3.6/dist-packages (from optuna) (4.6.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from optuna) (4.41.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from optuna) (1.18.5)\n",
            "Requirement already satisfied: scipy!=1.4.0 in /usr/local/lib/python3.6/dist-packages (from optuna) (1.4.1)\n",
            "Requirement already satisfied: alembic in /usr/local/lib/python3.6/dist-packages (from optuna) (1.4.3)\n",
            "Requirement already satisfied: pyaml>=16.9 in /usr/local/lib/python3.6/dist-packages (from scikit-optimize) (20.4.0)\n",
            "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.6/dist-packages (from scikit-optimize) (0.22.2.post1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.4 in /usr/local/lib/python3.6/dist-packages (from mbstrdecoder<2,>=1.0.0->pytablewriter) (3.0.4)\n",
            "Requirement already satisfied: python-dateutil<3.0.0,>=2.8.0; extra == \"datetime\" in /usr/local/lib/python3.6/dist-packages (from typepy[datetime]<2,>=1.1.1->pytablewriter) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2018.9; extra == \"datetime\" in /usr/local/lib/python3.6/dist-packages (from typepy[datetime]<2,>=1.1.1->pytablewriter) (2018.9)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging>=20.0->optuna) (2.4.7)\n",
            "Requirement already satisfied: PrettyTable<0.8,>=0.7.2 in /usr/local/lib/python3.6/dist-packages (from cliff->optuna) (0.7.2)\n",
            "Requirement already satisfied: cmd2!=0.8.3,>=0.8.0 in /usr/local/lib/python3.6/dist-packages (from cliff->optuna) (1.4.0)\n",
            "Requirement already satisfied: pbr!=2.1.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from cliff->optuna) (5.5.1)\n",
            "Requirement already satisfied: stevedore>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from cliff->optuna) (3.3.0)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from cliff->optuna) (1.15.0)\n",
            "Requirement already satisfied: python-editor>=0.3 in /usr/local/lib/python3.6/dist-packages (from alembic->optuna) (1.0.4)\n",
            "Requirement already satisfied: Mako in /usr/local/lib/python3.6/dist-packages (from alembic->optuna) (1.1.3)\n",
            "Requirement already satisfied: attrs>=16.3.0 in /usr/local/lib/python3.6/dist-packages (from cmd2!=0.8.3,>=0.8.0->cliff->optuna) (20.3.0)\n",
            "Requirement already satisfied: pyperclip>=1.6 in /usr/local/lib/python3.6/dist-packages (from cmd2!=0.8.3,>=0.8.0->cliff->optuna) (1.8.1)\n",
            "Requirement already satisfied: wcwidth>=0.1.7 in /usr/local/lib/python3.6/dist-packages (from cmd2!=0.8.3,>=0.8.0->cliff->optuna) (0.2.5)\n",
            "Requirement already satisfied: importlib-metadata>=1.6.0; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from cmd2!=0.8.3,>=0.8.0->cliff->optuna) (3.1.1)\n",
            "Requirement already satisfied: colorama>=0.3.7 in /usr/local/lib/python3.6/dist-packages (from cmd2!=0.8.3,>=0.8.0->cliff->optuna) (0.4.4)\n",
            "Requirement already satisfied: MarkupSafe>=0.9.2 in /usr/local/lib/python3.6/dist-packages (from Mako->alembic->optuna) (1.1.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=1.6.0; python_version < \"3.8\"->cmd2!=0.8.3,>=0.8.0->cliff->optuna) (3.4.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zWBk6gfV5uv6"
      },
      "source": [
        "import imageio\r\n",
        "import base64\r\n",
        "import IPython\r\n",
        "import PIL.Image\r\n",
        "import pyvirtualdisplay\r\n",
        "#Video stuff \r\n",
        "from pathlib import Path\r\n",
        "from IPython import display as ipythondisplay\r\n",
        "\r\n",
        "def show_videos(video_path='', prefix=''):\r\n",
        "  html = []\r\n",
        "  for mp4 in Path(video_path).glob(\"{}*.mp4\".format(prefix)):\r\n",
        "      video_b64 = base64.b64encode(mp4.read_bytes())\r\n",
        "      html.append('''<video alt=\"{}\" autoplay \r\n",
        "                    loop controls style=\"height: 400px;\">\r\n",
        "                    <source src=\"data:video/mp4;base64,{}\" type=\"video/mp4\" />\r\n",
        "                </video>'''.format(mp4, video_b64.decode('ascii')))\r\n",
        "  ipythondisplay.display(ipythondisplay.HTML(data=\"<br>\".join(html)))\r\n",
        "  \r\n",
        "import os\r\n",
        "os.system(\"Xvfb :1 -screen 0 1024x768x24 &\")\r\n",
        "os.environ['DISPLAY'] = ':1'"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7CcAq84v6SXZ"
      },
      "source": [
        "#!sudo apt-get update && sudo apt-get install cmake libopenmpi-dev zlib1g-dev\r\n",
        "#!pip install stable-baselines[mpi]\r\n",
        "\r\n",
        "%tensorflow_version 1.x\r\n",
        "#import tensorflow as tf\r\n",
        "\r\n",
        "#!apt-get install ffmpeg freeglut3-dev xvfb  # For visualization\r\n",
        "#!pip install stable-baselines[mpi]==2.10.0\r\n",
        "\r\n",
        "#!pip install gym\r\n",
        "#!pip install gym[atari]\r\n",
        "\r\n",
        "#import os\r\n",
        "#os.kill(os.getpid(), 9)\r\n",
        "\r\n",
        "import stable_baselines\r\n",
        "stable_baselines.__version__\r\n",
        "\r\n",
        "import gym\r\n",
        "import numpy as np"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9W7bC1ky5zgs",
        "outputId": "38851768-dfcc-42fe-9e5f-3329fe874aab"
      },
      "source": [
        "%tensorflow_version 1.x\r\n",
        "import tensorflow as tf\r\n",
        "tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)\r\n",
        "\r\n",
        "\r\n",
        "try:\r\n",
        "  import optuna\r\n",
        "except ImportError:\r\n",
        "  !pip install optuna\r\n",
        "  import optuna\r\n",
        "import gym\r\n",
        "import os\r\n",
        "import numpy as np\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "\r\n",
        "from stable_baselines import PPO2, DQN\r\n",
        "from stable_baselines.common.vec_env import VecVideoRecorder, DummyVecEnv\r\n",
        "from stable_baselines.common.evaluation import evaluate_policy\r\n",
        "from stable_baselines.common.cmd_util import make_vec_env\r\n",
        "\r\n",
        "def optimize_ppo2(trial):\r\n",
        "    \"\"\" Learning hyperparamters we want to optimise\"\"\"\r\n",
        "    return {\r\n",
        "        'n_steps': int(trial.suggest_loguniform('n_steps', 16, 2048)),\r\n",
        "        'gamma': trial.suggest_loguniform('gamma', 0.9, 0.9999),\r\n",
        "        'learning_rate': trial.suggest_loguniform('learning_rate', 1e-5, 1.),\r\n",
        "        'ent_coef': trial.suggest_loguniform('ent_coef', 1e-8, 1e-1),\r\n",
        "        'cliprange': trial.suggest_uniform('cliprange', 0.1, 0.4),\r\n",
        "        'noptepochs': int(trial.suggest_loguniform('noptepochs', 1, 48)),\r\n",
        "        'lam': trial.suggest_uniform('lam', 0.8, 1.)\r\n",
        "    }\r\n",
        "\r\n",
        "\r\n",
        "def optimize_agent(trial):\r\n",
        "    \"\"\" Train the model and optimize\r\n",
        "        Optuna maximises the negative log likelihood, so we\r\n",
        "        need to negate the reward here\r\n",
        "    \"\"\"\r\n",
        "    # Create log dir\r\n",
        "    log_dir = \"tmp/\"\r\n",
        "    os.makedirs(log_dir, exist_ok=True)\r\n",
        "\r\n",
        "    #env_name='CartPole-v1'\r\n",
        "    env_name='LunarLander-v2'\r\n",
        "    #env_name='MountainCar-v0'\r\n",
        "    #env_name='Acrobot-v1'\r\n",
        "    model_params = optimize_ppo2(trial)\r\n",
        "\r\n",
        "    #env = make_vec_env(lambda: gym.make(env_name), n_envs=16, seed=0)\r\n",
        "    env = gym.make(env_name)\r\n",
        "    env = DummyVecEnv([lambda: env])\r\n",
        "    \r\n",
        "    model = PPO2('MlpPolicy', env, verbose=0, nminibatches=1, **model_params,\r\n",
        "                 tensorboard_log=\"./ppo/\")\r\n",
        "    #model = PPO('MlpPolicy', env, verbose=0, **model_params)\r\n",
        "    \r\n",
        "    time_steps=600000\r\n",
        "    model.learn(total_timesteps=int(time_steps))\r\n",
        "\r\n",
        "\r\n",
        "    mean_reward, _ = evaluate_policy(model, gym.make(env_name), n_eval_episodes=10)\r\n",
        "    #print(model.)\r\n",
        "\r\n",
        "    !rm -rf video\r\n",
        "    #obs = env.reset()\r\n",
        "    length = 100000\r\n",
        "    env=VecVideoRecorder(env, video_folder='./video',\r\n",
        "                                record_video_trigger=lambda step: step == 0, video_length=length,\r\n",
        "                                name_prefix='abc')\r\n",
        "    obs = env.reset()\r\n",
        "    for i in range(length):\r\n",
        "      action, _states = model.predict(obs)\r\n",
        "      obs, rewards, done, info = env.step(action)\r\n",
        "      if done:\r\n",
        "        break\r\n",
        "      #print(\"here2\")\r\n",
        "      #env.render()\r\n",
        "\r\n",
        "    #print(\"here3\")\r\n",
        "    env.close()\r\n",
        "    \r\n",
        "\r\n",
        "    return mean_reward\r\n",
        "\r\n",
        "\r\n",
        "if __name__ == '__main__':\r\n",
        "    study = optuna.create_study(direction=\"maximize\",pruner=optuna.pruners.SuccessiveHalvingPruner())\r\n",
        "    try:\r\n",
        "        study.optimize(optimize_agent, n_trials=2, n_jobs=2,)\r\n",
        "        \r\n",
        "        #%tensorboard --logdir ./ppo/\r\n",
        "    except KeyboardInterrupt:\r\n",
        "        print('Interrupted by keyboard.')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[32m[I 2020-12-14 05:05:07,936]\u001b[0m A new study created in memory with name: no-name-d773db44-47ce-484f-af4a-33b46d3e8e17\u001b[0m\n",
            "\u001b[32m[I 2020-12-14 05:36:56,171]\u001b[0m Trial 0 finished with value: -133.90722672467132 and parameters: {'n_steps': 870.2832258576916, 'gamma': 0.906278247308756, 'learning_rate': 0.5018536936835667, 'ent_coef': 9.138625077453658e-05, 'cliprange': 0.32601010006330855, 'noptepochs': 2.681673432317369, 'lam': 0.8261842718933067}. Best is trial 0 with value: -133.90722672467132.\u001b[0m\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xyUdkCJg6x6E"
      },
      "source": [
        "show_videos('video','abc0-step-0-to-step-80000')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}