From 4645d4fc5343b5934f2c16a79c2ed60a2a71701e Mon Sep 17 00:00:00 2001 From: Lucas Alegre Date: Thu, 20 Jul 2023 13:33:58 -0300 Subject: [PATCH 1/5] Add RecordConstructorArgs --- mo_gymnasium/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mo_gymnasium/utils.py b/mo_gymnasium/utils.py index 24a0db11..6f424690 100644 --- a/mo_gymnasium/utils.py +++ b/mo_gymnasium/utils.py @@ -29,7 +29,7 @@ def make(env_name: str, disable_env_checker: bool = True, **kwargs) -> gym.Env: return gym.make(env_name, disable_env_checker=disable_env_checker, **kwargs) -class LinearReward(gym.Wrapper, EzPickle): +class LinearReward(gym.Wrapper, gym.utils.RecordConstructorArgs): """Makes the env return a scalar reward, which is the dot-product between the reward vector and the weight vector.""" def __init__(self, env: gym.Env, weight: np.ndarray = None): @@ -39,8 +39,8 @@ def __init__(self, env: gym.Env, weight: np.ndarray = None): env: env to wrap weight: weight vector to use in the dot product """ - super().__init__(env) - EzPickle.__init__(self, env, weight) + gym.utils.RecordConstructorArgs.__init__(self, weight=weight) + gym.Wrapper.__init__(self, env) if weight is None: weight = np.ones(shape=env.reward_space.shape) self.set_weight(weight) From c8b4bb3019f42d4a2cd7597916a087152e3fd2cd Mon Sep 17 00:00:00 2001 From: Lucas Alegre Date: Thu, 20 Jul 2023 14:56:19 -0300 Subject: [PATCH 2/5] Fix seed in mo-reacher-v0 --- mo_gymnasium/envs/reacher/reacher.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mo_gymnasium/envs/reacher/reacher.py b/mo_gymnasium/envs/reacher/reacher.py index b01ad915..f881f512 100644 --- a/mo_gymnasium/envs/reacher/reacher.py +++ b/mo_gymnasium/envs/reacher/reacher.py @@ -2,7 +2,7 @@ import numpy as np from gymnasium import spaces -from gymnasium.utils import EzPickle +from gymnasium.utils import EzPickle, seeding from pybulletgym.envs.roboschool.envs.env_bases import BaseBulletEnv from pybulletgym.envs.roboschool.robots.robot_bases import MJCFBasedRobot from pybulletgym.envs.roboschool.scenes.scene_bases import SingleRobotEmptyScene @@ -87,6 +87,8 @@ def camera_adjust(self): def reset(self, seed=None, **kwargs): self._seed(seed) + if seed is not None: + self._np_random, seed = seeding.np_random(seed) obs = super().reset() if self.render_mode == "human": self._render(mode="human") From e65272a01b3d9fcc0bdb1bf6d110471f5825a2d9 Mon Sep 17 00:00:00 2001 From: Lucas Alegre Date: Fri, 21 Jul 2023 22:51:29 -0300 Subject: [PATCH 3/5] Update notebook example --- mo_gymnasium_demo.ipynb | 243 +++++++++++++++++++++++++++++----------- 1 file changed, 179 insertions(+), 64 deletions(-) diff --git a/mo_gymnasium_demo.ipynb b/mo_gymnasium_demo.ipynb index 6cd44ac5..4783901f 100644 --- a/mo_gymnasium_demo.ipynb +++ b/mo_gymnasium_demo.ipynb @@ -8,7 +8,7 @@ "id": "view-in-github" }, "source": [ - "\"Open" + "\"Open" ] }, { @@ -86,13 +86,12 @@ } ], "source": [ - "!git clone https://github.com/Farama-Foundation/MO-Gymnasium\n", - "!pip install -e mo-gymnasium" + "!pip install mo-gymnasium" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 15, "id": "43bddbf7", "metadata": {}, "outputs": [ @@ -100,24 +99,37 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001b[33mDEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\u001b[33m\n", - "\u001b[0mRequirement already satisfied: moviepy in /usr/local/lib/python3.9/site-packages (1.0.3)\n", - "Requirement already satisfied: numpy>=1.17.3 in /usr/local/lib/python3.9/site-packages (from moviepy) (1.21.0)\n", - "Requirement already satisfied: tqdm<5.0,>=4.11.2 in /usr/local/lib/python3.9/site-packages (from moviepy) (4.64.1)\n", - "Requirement already satisfied: requests<3.0,>=2.8.1 in /usr/local/lib/python3.9/site-packages (from moviepy) (2.27.1)\n", - "Requirement already satisfied: imageio<3.0,>=2.5 in /usr/local/lib/python3.9/site-packages (from moviepy) (2.20.0)\n", - "Requirement already satisfied: proglog<=1.0.0 in /usr/local/lib/python3.9/site-packages (from moviepy) (0.1.10)\n", - "Requirement already satisfied: decorator<5.0,>=4.0.2 in /usr/local/lib/python3.9/site-packages (from moviepy) (4.4.2)\n", - "Requirement already satisfied: imageio-ffmpeg>=0.2.0 in /usr/local/lib/python3.9/site-packages (from moviepy) (0.4.7)\n", - "Requirement already satisfied: pillow>=8.3.2 in /usr/local/lib/python3.9/site-packages (from imageio<3.0,>=2.5->moviepy) (9.2.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (2021.10.8)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (3.3)\n", - "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (2.0.12)\n", - "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (1.26.8)\n", - "\u001b[33mDEPRECATION: Configuring installation scheme with distutils config files is deprecated and will no longer work in the near future. If you are using a Homebrew or Linuxbrew Python, please see discussion at https://github.com/Homebrew/homebrew-core/issues/76621\u001b[0m\u001b[33m\n", - "\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.2.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m22.3.1\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpython3.9 -m pip install --upgrade pip\u001b[0m\n" + "Collecting moviepy\n", + " Downloading moviepy-1.0.3.tar.gz (388 kB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.3/388.3 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m[36m0:00:01\u001b[0m[36m0:00:01\u001b[0m\n", + "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25hCollecting decorator<5.0,>=4.0.2 (from moviepy)\n", + " Using cached decorator-4.4.2-py2.py3-none-any.whl (9.2 kB)\n", + "Requirement already satisfied: tqdm<5.0,>=4.11.2 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from moviepy) (4.62.3)\n", + "Requirement already satisfied: requests<3.0,>=2.8.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from moviepy) (2.26.0)\n", + "Collecting proglog<=1.0.0 (from moviepy)\n", + " Downloading proglog-0.1.10-py3-none-any.whl (6.1 kB)\n", + "Requirement already satisfied: numpy>=1.17.3 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from moviepy) (1.25.1)\n", + "Requirement already satisfied: imageio<3.0,>=2.5 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from moviepy) (2.19.2)\n", + "Collecting imageio_ffmpeg>=0.2.0 (from moviepy)\n", + " Downloading imageio_ffmpeg-0.4.8-py3-none-manylinux2010_x86_64.whl (26.9 MB)\n", + "\u001b[2K \u001b[38;2;114;156;31m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.9/26.9 MB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mm eta \u001b[36m0:00:01\u001b[0m0:01\u001b[0m:01\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pillow>=8.3.2 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from imageio<3.0,>=2.5->moviepy) (8.4.0)\n", + "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (1.26.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (2021.10.8)\n", + "Requirement already satisfied: charset-normalizer~=2.0.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (2.0.7)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from requests<3.0,>=2.8.1->moviepy) (3.3)\n", + "Building wheels for collected packages: moviepy\n", + " Building wheel for moviepy (setup.py) ... \u001b[?25ldone\n", + "\u001b[?25h Created wheel for moviepy: filename=moviepy-1.0.3-py3-none-any.whl size=110744 sha256=ebfb73108e06c510590c09d7342c453afbe3c16206070110f5707d8707ac295e\n", + " Stored in directory: /home/lucas/.cache/pip/wheels/29/15/e4/4f790bec6acd51a00b67e8ee1394f0bc6e0135c315f8ff399a\n", + "Successfully built moviepy\n", + "Installing collected packages: proglog, imageio_ffmpeg, decorator, moviepy\n", + " Attempting uninstall: decorator\n", + " Found existing installation: decorator 5.1.0\n", + " Uninstalling decorator-5.1.0:\n", + " Successfully uninstalled decorator-5.1.0\n", + "Successfully installed decorator-4.4.2 imageio_ffmpeg-0.4.8 moviepy-1.0.3 proglog-0.1.10\n" ] } ], @@ -128,7 +140,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "id": "846b9364", "metadata": { "id": "846b9364" @@ -151,14 +163,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "id": "d5415397", "metadata": { "id": "d5415397" }, "outputs": [], "source": [ - "env = mo_gym.make(\"mo-mountaincar-v0\", render_mode=\"human\")" + "env = mo_gym.make(\"mo-mountaincar-v0\", render_mode=\"rgb_array\")" ] }, { @@ -173,7 +185,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 3, "id": "8db784fa", "metadata": { "colab": { @@ -189,7 +201,7 @@ "Box([-1.2 -0.07], [0.6 0.07], (2,), float32)" ] }, - "execution_count": 7, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -200,7 +212,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 4, "id": "8e8d91c7", "metadata": { "colab": { @@ -216,7 +228,7 @@ "Discrete(3)" ] }, - "execution_count": 8, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -227,7 +239,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 5, "id": "5bff02b6", "metadata": { "colab": { @@ -237,13 +249,21 @@ "outputId": "fe8ee38e-ff94-4ad2-9d9c-e2cc58d89159" }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages/gymnasium/core.py:297: UserWarning: \u001b[33mWARN: env.reward_space to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.reward_space` for environment variables or `env.get_attr('reward_space')` that will search the reminding wrappers.\u001b[0m\n", + " logger.warn(\n" + ] + }, { "data": { "text/plain": [ - "Box(-1.0, 1.0, (3,), float32)" + "Box(-1.0, [-1. 0. 0.], (3,), float32)" ] }, - "execution_count": 9, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -265,19 +285,19 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "id": "c64e5417", "metadata": { "id": "c64e5417" }, "outputs": [], "source": [ - "from gym.wrappers.record_video import RecordVideo" + "from gymnasium.wrappers.record_video import RecordVideo" ] }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 4, "id": "a4190496", "metadata": { "colab": { @@ -291,7 +311,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "/usr/local/lib/python3.9/site-packages/gym/wrappers/record_video.py:75: UserWarning: \u001b[33mWARN: Overwriting existing videos at /Users/florian.felten/Documents/mo-gym/videos/demo folder (try specifying a different `video_folder` for the `RecordVideo` wrapper if this is not desired)\u001b[0m\n", + "/home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages/gymnasium/wrappers/record_video.py:94: UserWarning: \u001b[33mWARN: Overwriting existing videos at /home/lucas/Desktop/MO-Gymnasium/videos/demo folder (try specifying a different `video_folder` for the `RecordVideo` wrapper if this is not desired)\u001b[0m\n", + " logger.warn(\n", + "/home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages/gymnasium/core.py:297: UserWarning: \u001b[33mWARN: env.is_vector_env to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.is_vector_env` for environment variables or `env.get_attr('is_vector_env')` that will search the reminding wrappers.\u001b[0m\n", " logger.warn(\n" ] } @@ -312,21 +334,51 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 5, "id": "066db701", "metadata": { "id": "066db701" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Moviepy - Building video /home/lucas/Desktop/MO-Gymnasium/videos/demo/rl-video-episode-0.mp4.\n", + "Moviepy - Writing video /home/lucas/Desktop/MO-Gymnasium/videos/demo/rl-video-episode-0.mp4\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " " + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Moviepy - Done !\n", + "Moviepy - video ready /home/lucas/Desktop/MO-Gymnasium/videos/demo/rl-video-episode-0.mp4\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r" + ] + } + ], "source": [ "env.reset()\n", "done = False\n", "\n", "while not done:\n", " obs, vec_reward, terminated, truncated, info = env.step(env.action_space.sample())\n", - " done = terminated or truncated\n", - " \n", - "env.close()" + " done = terminated or truncated" ] }, { @@ -342,7 +394,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "DHkO3R7uccOy", "metadata": { "colab": { @@ -351,14 +403,75 @@ "id": "DHkO3R7uccOy", "outputId": "7dcaa4a2-f566-40e8-f62a-12d4320c3df4" }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: stable-baselines3 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (1.5.0)\n", + "Collecting stable-baselines3\n", + " Obtaining dependency information for stable-baselines3 from https://files.pythonhosted.org/packages/d9/57/13d4e4b7bbbc940815964ac31e205263b8133f1f2a0147bd4ca884a6e174/stable_baselines3-2.0.0-py3-none-any.whl.metadata\n", + " Using cached stable_baselines3-2.0.0-py3-none-any.whl.metadata (5.4 kB)\n", + "Requirement already satisfied: gymnasium==0.28.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (0.28.1)\n", + "Requirement already satisfied: numpy>=1.20 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (1.25.1)\n", + "Collecting torch>=1.11 (from stable-baselines3)\n", + " Using cached torch-2.0.1-cp39-cp39-manylinux1_x86_64.whl (619.9 MB)\n", + "Requirement already satisfied: cloudpickle in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (2.0.0)\n", + "Requirement already satisfied: pandas in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (1.3.4)\n", + "Requirement already satisfied: matplotlib in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from stable-baselines3) (3.5.1)\n", + "Requirement already satisfied: jax-jumpy>=1.0.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from gymnasium==0.28.1->stable-baselines3) (1.0.0)\n", + "Requirement already satisfied: typing-extensions>=4.3.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from gymnasium==0.28.1->stable-baselines3) (4.7.1)\n", + "Requirement already satisfied: farama-notifications>=0.0.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from gymnasium==0.28.1->stable-baselines3) (0.0.4)\n", + "Requirement already satisfied: importlib-metadata>=4.8.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from gymnasium==0.28.1->stable-baselines3) (4.11.2)\n", + "Requirement already satisfied: filelock in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (3.4.0)\n", + "Requirement already satisfied: sympy in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (1.12)\n", + "Requirement already satisfied: networkx in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (2.6.3)\n", + "Requirement already satisfied: jinja2 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (3.0.3)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.99)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.99)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu11==11.7.101 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.101)\n", + "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (8.5.0.96)\n", + "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.10.3.66)\n", + "Requirement already satisfied: nvidia-cufft-cu11==10.9.0.58 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (10.9.0.58)\n", + "Requirement already satisfied: nvidia-curand-cu11==10.2.10.91 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (10.2.10.91)\n", + "Requirement already satisfied: nvidia-cusolver-cu11==11.4.0.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.4.0.1)\n", + "Requirement already satisfied: nvidia-cusparse-cu11==11.7.4.91 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.4.91)\n", + "Requirement already satisfied: nvidia-nccl-cu11==2.14.3 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (2.14.3)\n", + "Requirement already satisfied: nvidia-nvtx-cu11==11.7.91 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (11.7.91)\n", + "Requirement already satisfied: triton==2.0.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from torch>=1.11->stable-baselines3) (2.0.0)\n", + "Requirement already satisfied: setuptools in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.11->stable-baselines3) (58.0.4)\n", + "Requirement already satisfied: wheel in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.11->stable-baselines3) (0.37.0)\n", + "Requirement already satisfied: cmake in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.11->stable-baselines3) (3.27.0)\n", + "Requirement already satisfied: lit in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from triton==2.0.0->torch>=1.11->stable-baselines3) (16.0.6)\n", + "Requirement already satisfied: cycler>=0.10 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (0.10.0)\n", + "Requirement already satisfied: fonttools>=4.22.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (4.28.5)\n", + "Requirement already satisfied: kiwisolver>=1.0.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (1.3.2)\n", + "Requirement already satisfied: packaging>=20.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (21.3)\n", + "Requirement already satisfied: pillow>=6.2.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (8.4.0)\n", + "Requirement already satisfied: pyparsing>=2.2.1 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (2.4.7)\n", + "Requirement already satisfied: python-dateutil>=2.7 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from matplotlib->stable-baselines3) (2.8.2)\n", + "Requirement already satisfied: pytz>=2017.3 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from pandas->stable-baselines3) (2021.3)\n", + "Requirement already satisfied: six in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from cycler>=0.10->matplotlib->stable-baselines3) (1.16.0)\n", + "Requirement already satisfied: zipp>=0.5 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from importlib-metadata>=4.8.0->gymnasium==0.28.1->stable-baselines3) (3.7.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from jinja2->torch>=1.11->stable-baselines3) (2.1.0)\n", + "Requirement already satisfied: mpmath>=0.19 in /home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages (from sympy->torch>=1.11->stable-baselines3) (1.3.0)\n", + "Using cached stable_baselines3-2.0.0-py3-none-any.whl (178 kB)\n", + "Installing collected packages: torch, stable-baselines3\n", + " Attempting uninstall: stable-baselines3\n", + " Found existing installation: stable-baselines3 1.5.0\n", + " Uninstalling stable-baselines3-1.5.0:\n", + " Successfully uninstalled stable-baselines3-1.5.0\n", + "Successfully installed stable-baselines3-2.0.0 torch-2.0.1\n" + ] + } + ], "source": [ - "!pip install stable-baselines3" + "!pip install stable-baselines3 --upgrade" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "lgPC1qU8c-0A", "metadata": { "colab": { @@ -376,24 +489,7 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "SdkUeJHudZ17", - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 592 - }, - "id": "SdkUeJHudZ17", - "outputId": "9557593d-4893-4c4d-dccb-76fefd73dddd" - }, - "outputs": [], - "source": [ - "!pip install gym==0.24.0" - ] - }, - { - "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "mn7dMweScz1U", "metadata": { "colab": { @@ -402,7 +498,26 @@ "id": "mn7dMweScz1U", "outputId": "fe80d623-a46d-4d9e-bd27-75188c12aa19" }, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/lucas/miniconda3/envs/PhD/lib/python3.9/site-packages/gymnasium/core.py:297: UserWarning: \u001b[33mWARN: env.reward_space to get variables from other wrappers is deprecated and will be removed in v1.0, to get this variable you can do `env.unwrapped.reward_space` for environment variables or `env.get_attr('reward_space')` that will search the reminding wrappers.\u001b[0m\n", + " logger.warn(\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Linear scalarizes the environment\n", "env = mo_gym.LinearReward(mo_gym.make(\"mo-mountaincar-v0\"), weight=np.array([0.9, 0.1, 0.0]))\n", From d4d11b1424087528d51252e809dab1675d71efdd Mon Sep 17 00:00:00 2001 From: Lucas Alegre Date: Mon, 24 Jul 2023 10:39:30 -0300 Subject: [PATCH 4/5] Add RecordConstructorArgs --- .gitignore | 6 ++++++ mo_gymnasium/utils.py | 26 ++++++++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index b6e47617..c85bad37 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,9 @@ dmypy.json # Pyre type checker .pyre/ + +# vscode +.vscode/ + +# videos +videos/ diff --git a/mo_gymnasium/utils.py b/mo_gymnasium/utils.py index 6f424690..95b7ec9a 100644 --- a/mo_gymnasium/utils.py +++ b/mo_gymnasium/utils.py @@ -6,10 +6,9 @@ import gymnasium as gym import numpy as np -from gymnasium.utils import EzPickle from gymnasium.vector import SyncVectorEnv -from gymnasium.wrappers import RecordEpisodeStatistics from gymnasium.wrappers.normalize import RunningMeanStd +from gymnasium.wrappers.record_episode_statistics import RecordEpisodeStatistics ObsType = TypeVar("ObsType") @@ -70,7 +69,7 @@ def step(self, action: ActType) -> Tuple[ObsType, float, bool, bool, dict]: return observation, scalar_reward, terminated, truncated, info -class MONormalizeReward(gym.Wrapper, EzPickle): +class MONormalizeReward(gym.Wrapper, gym.utils.RecordConstructorArgs): """Wrapper to normalize the reward component at index idx. Does not touch other reward components.""" def __init__(self, env: gym.Env, idx: int, gamma: float = 0.99, epsilon: float = 1e-8): @@ -82,8 +81,8 @@ def __init__(self, env: gym.Env, idx: int, gamma: float = 0.99, epsilon: float = epsilon (float): A stability parameter gamma (float): The discount factor that is used in the exponential moving average. """ - super().__init__(env) - EzPickle.__init__(self, env, idx, gamma, epsilon) + gym.utils.RecordConstructorArgs.__init__(self, idx=idx, gamma=gamma, epsilon=epsilon) + gym.Wrapper.__init__(self, env) self.idx = idx self.num_envs = getattr(env, "num_envs", 1) self.is_vector_env = getattr(env, "is_vector_env", False) @@ -125,7 +124,7 @@ def normalize(self, rews): return rews / np.sqrt(self.return_rms.var + self.epsilon) -class MOClipReward(gym.RewardWrapper, EzPickle): +class MOClipReward(gym.RewardWrapper, gym.utils.RecordConstructorArgs): """Clip reward[idx] to [min, max].""" def __init__(self, env: gym.Env, idx: int, min_r, max_r): @@ -137,8 +136,8 @@ def __init__(self, env: gym.Env, idx: int, min_r, max_r): min_r: min reward max_r: max reward """ - super().__init__(env) - EzPickle.__init__(self, env, idx, min_r, max_r) + gym.utils.RecordConstructorArgs.__init__(self, idx=idx, min_r=min_r, max_r=max_r) + gym.RewardWrapper.__init__(self, env) self.idx = idx self.min_r = min_r self.max_r = max_r @@ -154,7 +153,7 @@ def reward(self, reward): return reward -class MOSyncVectorEnv(SyncVectorEnv, EzPickle): +class MOSyncVectorEnv(SyncVectorEnv): """Vectorized environment that serially runs multiple environments.""" def __init__( @@ -168,8 +167,7 @@ def __init__( env_fns: env constructors copy: If ``True``, then the :meth:`reset` and :meth:`step` methods return a copy of the observations. """ - super().__init__(env_fns, copy=copy) - EzPickle.__init__(self, env_fns, copy=copy) + SyncVectorEnv.__init__(self, env_fns, copy=copy) # Just overrides the rewards memory to add the number of objectives self.reward_space = self.envs[0].reward_space self._rewards = np.zeros( @@ -181,7 +179,7 @@ def __init__( ) -class MORecordEpisodeStatistics(RecordEpisodeStatistics, EzPickle): +class MORecordEpisodeStatistics(RecordEpisodeStatistics, gym.utils.RecordConstructorArgs): """This wrapper will keep track of cumulative rewards and episode lengths. After the completion of an episode, ``info`` will look like this:: @@ -220,8 +218,8 @@ def __init__(self, env: gym.Env, gamma: float = 1.0, deque_size: int = 100): gamma (float): Discounting factor deque_size: The size of the buffers :attr:`return_queue` and :attr:`length_queue` """ - super().__init__(env, deque_size) - EzPickle.__init__(self, env, gamma, deque_size) + gym.utils.RecordConstructorArgs.__init__(self, gamma=gamma, deque_size=deque_size) + RecordEpisodeStatistics.__init__(self, env, deque_size=deque_size) # CHANGE: Here we just override the standard implementation to extend to MO # We also take care of the case where the env is vectorized self.reward_dim = self.env.reward_space.shape[0] From 85508d1e48130869103c5de3478a49625f48b1ef Mon Sep 17 00:00:00 2001 From: Lucas Alegre Date: Mon, 24 Jul 2023 12:08:40 -0300 Subject: [PATCH 5/5] Bump Gymnasium requirement --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 14ab961e..ce79c86f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ 'Topic :: Scientific/Engineering :: Artificial Intelligence', ] dependencies = [ - "gymnasium >=0.27", + "gymnasium>=0.28.1,<0.30", "numpy >=1.21.0", "pygame >=2.1.0", "scipy >=1.7.3",