New Depth Cam for all platform (#798)

* a new depth camera is implemented to use the depth test! * add installation trouble shooting * add message * use compute shader only for cuda * new feature tested with cuda windows * depth camera is tested on all platforms, let's restore the test * update readme * fix ramp bug * format * not run make video
metadriverse · Jan 17, 2025 · aadad4d · aadad4d
1 parent 5656fd2
commit aadad4d
Show file tree

Hide file tree

Showing 16 changed files with 215 additions and 182 deletions.
diff --git a/README.md b/README.md
@@ -34,9 +34,9 @@
 
 MetaDrive is a driving simulator with the following key features:
 
-- **Compositional**: It supports generating infinite scenes with various road maps and traffic settings for the research of generalizable RL. 
-- **Lightweight**: It is easy to install and run. It can run up to +1000 FPS on a standard PC.
-- **Realistic**: Accurate physics simulation and multiple sensory input including Lidar, RGB images, top-down semantic map and first-person view images. 
+- **Compositional**: It supports synthesising infinite scenes with various road maps and traffic settings or loading real-world driving logs for the research of generalizable RL. 
+- **Lightweight**: It is easy to install and run on Linux/Windows/MacOS with sensor simulation support. It can run up to +1000 FPS on a standard PC.
+- **Realistic**: Accurate physics simulation and multiple sensory input including Lidar, RGB/Depth/Semantic images, top-down semantic map and first-person view images. 
 
 
 ## 🛠 Quick Start
@@ -48,8 +48,6 @@ cd metadrive
 pip install -e .
 ```
 
-*Note that the program is tested on both Linux and Windows, but some issues regarding depth camera exist when using macOS. 
-
 You can verify the installation of MetaDrive via running the testing script:
 
 ```bash

diff --git a/documentation/source/install.rst b/documentation/source/install.rst
@@ -107,5 +107,6 @@ Known Issues
 ######################
 
 
-* We find that Mac with M1 chip is incompatible with Panda3D rendering utilities.
-* Pygame 1.9.6 causes blank visualization window in Mac with M1 chip. Upgrading to pygame==2.0.0 solves.
+* Pygame 1.9.6 causes blank visualization window in Mac with M1 chip. Upgrading to pygame==2.0.0 solves.
+* If you are using cuda feature on Windows and encounter compiling issue when install  PyOpenGL-accelerate, lower the Python version to i.e. 3.8
+* To use cuda feature, you need OpenGL >= 4.3, which has the compute shader support
diff --git a/documentation/source/simgen_render.ipynb b/documentation/source/simgen_render.ipynb
@@ -191,11 +191,11 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\u001B[38;20m[INFO] Environment: ScenarioEnv\u001B[0m\n",
-      "\u001B[38;20m[INFO] MetaDrive version: 0.4.3\u001B[0m\n",
-      "\u001B[38;20m[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector(), depth_camera: DepthCamera(800, 450), rgb_camera: RGBCamera(800, 450), seg_camera: SemanticCamera(800, 450)]\u001B[0m\n",
-      "\u001B[38;20m[INFO] Render Mode: offscreen\u001B[0m\n",
-      "\u001B[38;20m[INFO] Horizon (Max steps per agent): 1000\u001B[0m\n"
+      "\u001b[38;20m[INFO] Environment: ScenarioEnv\u001b[0m\n",
+      "\u001b[38;20m[INFO] MetaDrive version: 0.4.3\u001b[0m\n",
+      "\u001b[38;20m[INFO] Sensors: [lidar: Lidar(), side_detector: SideDetector(), lane_line_detector: LaneLineDetector(), depth_camera: DepthCamera(800, 450), rgb_camera: RGBCamera(800, 450), seg_camera: SemanticCamera(800, 450)]\u001b[0m\n",
+      "\u001b[38;20m[INFO] Render Mode: offscreen\u001b[0m\n",
+      "\u001b[38;20m[INFO] Horizon (Max steps per agent): 1000\u001b[0m\n"
      ]
     }
    ],
@@ -272,24 +272,70 @@
     "### Rollout"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c394d21c0f053a32",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "skip_steps = 1\n",
+    "fps = 10\n",
+    "\n",
+    "frames = []\n",
+    "\n",
+    "try:\n",
+    "    env.reset()\n",
+    "    scenario = env.engine.data_manager.current_scenario\n",
+    "    scenario_id = scenario['id']\n",
+    "    print(\n",
+    "        \"Current scenario ID {}, dataset version {}, len: {}\".format(\n",
+    "            scenario_id, scenario['version'], scenario['length']\n",
+    "        )\n",
+    "    )\n",
+    "    horizon = scenario['length']\n",
+    "    for t in tqdm.trange(horizon):\n",
+    "        o, r, d, _, _ = env.step([1, 0.88])\n",
+    "        if t % skip_steps == 0:\n",
+    "            depth_img = Image.fromarray(o[\"depth\"].repeat(3, axis=-1), mode=\"RGB\")\n",
+    "            seg_img = Image.fromarray(o[\"seg\"], mode=\"RGB\")\n",
+    "            rgb_img = Image.fromarray(o[\"rgb\"], mode=\"RGB\")\n",
+    "    \n",
+    "            vis = cv2.hconcat([o[\"seg\"], o[\"depth\"].repeat(3, axis=-1)])\n",
+    "            h, w, _ = o[\"rgb\"].shape\n",
+    "            vis_w = vis.shape[1]\n",
+    "            image = cv2.resize(o[\"rgb\"], (vis_w, int(h * vis_w / w)))\n",
+    "            vis = cv2.vconcat([vis, image])\n",
+    "    \n",
+    "            # Quick visualization:\n",
+    "            # import matplotlib.pyplot as plt;plt.imshow(vis);plt.show()\n",
+    "            frames.append(vis)\n",
+    "finally:\n",
+    "    env.close()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 4,
    "id": "080d5870-0abd-4826-95bf-5a6ed373bf9b",
-   "metadata": {},
+   "metadata": {
+    "tags": [
+     "skip-execution"
+    ]
+   },
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\u001B[38;20m[INFO] Assets version: 0.4.3\u001B[0m\n",
-      "\u001B[38;20m[INFO] Known Pipes: glxGraphicsPipe\u001B[0m\n",
-      "\u001B[38;20m[INFO] Assets version: 0.4.3\u001B[0m\n",
-      "\u001B[38;20m[INFO] Known Pipes: glxGraphicsPipe\u001B[0m\n",
-      "\u001B[33;20m[WARNING] You are using too large buffer! The height is 450, and width is 800. It may lower the sample efficiency! Consider reducing buffer size or use cuda image by set [image_on_cuda=True]. (base_camera.py:49)\u001B[0m\n",
-      "\u001B[33;20m[WARNING] You are using too large buffer! The height is 450, and width is 800. It may lower the sample efficiency! Consider reducing buffer size or use cuda image by set [image_on_cuda=True]. (base_camera.py:49)\u001B[0m\n",
-      "\u001B[33;20m[WARNING] You are using too large buffer! The height is 450, and width is 800. It may lower the sample efficiency! Consider reducing buffer size or use cuda image by set [image_on_cuda=True]. (base_camera.py:49)\u001B[0m\n",
-      "\u001B[38;20m[INFO] Start Scenario Index: 0, Num Scenarios : 9\u001B[0m\n"
+      "\u001b[38;20m[INFO] Assets version: 0.4.3\u001b[0m\n",
+      "\u001b[38;20m[INFO] Known Pipes: glxGraphicsPipe\u001b[0m\n",
+      "\u001b[38;20m[INFO] Assets version: 0.4.3\u001b[0m\n",
+      "\u001b[38;20m[INFO] Known Pipes: glxGraphicsPipe\u001b[0m\n",
+      "\u001b[33;20m[WARNING] You are using too large buffer! The height is 450, and width is 800. It may lower the sample efficiency! Consider reducing buffer size or use cuda image by set [image_on_cuda=True]. (base_camera.py:49)\u001b[0m\n",
+      "\u001b[33;20m[WARNING] You are using too large buffer! The height is 450, and width is 800. It may lower the sample efficiency! Consider reducing buffer size or use cuda image by set [image_on_cuda=True]. (base_camera.py:49)\u001b[0m\n",
+      "\u001b[33;20m[WARNING] You are using too large buffer! The height is 450, and width is 800. It may lower the sample efficiency! Consider reducing buffer size or use cuda image by set [image_on_cuda=True]. (base_camera.py:49)\u001b[0m\n",
+      "\u001b[38;20m[INFO] Start Scenario Index: 0, Num Scenarios : 9\u001b[0m\n"
      ]
     },
     {
@@ -303,7 +349,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "\u001B[38;20m[INFO] Episode ended! Scenario Index: 0 Scenario id: scene-0061 Reason: arrive_dest.\u001B[0m\n",
+      "\u001b[38;20m[INFO] Episode ended! Scenario Index: 0 Scenario id: scene-0061 Reason: arrive_dest.\u001b[0m\n",
       "\n",
       "00%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 191/191 [00:10<00:00, 17.68it/s]"
      ]
@@ -325,40 +371,6 @@
     }
    ],
    "source": [
-    "skip_steps = 1\n",
-    "fps = 10\n",
-    "\n",
-    "frames = []\n",
-    "\n",
-    "try:\n",
-    "    env.reset()\n",
-    "    scenario = env.engine.data_manager.current_scenario\n",
-    "    scenario_id = scenario['id']\n",
-    "    print(\n",
-    "        \"Current scenario ID {}, dataset version {}, len: {}\".format(\n",
-    "            scenario_id, scenario['version'], scenario['length']\n",
-    "        )\n",
-    "    )\n",
-    "    horizon = scenario['length']\n",
-    "    for t in tqdm.trange(horizon):\n",
-    "        o, r, d, _, _ = env.step([1, 0.88])\n",
-    "        if t % skip_steps == 0:\n",
-    "            depth_img = Image.fromarray(o[\"depth\"].repeat(3, axis=-1), mode=\"RGB\")\n",
-    "            seg_img = Image.fromarray(o[\"seg\"], mode=\"RGB\")\n",
-    "            rgb_img = Image.fromarray(o[\"rgb\"], mode=\"RGB\")\n",
-    "    \n",
-    "            vis = cv2.hconcat([o[\"seg\"], o[\"depth\"].repeat(3, axis=-1)])\n",
-    "            h, w, _ = o[\"rgb\"].shape\n",
-    "            vis_w = vis.shape[1]\n",
-    "            image = cv2.resize(o[\"rgb\"], (vis_w, int(h * vis_w / w)))\n",
-    "            vis = cv2.vconcat([vis, image])\n",
-    "    \n",
-    "            # Quick visualization:\n",
-    "            # import matplotlib.pyplot as plt;plt.imshow(vis);plt.show()\n",
-    "            frames.append(vis)\n",
-    "finally:\n",
-    "    env.close()\n",
-    "\n",
     "# Show video\n",
     "media.show_video(frames, fps=fps, width=600)"
    ]
@@ -380,10 +392,10 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.8.20"
   },
   "mystnb": {
-   "execution_mode": "force"
+   "execution_mode": "auto"
   }
  },
  "nbformat": 4,

diff --git a/metadrive/component/lane/extension_lane.py b/metadrive/component/lane/extension_lane.py
@@ -14,20 +14,19 @@ def __init__(self, extension_direction: ExtensionDirection, *args, **kwargs):
         super(ExtendingLane, self).__init__(*args, **kwargs)
         self.extension_direction = extension_direction
 
-    def width_at(self, longitudinal: float) -> float:
-        if self.extension_direction == ExtensionDirection.EXTEND:
-            return (longitudinal / self.length) * self.width
-        else:
-            return self.width - (longitudinal / self.length) * self.width
-
-    def get_polyline(self, interval=2, lateral=0):
-        ret = []
-        for i in np.arange(0, self.length, interval):
-            max_lateral = self.width_at(i) - self.width / 2
-            ret.append(self.position(i, min(lateral, max_lateral)))
-        last_lateral = self.width_at(self.length) - self.width / 2
-        ret.append(self.position(self.length, min(lateral, last_lateral)))
-        return np.array(ret)
+    # def width_at(self, longitudinal: float) -> float:
+    #     if self.extension_direction == ExtensionDirection.EXTEND:
+    #         return (longitudinal / self.length) * self.width
+    #     else:
+    #         return self.width - (longitudinal / self.length) * self.width
+    #
+    # def get_polyline(self, interval=2, lateral=0):
+    #     ret = []
+    #     for i in np.arange(0, self.length, interval):
+    #         ret.append(self.position(i, lateral))
+    #     last_lateral = self.width_at(self.length) - self.width / 2
+    #     ret.append(self.position(self.length, min(lateral, last_lateral)))
+    #     return np.array(ret)
 
     @property
     def polygon(self):

diff --git a/metadrive/component/pgblock/create_pg_block_utils.py b/metadrive/component/pgblock/create_pg_block_utils.py
@@ -19,16 +19,15 @@
 
 def create_extension(
     start_position: np.ndarray,
-    extension_lane_length: float,
+    end_position: np.ndarray,
     direction: ExtensionDirection,
     width: float = PGLane.DEFAULT_WIDTH,
     forbidden: bool = False,
     speed_limit: float = 20,
     priority: int = 0
 ):
-    extension_end = start_position + [extension_lane_length, 0.0]
     extension = ExtendingLane(
-        direction, start_position, extension_end, width, [PGLineType.NONE, PGLineType.CONTINUOUS], forbidden,
+        direction, start_position, end_position, width, [PGLineType.NONE, PGLineType.CONTINUOUS], forbidden,
         speed_limit, priority
     )
     return extension

diff --git a/metadrive/component/pgblock/ramp.py b/metadrive/component/pgblock/ramp.py
@@ -7,7 +7,7 @@
 from metadrive.component.pgblock.pg_block import PGBlock
 from metadrive.component.lane.extension_lane import ExtensionDirection
 from metadrive.component.road_network import Road
-from metadrive.constants import Decoration, PGLineType
+from metadrive.constants import PGLineType
 from metadrive.utils.pg.utils import check_lane_on_road
 from metadrive.component.pg_space import ParameterSpace, Parameter, BlockParameterSpace
 
@@ -42,7 +42,8 @@ class InRampOnStraight(Ramp):
 
     def _get_merge_part(self, att_lane: StraightLane, length: float):
         start = att_lane.end
-        merge_part = create_extension(start, length, ExtensionDirection.SHRINK, width=self.lane_width)
+        end = att_lane.position(att_lane.length + length, -self.lane_width)
+        merge_part = create_extension(start, end, ExtensionDirection.SHRINK, width=self.lane_width)
         return merge_part
 
     def _try_plug_into_previous_block(self) -> bool:
@@ -185,10 +186,11 @@ def _try_plug_into_previous_block(self) -> bool:
             self.LANE_TYPE,
             speed_limit=self.SPEED_LIMIT
         )
-        acc_lane.line_types = [PGLineType.BROKEN, PGLineType.CONTINUOUS]
+        acc_lane.line_types = [PGLineType.NONE, PGLineType.CONTINUOUS]
         bend_2_road = Road(connect_road.end_node, self.road_node(0, 0))  # end at part1 road 0, extend road
         self.block_network.add_lane(bend_2_road.start_node, bend_2_road.end_node, bend_2)
-        self.block_network.add_lane(acc_road.start_node, acc_road.end_node, acc_lane)
+        next_node = self.add_road_node()
+        self.block_network.add_lane(acc_road.start_node, next_node, acc_lane)
         no_cross = (
             not check_lane_on_road(
                 self._global_network, bend_2, 0.95, ignore_intersection_checking=self.ignore_intersection_checking
@@ -202,7 +204,7 @@ def _try_plug_into_previous_block(self) -> bool:
 
         # p1, road 4, small circular to decorate
         merge_lane = self._get_merge_part(acc_lane, extension_length)
-        self.block_network.add_lane(Decoration.start, Decoration.end, merge_lane)
+        self.block_network.add_lane(next_node, acc_road.end_node, merge_lane)
 
         return no_cross
 
@@ -228,8 +230,9 @@ class OutRampOnStraight(Ramp):
     EXTRA_LEN = 15
 
     def _get_merge_part(self, att_lane: StraightLane, length: float):
-        start = att_lane.position(0, self.lane_width)
-        merge_part = create_extension(start, length, ExtensionDirection.EXTEND, width=self.lane_width)
+        start = att_lane.position(0, 0)
+        end = att_lane.position(length, self.lane_width)
+        merge_part = create_extension(start, end, ExtensionDirection.EXTEND, width=self.lane_width)
         return merge_part
 
     def _try_plug_into_previous_block(self) -> bool:
@@ -295,13 +298,14 @@ def _try_plug_into_previous_block(self) -> bool:
         # part 1 road 0
         self.set_part_idx(1)
         merge_part_lane = self._get_merge_part(dec_right_lane, extension_len)
-        self.block_network.add_lane(Decoration.start, Decoration.end, merge_part_lane)
+        next_node = self.add_road_node()
+        self.block_network.add_lane(dec_road.start_node, next_node, merge_part_lane)
 
         deacc_lane_end = dec_right_lane.position(dec_right_lane.length, self.lane_width)
         deacc_lane = StraightLane(
-            merge_part_lane.end, deacc_lane_end, self.lane_width, (PGLineType.BROKEN, PGLineType.CONTINUOUS)
+            merge_part_lane.end, deacc_lane_end, self.lane_width, (PGLineType.NONE, PGLineType.CONTINUOUS)
         )
-        self.block_network.add_lane(dec_road.start_node, dec_road.end_node, deacc_lane)
+        self.block_network.add_lane(next_node, dec_road.end_node, deacc_lane)
         no_cross = (
             not check_lane_on_road(
                 self._global_network, deacc_lane, 0.95, ignore_intersection_checking=self.ignore_intersection_checking

diff --git a/metadrive/component/sensors/base_camera.py b/metadrive/component/sensors/base_camera.py
@@ -53,7 +53,7 @@ def __init__(self, engine, need_cuda=False, frame_buffer_property=None):
             )
         self.cuda_graphics_resource = None
         if self.enable_cuda:
-            assert _cuda_enable, "Can not enable cuda rendering pipeline"
+            assert _cuda_enable, "Can not enable cuda rendering pipeline, if you are on Windows, try 'pip install pypiwin32'"
 
             # returned tensor property
             self.cuda_dtype = np.uint8
@@ -203,7 +203,12 @@ def perceive(
             self.cam.reparentTo(original_object)
             self.cam.setHpr(original_hpr)
             self.cam.setPos(original_position)
+        return self._format(ret, to_float)
 
+    def _format(self, ret, to_float):
+        """
+        Format the image to the desired type, float32 or uint8
+        """
         if not to_float:
             return ret.astype(np.uint8, copy=False, order="C")
         else: