From 269ebb06c5117671c5d06eb665e9ed9dac3c1318 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 12:23:41 -0400 Subject: [PATCH 01/22] Remove zoo evaluation dependencies --- setup.cfg | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 7305cd7747..84033a48dc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,10 +30,8 @@ install_requires = # numpy>=1.19.5 required for tf 2.4 # numpy<1.24 required for ray (see https://github.com/ray-project/ray/issues/31258) numpy>=1.19.5,<1.24.0 - pandas>=1.3.4 # only used by zoo/evaluation psutil>=5.4.8 pybullet>=3,<4.0 - rich>=11.2.0 Rtree>=0.9.7 sh>=1.14.2 shapely>=2.0.0 @@ -53,7 +51,6 @@ install_requires = # The following are for /smarts/zoo and remote agents protobuf>=3.17.3, <4.0.0 PyYAML>=3.13 - twisted>=21.7.0 [options.packages.find] exclude = From 292488257efca40bf91280c95d9fcb83f27816df Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 12:29:21 -0400 Subject: [PATCH 02/22] Mock imports --- docs/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 9ccdaf876a..9aa26cc457 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -85,10 +85,12 @@ "mdutils", "moviepy", "opendrive2lanelet", + "pandas", "pathos", "PIL", "pynput", "ray", + "rich", "tabulate", "tools", "torch", From 543af04e179723c79b2d37423465630e5dce61f9 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 13:02:22 -0400 Subject: [PATCH 03/22] Replace sh package with subprocess --- cli/zoo.py | 1 - setup.cfg | 28 ++++++++++++------------ smarts/sstudio/generators.py | 41 +++++++++++++++++++++++------------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/cli/zoo.py b/cli/zoo.py index 9f051684e0..e01f2dfd5a 100644 --- a/cli/zoo.py +++ b/cli/zoo.py @@ -25,7 +25,6 @@ from pathlib import Path import click -from rich import print @click.group(name="zoo", help="Build, install, or instantiate workers.") diff --git a/setup.cfg b/setup.cfg index 84033a48dc..671deb66da 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,25 +23,22 @@ install_requires = # tensorboard needs >=41 # 50.0 is broken: https://github.com/pypa/setupatools/issues/2353 setuptools>=41.0.0,!=50.0 - click>=7.1.2 # used in scl - eclipse-sumo>=1.12.0 # sumo - gym>=0.17.3,<=0.19.0 - gymnasium==0.27.0 # numpy>=1.19.5 required for tf 2.4 # numpy<1.24 required for ray (see https://github.com/ray-project/ray/issues/31258) numpy>=1.19.5,<1.24.0 + click>=7.1.2 # used in scl psutil>=5.4.8 - pybullet>=3,<4.0 - Rtree>=0.9.7 - sh>=1.14.2 shapely>=2.0.0 tableprint>=0.9.1 trimesh==3.9.29 # Used for writing .glb files + PyYAML>=3.13 + + eclipse-sumo>=1.12.0 # sumo + gymnasium==0.27.0 + pybullet>=3,<4.0 + Rtree>=0.9.7 visdom>=0.1.8.9 - # The following are for Scenario Studio - yattag>=1.14.0 - # The following is for both SS and Envision - cloudpickle>=1.3.0,<=2.1.0 + yattag>=1.14.0 # for Scenario Studio # The following are for /envision tornado>=5.1.1 websocket-client>=1.2.1 @@ -50,7 +47,10 @@ install_requires = matplotlib>=3.2.2 # The following are for /smarts/zoo and remote agents protobuf>=3.17.3, <4.0.0 - PyYAML>=3.13 + + # The following are planned for removal + gym>=0.17.3,<=0.19.0 + cloudpickle>=1.3.0,<=2.1.0 [options.packages.find] exclude = @@ -111,10 +111,10 @@ torch = torchvision==0.5.0 train = tensorflow>=2.4.0 -opendrive = - opendrive2lanelet>=1.2.1 argoverse = av2>=0.2.1 +opendrive = + opendrive2lanelet>=1.2.1 [aliases] test=pytest diff --git a/smarts/sstudio/generators.py b/smarts/sstudio/generators.py index 83e5c14ca2..c18dc0c8f8 100644 --- a/smarts/sstudio/generators.py +++ b/smarts/sstudio/generators.py @@ -20,10 +20,10 @@ import logging import os import random +import subprocess import tempfile from typing import Optional -import sh from yattag import Doc, indent from smarts.core.road_map import RoadMap @@ -176,22 +176,33 @@ def plan_and_save( log_path = f"{self._log_dir}/{scenario_name}" os.makedirs(log_path, exist_ok=True) - import smarts.core.utils.sumo # Set SUMO_HOME environment variable from smarts.core.utils.sumo import sumolib - duarouter = sh.Command(sumolib.checkBinary("duarouter")) - - # Validates, and runs route planner - duarouter( - unsorted_input=True, - net_file=self.road_network.source, - route_files=trips_path, - output_file=route_path, - seed=seed, - ignore_errors=False, - no_step_log=True, - repair=True, - error_log=f"{log_path}/{name}.log", + duarouter_path = sumolib.checkBinary("duarouter") + subprocess.check_call( + [ + duarouter_path, + "--unsorted-input", + "true", + "--net-file", + self.road_network.source, + "--route-files", + trips_path, + "--output-file", + route_path, + "--seed", + str(seed), + "--ignore-errors", + "false", + "--no-step-log", + "true", + "--repair", + "true", + "--error-log", + f"{log_path}/{name}.log", + ], + stderr=subprocess.DEVNULL, # suppress warnings from duarouter + stdout=subprocess.DEVNULL, # suppress success messages from duarouter ) # Remove the rou.alt.xml file From fdba2c14e58c0b88c2f270d227958bf8b78c4114 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 14:10:56 -0400 Subject: [PATCH 04/22] Extract envision dependencies out of core --- cli/envision.py | 4 ++-- envision/client.py | 9 ++++++++- envision/server.py | 19 ++++++++++++------- setup.cfg | 8 ++++---- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/cli/envision.py b/cli/envision.py index c4ed79cdd9..688419e93d 100644 --- a/cli/envision.py +++ b/cli/envision.py @@ -19,8 +19,6 @@ # THE SOFTWARE. import click -from envision.server import run - @click.group( name="envision", @@ -43,6 +41,8 @@ def envision_cli(): type=float, ) def start_server(port, max_capacity): + from envision.server import run + run(max_capacity_mb=max_capacity, port=port) diff --git a/envision/client.py b/envision/client.py index 9b1189a4fc..be82ffb78b 100644 --- a/envision/client.py +++ b/envision/client.py @@ -31,7 +31,14 @@ from typing import Any, Optional, Union import numpy as np -import websocket + +try: + import websocket +except: + raise ImportError( + "Missing dependencies for Envision. Install them using the command `pip install -e .[envision]` at the source directory." + ) + from envision import types from envision.client_config import EnvisionStateFilter diff --git a/envision/server.py b/envision/server.py index babf0052c9..13e81f30e3 100644 --- a/envision/server.py +++ b/envision/server.py @@ -33,13 +33,18 @@ from pathlib import Path from typing import Dict, List, Optional, Sequence, Union -import ijson -import tornado.gen -import tornado.ioloop -import tornado.iostream -import tornado.web -import tornado.websocket -from tornado.websocket import WebSocketClosedError +try: + import ijson + import tornado.gen + import tornado.ioloop + import tornado.iostream + import tornado.web + import tornado.websocket + from tornado.websocket import WebSocketClosedError +except: + raise ImportError( + "Missing dependencies for Envision. Install them using the command `pip install -e .[envision]` at the source directory." + ) import smarts.core.models from envision.web import dist as web_dist diff --git a/setup.cfg b/setup.cfg index 671deb66da..004dc5541a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,10 +39,6 @@ install_requires = Rtree>=0.9.7 visdom>=0.1.8.9 yattag>=1.14.0 # for Scenario Studio - # The following are for /envision - tornado>=5.1.1 - websocket-client>=1.2.1 - ijson>=3.1.4 # The following are for the /smarts/algorithms matplotlib>=3.2.2 # The following are for /smarts/zoo and remote agents @@ -82,6 +78,10 @@ doc = sphinxcontrib-apidoc>=0.3.0 sphinx-click>=4.4.0 sphinx-rtd-theme>=1.1.1 +envision = + tornado>=5.1.1 + websocket-client>=1.2.1 + ijson>=3.1.4 extras = pynput>=1.7.4 # Used by HumanKeyboardAgent gym = moviepy == 1.0.3 From b854304fdc6960a2cd772b2d03a1e0b4ff459196 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 14:31:52 -0400 Subject: [PATCH 05/22] Move matplotlib out of core --- .github/workflows/ci-base-tests-linux.yml | 2 +- .github/workflows/ci-base-tests-mac.yml | 2 +- cli/waymo.py | 8 ++++++-- setup.cfg | 10 +++++++--- smarts/waymo/waymo_utils.py | 12 +++++++++--- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci-base-tests-linux.yml b/.github/workflows/ci-base-tests-linux.yml index e81a3020e2..c275de66e6 100644 --- a/.github/workflows/ci-base-tests-linux.yml +++ b/.github/workflows/ci-base-tests-linux.yml @@ -30,7 +30,7 @@ jobs: . ${{env.venv_dir}}/bin/activate pip install --upgrade pip pip install wheel==0.38.4 - pip install -e .[camera_obs,opendrive,test,test_notebook,torch,train,gym,argoverse] + pip install -e .[camera_obs,opendrive,test,test_notebook,torch,train,gym,argoverse,envision] if echo ${{matrix.tests}} | grep -q -e "env" -e "examples"; then pip install -e .[rllib]; fi if echo ${{matrix.tests}} | grep -q "/ray"; then pip install -e .[ray]; fi - name: Run smoke tests diff --git a/.github/workflows/ci-base-tests-mac.yml b/.github/workflows/ci-base-tests-mac.yml index 144cebc5be..5a749bc651 100644 --- a/.github/workflows/ci-base-tests-mac.yml +++ b/.github/workflows/ci-base-tests-mac.yml @@ -48,7 +48,7 @@ jobs: pip install --upgrade pip pip install wheel==0.38.4 pip install -r utils/setup/mac_requirements.txt - pip install -e .[camera_obs,opendrive,rllib,test,test_notebook,torch,train,argoverse] + pip install -e .[camera_obs,opendrive,rllib,test,test_notebook,torch,train,argoverse,envision] if echo ${{matrix.tests}} | grep -q -e "/env" -e "/examples"; then pip install -e .[rllib]; fi if echo ${{matrix.tests}} | grep -q "/ray"; then pip install -e .[ray]; fi - name: Run smoke tests diff --git a/cli/waymo.py b/cli/waymo.py index 93fb998ed6..558b8a3102 100644 --- a/cli/waymo.py +++ b/cli/waymo.py @@ -22,8 +22,6 @@ import click import tableprint as tp -from smarts.waymo import waymo_utils - @click.group( name="waymo", @@ -40,6 +38,8 @@ def waymo_cli(): "tfrecord_file", type=click.Path(exists=True), metavar="" ) def overview(tfrecord_file: str): + from smarts.waymo import waymo_utils + scenarios = waymo_utils.get_tfrecord_info(tfrecord_file) rows = [ [k, v["timestamps"], v["vehicles"], v["pedestrians"]] @@ -79,6 +79,8 @@ def preview( animate: bool, label_vehicles: bool, ): + from smarts.waymo import waymo_utils + waymo_utils.plot_scenario(tfrecord_file, scenario_id, animate, label_vehicles) @@ -97,6 +99,8 @@ def export( scenario_id: str, export_folder: str, ): + from smarts.waymo import waymo_utils + scenario_folder = os.path.join(export_folder, scenario_id) if not os.path.exists(scenario_folder): os.makedirs(scenario_folder) diff --git a/setup.cfg b/setup.cfg index 004dc5541a..5307994d61 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,15 +32,14 @@ install_requires = tableprint>=0.9.1 trimesh==3.9.29 # Used for writing .glb files PyYAML>=3.13 + yattag>=1.14.0 # for Scenario Studio + # The following are planned to be made optional eclipse-sumo>=1.12.0 # sumo gymnasium==0.27.0 pybullet>=3,<4.0 Rtree>=0.9.7 visdom>=0.1.8.9 - yattag>=1.14.0 # for Scenario Studio - # The following are for the /smarts/algorithms - matplotlib>=3.2.2 # The following are for /smarts/zoo and remote agents protobuf>=3.17.3, <4.0.0 @@ -61,6 +60,8 @@ console_scripts = diagnostic = py-cpuinfo==9.0.0 mdutils==1.4.0 + gym>=0.17.3,<=0.19.0 + matplotlib>=3.2.2 camera_obs = Panda3D==1.10.9 panda3d-gltf==0.13 @@ -115,6 +116,9 @@ argoverse = av2>=0.2.1 opendrive = opendrive2lanelet>=1.2.1 +waymo = + matplotlib>=3.2.2 + protobuf>=3.17.3, <4.0.0 [aliases] test=pytest diff --git a/smarts/waymo/waymo_utils.py b/smarts/waymo/waymo_utils.py index 0069dfa8df..28e186d665 100644 --- a/smarts/waymo/waymo_utils.py +++ b/smarts/waymo/waymo_utils.py @@ -21,9 +21,15 @@ from typing import Any, Dict, List, Optional, Tuple import numpy as np -from matplotlib import pyplot as plt -from matplotlib.animation import FuncAnimation -from matplotlib.lines import Line2D + +try: + from matplotlib import pyplot as plt + from matplotlib.animation import FuncAnimation + from matplotlib.lines import Line2D +except: + raise ImportError( + "Missing dependencies for Waymo. Install them using the command `pip install -e .[waymo]` at the source directory." + ) from smarts.core.utils.file import read_tfrecord_file From 7c37a2e25e02fddd169843c7118f539f352a87a3 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 14:41:49 -0400 Subject: [PATCH 06/22] Move rtree out of core --- setup.cfg | 4 +++- smarts/core/argoverse_map.py | 4 +--- smarts/core/opendrive_road_network.py | 4 +--- smarts/core/waymo_map.py | 33 +++++++++++++++------------ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/setup.cfg b/setup.cfg index 5307994d61..86ce6bb1bb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,6 @@ install_requires = eclipse-sumo>=1.12.0 # sumo gymnasium==0.27.0 pybullet>=3,<4.0 - Rtree>=0.9.7 visdom>=0.1.8.9 # The following are for /smarts/zoo and remote agents protobuf>=3.17.3, <4.0.0 @@ -114,11 +113,14 @@ train = tensorflow>=2.4.0 argoverse = av2>=0.2.1 + Rtree>=0.9.7 opendrive = opendrive2lanelet>=1.2.1 + Rtree>=0.9.7 waymo = matplotlib>=3.2.2 protobuf>=3.17.3, <4.0.0 + Rtree>=0.9.7 [aliases] test=pytest diff --git a/smarts/core/argoverse_map.py b/smarts/core/argoverse_map.py index aab0d8794b..2b07fa38ba 100644 --- a/smarts/core/argoverse_map.py +++ b/smarts/core/argoverse_map.py @@ -17,7 +17,6 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. - import heapq import logging import random @@ -27,7 +26,6 @@ from typing import Dict, List, Optional, Sequence, Set, Tuple import numpy as np -import rtree from shapely.geometry import Point as SPoint from shapely.geometry import Polygon @@ -45,13 +43,13 @@ from smarts.sstudio.types import MapSpec try: + import rtree from av2.geometry.interpolate import interp_arc from av2.map.lane_segment import LaneMarkType, LaneSegment from av2.map.map_api import ArgoverseStaticMap except: raise ImportError( "You may not have installed the [argoverse] dependencies required for using Argoverse 2 maps with SMARTS. Install it first using the command `pip install -e .[argoverse]` at the source directory." - "" ) diff --git a/smarts/core/opendrive_road_network.py b/smarts/core/opendrive_road_network.py index 37c6fb8727..532b863c66 100644 --- a/smarts/core/opendrive_road_network.py +++ b/smarts/core/opendrive_road_network.py @@ -31,14 +31,13 @@ from typing import Dict, Generator, List, Optional, Sequence, Set, Tuple import numpy as np -import rtree from smarts.core.utils.glb import make_map_glb, make_road_line_glb # pytype: disable=import-error - try: + import rtree from lxml import etree from opendrive2lanelet.opendriveparser.elements.geometry import Line as LineGeometry from opendrive2lanelet.opendriveparser.elements.junction import ( @@ -68,7 +67,6 @@ except ImportError: raise ImportError( "You may not have installed the [opendrive] dependencies required for using the OpenDRIVE maps with SMARTS. Install it first using the command `pip install -e .[opendrive]` at the source directory." - "" ) # pytype: enable=import-error diff --git a/smarts/core/waymo_map.py b/smarts/core/waymo_map.py index c5fbc91d28..d032c17eb6 100644 --- a/smarts/core/waymo_map.py +++ b/smarts/core/waymo_map.py @@ -31,11 +31,23 @@ from typing import Any, Dict, Generator, List, Optional, Sequence, Set, Tuple, Union import numpy as np -import rtree from shapely.geometry import Point as SPoint from shapely.geometry import Polygon +from smarts.core.coordinates import BoundingBox, Heading, Point, Pose, RefLinePoint +from smarts.core.lanepoints import LanePoints, LinkedLanePoint +from smarts.core.road_map import RoadMap, RoadMapWithCaches, Waypoint +from smarts.core.route_cache import RouteWithCache +from smarts.core.utils.file import read_tfrecord_file +from smarts.core.utils.geometry import buffered_shape from smarts.core.utils.glb import make_map_glb, make_road_line_glb +from smarts.core.utils.math import ( + inplace_unwrap, + line_intersect_vectorized, + radians_to_vec, + ray_boundary_intersect, + vec_2d, +) from smarts.sstudio.types import MapSpec from smarts.waymo.waymo_open_dataset.protos import scenario_pb2 from smarts.waymo.waymo_open_dataset.protos.map_pb2 import ( @@ -47,19 +59,12 @@ ) from smarts.waymo.waymo_utils import WaymoDatasetError -from .coordinates import BoundingBox, Heading, Point, Pose, RefLinePoint -from .lanepoints import LanePoints, LinkedLanePoint -from .road_map import RoadMap, RoadMapWithCaches, Waypoint -from .route_cache import RouteWithCache -from .utils.file import read_tfrecord_file -from .utils.geometry import buffered_shape -from .utils.math import ( - inplace_unwrap, - line_intersect_vectorized, - radians_to_vec, - ray_boundary_intersect, - vec_2d, -) +try: + import rtree +except: + raise ImportError( + "You may not have installed the [argoverse] dependencies required for using Argoverse 2 maps with SMARTS. Install it first using the command `pip install -e .[argoverse]` at the source directory." + ) class WaymoMap(RoadMapWithCaches): From a7a360edc8f4140e04af5c36d5c63dad2d79b555 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 15:33:01 -0400 Subject: [PATCH 07/22] Fix CI --- .github/workflows/ci-format.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-format.yml b/.github/workflows/ci-format.yml index a8c01bc6bf..86bbb43092 100644 --- a/.github/workflows/ci-format.yml +++ b/.github/workflows/ci-format.yml @@ -94,7 +94,7 @@ jobs: . ${{env.venv_dir}}/bin/activate pip install --upgrade pip "setuptools<58.3.0" pip install wheel==0.38.4 - pip install .[camera_obs,doc,train,ray] + pip install .[camera_obs,doc,train,ray,envision,argoverse,opendrive,waymo] cd ${GITHUB_WORKSPACE}/docs make html SPHINXOPTS="-W -T -E --keep-going" - name: Check build output @@ -105,4 +105,4 @@ jobs: else echo "Sphinx build failed." exit 1 - fi \ No newline at end of file + fi From b78a23c6bb81f4698d6b079a811b1586895d96df Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 16:06:54 -0400 Subject: [PATCH 08/22] Tweak setup.cfg --- setup.cfg | 6 ++++-- smarts/core/sumo_road_network.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 86ce6bb1bb..69cbc6d098 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,10 +23,10 @@ install_requires = # tensorboard needs >=41 # 50.0 is broken: https://github.com/pypa/setupatools/issues/2353 setuptools>=41.0.0,!=50.0 + click>=7.1.2 # used in scl # numpy>=1.19.5 required for tf 2.4 # numpy<1.24 required for ray (see https://github.com/ray-project/ray/issues/31258) - numpy>=1.19.5,<1.24.0 - click>=7.1.2 # used in scl + numpy>=1.19.5,<1.24.0 psutil>=5.4.8 shapely>=2.0.0 tableprint>=0.9.1 @@ -36,6 +36,7 @@ install_requires = # The following are planned to be made optional eclipse-sumo>=1.12.0 # sumo + Rtree>=0.9.7 # technically optional, but used by sumo internally for performance (see `getNeighboringLanes()`) gymnasium==0.27.0 pybullet>=3,<4.0 visdom>=0.1.8.9 @@ -45,6 +46,7 @@ install_requires = # The following are planned for removal gym>=0.17.3,<=0.19.0 cloudpickle>=1.3.0,<=2.1.0 + scipy [options.packages.find] exclude = diff --git a/smarts/core/sumo_road_network.py b/smarts/core/sumo_road_network.py index b7396b53aa..3d1e41d1b7 100644 --- a/smarts/core/sumo_road_network.py +++ b/smarts/core/sumo_road_network.py @@ -795,7 +795,7 @@ def nearest_lanes( point[1], r=radius, includeJunctions=not include_junctions, - allowFallback=False, + allowFallback=False, # makes this call fail if rtree is not installed ) if not include_junctions: candidate_lanes = [ From 529a5a3d13463764eb14fcb1d5cb4df6153d219d Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 18:07:15 -0400 Subject: [PATCH 09/22] Add warning for missing waymo dependencies --- setup.cfg | 3 +-- smarts/waymo/waymo_open_dataset/protos/__init__.py | 7 +++++++ smarts/waymo/waymo_utils.py | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 69cbc6d098..755a6cb2a4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,8 +40,7 @@ install_requires = gymnasium==0.27.0 pybullet>=3,<4.0 visdom>=0.1.8.9 - # The following are for /smarts/zoo and remote agents - protobuf>=3.17.3, <4.0.0 + protobuf>=3.17.3, <4.0.0 # for /smarts/zoo and remote agents # The following are planned for removal gym>=0.17.3,<=0.19.0 diff --git a/smarts/waymo/waymo_open_dataset/protos/__init__.py b/smarts/waymo/waymo_open_dataset/protos/__init__.py index e2c6d557d2..e7c14814ef 100644 --- a/smarts/waymo/waymo_open_dataset/protos/__init__.py +++ b/smarts/waymo/waymo_open_dataset/protos/__init__.py @@ -12,3 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== + +try: + import google.protobuf +except: + raise ImportError( + "You may not have installed the [waymo] dependencies required for using Argoverse 2 maps with SMARTS. Install it first using the command `pip install -e .[waymo]` at the source directory." + ) \ No newline at end of file diff --git a/smarts/waymo/waymo_utils.py b/smarts/waymo/waymo_utils.py index 28e186d665..5de6a7bd47 100644 --- a/smarts/waymo/waymo_utils.py +++ b/smarts/waymo/waymo_utils.py @@ -32,6 +32,8 @@ ) from smarts.core.utils.file import read_tfrecord_file +from smarts.core.waymo_map import WaymoMap +from smarts.waymo.waymo_open_dataset.protos import scenario_pb2 class WaymoDatasetError(Exception): @@ -272,7 +274,6 @@ def _plot_trajectories( def get_tfrecord_info(tfrecord_file: str) -> Dict[str, Dict[str, Any]]: """Extract info about each scenario in the TFRecord file.""" - from smarts.waymo.waymo_open_dataset.protos import scenario_pb2 scenarios = dict() records = read_tfrecord_file(tfrecord_file) @@ -304,7 +305,6 @@ def plot_scenario( ): """Plot the map features of a Waymo scenario, and optionally plot/animate the vehicle trajectories.""" - from smarts.core.waymo_map import WaymoMap source = f"{tfrecord_file}#{scenario_id}" scenario = WaymoMap.parse_source_to_scenario(source) From 979b4ba166c6802f89318170892756e5bf9cbc67 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 18:15:17 -0400 Subject: [PATCH 10/22] Update changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8045e7000f..3ab359f1a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,10 +13,13 @@ Copy and pasting the git commit messages is __NOT__ enough. ### Changed - Changed waypoints in sumo maps to use more incoming lanes into junctions. - Increased the cutoff radius for filtering out waypoints that are too far away in junctions in sumo maps. +- Made Envision dependencies optional. Use `pip install -e .[envision]` to install them. +- Made Waymo dependencies optional. Use `pip install -e .[waymo]` to install them. ### Deprecated ### Fixed - Fixed implementations of `RoadMap.waypoint_paths()` to ensure that the result is never empty. ### Removed +- Removed the following dependencies from smarts: `pandas`, `rich`, `twisted`, `sh`. ### Security ## [1.1.0] # 2023-04-28 From 7c493a217eec3bb5336fa9c6ebaaeb9297dc53fe Mon Sep 17 00:00:00 2001 From: Saul Field Date: Wed, 3 May 2023 18:21:50 -0400 Subject: [PATCH 11/22] Fix circular import --- smarts/waymo/waymo_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smarts/waymo/waymo_utils.py b/smarts/waymo/waymo_utils.py index 5de6a7bd47..673a9ca140 100644 --- a/smarts/waymo/waymo_utils.py +++ b/smarts/waymo/waymo_utils.py @@ -32,7 +32,6 @@ ) from smarts.core.utils.file import read_tfrecord_file -from smarts.core.waymo_map import WaymoMap from smarts.waymo.waymo_open_dataset.protos import scenario_pb2 @@ -305,6 +304,7 @@ def plot_scenario( ): """Plot the map features of a Waymo scenario, and optionally plot/animate the vehicle trajectories.""" + from smarts.core.waymo_map import WaymoMap source = f"{tfrecord_file}#{scenario_id}" scenario = WaymoMap.parse_source_to_scenario(source) From d281f26ea1bf0c6551e769a2a5c3041eeaeb367b Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 12:42:10 -0400 Subject: [PATCH 12/22] Make websocket import dynamic for envision client --- envision/client.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/envision/client.py b/envision/client.py index be82ffb78b..6c4a79c8ab 100644 --- a/envision/client.py +++ b/envision/client.py @@ -32,14 +32,6 @@ import numpy as np -try: - import websocket -except: - raise ImportError( - "Missing dependencies for Envision. Install them using the command `pip install -e .[envision]` at the source directory." - ) - - from envision import types from envision.client_config import EnvisionStateFilter from envision.data_formatter import EnvisionDataFormatter, EnvisionDataFormatterArgs @@ -274,6 +266,13 @@ def on_open(ws): ws.close() def run_socket(endpoint, wait_between_retries): + try: + import websocket + except: + raise ImportError( + "Missing dependencies for Envision. Install them using the command `pip install -e .[envision]` at the source directory." + ) + nonlocal connection_established tries = 1 while True: From 571e58a326628ca1cbb9f40c317f32bafe2354ab Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 12:44:34 -0400 Subject: [PATCH 13/22] Update docs --- docs/sim/visualization.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/sim/visualization.rst b/docs/sim/visualization.rst index 167195195e..e9834bb78d 100644 --- a/docs/sim/visualization.rst +++ b/docs/sim/visualization.rst @@ -18,6 +18,8 @@ An example is shown below to run SMARTS with Envision. .. code-block:: bash $ cd /SMARTS + # Install the dependencies for Envision + $ pip install smarts[envision] # Build scenarios/sumo/loop $ scl scenario build --clean scenarios/sumo/loop # Run the chase_via_points.py example with the loop scenario From dc15cf895adf6b8e17d09d39bda605c0721c1785 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 13:35:07 -0400 Subject: [PATCH 14/22] Add [all] extra and CI test for package conflicts --- .github/workflows/ci-format.yml | 20 ++++++++++++++++++++ setup.cfg | 22 +++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-format.yml b/.github/workflows/ci-format.yml index 86bbb43092..0a23b8f510 100644 --- a/.github/workflows/ci-format.yml +++ b/.github/workflows/ci-format.yml @@ -106,3 +106,23 @@ jobs: echo "Sphinx build failed." exit 1 fi + + test-package-conflicts: + runs-on: ubuntu-20.04 + if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository + container: huaweinoah/smarts:v0.6.1-minimal + steps: + - name: Add safe directory + run: git config --global --add safe.directory $GITHUB_WORKSPACE + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Install SMARTS with all extras and check for conflicts + run: | + python3.8 -m venv ${{env.venv_dir}} + . ${{env.venv_dir}}/bin/activate + pip install --upgrade pip "setuptools<58.3.0" + pip install wheel==0.38.4 + pip install .[all] + pip check diff --git a/setup.cfg b/setup.cfg index 9dcd0f17f2..493a1afa4f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -90,7 +90,7 @@ remote_agent = rllib = opencv-python==4.1.2.30 opencv-python-headless==4.1.2.30 - ray[rllib]==1.4.0 + ray[rllib]~=2.2.0 ray = ray~=2.2.0 ros = @@ -123,6 +123,26 @@ waymo = Rtree>=0.9.7 visdom = visdom>=0.1.8.9 +all = + %(diagnostic)s + %(camera_obs)s + %(dev)s + %(doc)s + %(envision)s + %(extras)s + %(gym)s + %(remote_agent)s + %(rllib)s + %(ray)s + %(ros)s + %(test)s + %(test_notebook)s + %(torch)s + %(train)s + %(argoverse)s + %(opendrive)s + %(waymo)s + %(visdom)s [aliases] test=pytest From f74bdaa20b4d79b376fc9f01e7fc4437aed78bcc Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 13:37:27 -0400 Subject: [PATCH 15/22] Fix package conflict and remove redundant test --- setup.cfg | 4 +-- smarts/core/tests/test_argoverse.py | 49 ----------------------------- 2 files changed, 2 insertions(+), 51 deletions(-) delete mode 100644 smarts/core/tests/test_argoverse.py diff --git a/setup.cfg b/setup.cfg index 493a1afa4f..0f11d1992e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ install_requires = protobuf>=3.17.3,<4.0.0 # for /smarts/zoo and remote agents # The following are planned for removal - gym>=0.17.3,<=0.19.0 + gym>=0.17.3,<=0.21.0 cloudpickle>=1.3.0,<=2.1.0 scipy @@ -59,7 +59,7 @@ console_scripts = diagnostic = py-cpuinfo==9.0.0 mdutils==1.4.0 - gym>=0.17.3,<=0.19.0 + gym>=0.17.3,<=0.21.0 matplotlib>=3.2.2 camera_obs = Panda3D==1.10.9 diff --git a/smarts/core/tests/test_argoverse.py b/smarts/core/tests/test_argoverse.py deleted file mode 100644 index 3c59da96d6..0000000000 --- a/smarts/core/tests/test_argoverse.py +++ /dev/null @@ -1,49 +0,0 @@ -# MIT License -# -# Copyright (C) 2022. Huawei Technologies Co., Ltd. All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. - -import subprocess -import sys - - -def test_argoverse_package_compatibility(): - """Test that the av2 package is installed and has no conflicts with the rest of the SMARTS dependencies.""" - - try: - import av2 - except ModuleNotFoundError: - print( - "The argoverse dependencies must be installed using `pip install -e .[argoverse]`" - ) - raise - - try: - subprocess.check_call( - [ - sys.executable, - "-m", - "pip", - "check", - ] - ) - except subprocess.CalledProcessError: - print("There is an incompatibility with the installed packages") - raise From c7ac1c906f238a8489adb7f104e41fd9cf4d9a6c Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 13:43:18 -0400 Subject: [PATCH 16/22] Update CI --- .github/workflows/ci-format.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-format.yml b/.github/workflows/ci-format.yml index 0a23b8f510..b9b65ac1c9 100644 --- a/.github/workflows/ci-format.yml +++ b/.github/workflows/ci-format.yml @@ -112,6 +112,10 @@ jobs: if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository container: huaweinoah/smarts:v0.6.1-minimal steps: + - name: Install packages + run: | + apt-get update + apt-get install python3-dev - name: Add safe directory run: git config --global --add safe.directory $GITHUB_WORKSPACE - name: Checkout From b03e1e7dd8cc001586bd85f9e81425a6a5eca8bc Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 13:46:36 -0400 Subject: [PATCH 17/22] Fix CI --- .github/workflows/ci-format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-format.yml b/.github/workflows/ci-format.yml index b9b65ac1c9..d9132cbea0 100644 --- a/.github/workflows/ci-format.yml +++ b/.github/workflows/ci-format.yml @@ -115,7 +115,7 @@ jobs: - name: Install packages run: | apt-get update - apt-get install python3-dev + apt-get install -y python3-dev - name: Add safe directory run: git config --global --add safe.directory $GITHUB_WORKSPACE - name: Checkout From 9ae3f393673fd94567265f07015018357442cdaa Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 13:55:11 -0400 Subject: [PATCH 18/22] Fix CI --- .github/workflows/ci-format.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-format.yml b/.github/workflows/ci-format.yml index d9132cbea0..304b44ddbc 100644 --- a/.github/workflows/ci-format.yml +++ b/.github/workflows/ci-format.yml @@ -115,7 +115,7 @@ jobs: - name: Install packages run: | apt-get update - apt-get install -y python3-dev + apt-get install -y python3.8-dev - name: Add safe directory run: git config --global --add safe.directory $GITHUB_WORKSPACE - name: Checkout From 1dea25419e03d40b62acb2032d9fb851e0c26511 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 14:36:32 -0400 Subject: [PATCH 19/22] Fix tests --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 0f11d1992e..66d4059dc2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -90,7 +90,7 @@ remote_agent = rllib = opencv-python==4.1.2.30 opencv-python-headless==4.1.2.30 - ray[rllib]~=2.2.0 + ray[rllib]==1.4.0 ray = ray~=2.2.0 ros = @@ -133,7 +133,7 @@ all = %(gym)s %(remote_agent)s %(rllib)s - %(ray)s + # %(ray)s # incompatible with [rllib] for now %(ros)s %(test)s %(test_notebook)s From 4dff58116ca916713a7caedf2ce8ebc4b53888a9 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 15:12:05 -0400 Subject: [PATCH 20/22] Revert gym version and fix error messages --- setup.cfg | 4 ++-- smarts/core/argoverse_map.py | 2 +- smarts/core/default_map_builder.py | 2 +- smarts/core/waymo_map.py | 2 +- smarts/sstudio/genhistories.py | 2 +- smarts/waymo/waymo_open_dataset/protos/__init__.py | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/setup.cfg b/setup.cfg index 66d4059dc2..5c6386fe9b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ install_requires = protobuf>=3.17.3,<4.0.0 # for /smarts/zoo and remote agents # The following are planned for removal - gym>=0.17.3,<=0.21.0 + gym>=0.17.3,<=0.19.0 cloudpickle>=1.3.0,<=2.1.0 scipy @@ -59,7 +59,7 @@ console_scripts = diagnostic = py-cpuinfo==9.0.0 mdutils==1.4.0 - gym>=0.17.3,<=0.21.0 + gym>=0.17.3,<=0.19.0 matplotlib>=3.2.2 camera_obs = Panda3D==1.10.9 diff --git a/smarts/core/argoverse_map.py b/smarts/core/argoverse_map.py index 2b07fa38ba..4ec151adae 100644 --- a/smarts/core/argoverse_map.py +++ b/smarts/core/argoverse_map.py @@ -49,7 +49,7 @@ from av2.map.map_api import ArgoverseStaticMap except: raise ImportError( - "You may not have installed the [argoverse] dependencies required for using Argoverse 2 maps with SMARTS. Install it first using the command `pip install -e .[argoverse]` at the source directory." + "Missing dependencies for Argoverse. Install them using the command `pip install -e .[argoverse]` at the source directory." ) diff --git a/smarts/core/default_map_builder.py b/smarts/core/default_map_builder.py index 6274a82b11..1395b37c54 100644 --- a/smarts/core/default_map_builder.py +++ b/smarts/core/default_map_builder.py @@ -142,7 +142,7 @@ def get_road_map(map_spec) -> Tuple[Optional[RoadMap], Optional[str]]: except (ImportError, ModuleNotFoundError): print(sys.exc_info()) print( - "You may not have installed the [argoverse] dependencies required to build and use Argoverse scenarios. Install them first using the command `pip install -e .[argoverse]` at the source directory." + "Missing dependencies for Argoverse. Install them using the command `pip install -e .[argoverse]` at the source directory." ) return None, None map_class = ArgoverseMap diff --git a/smarts/core/waymo_map.py b/smarts/core/waymo_map.py index d032c17eb6..39ba80536c 100644 --- a/smarts/core/waymo_map.py +++ b/smarts/core/waymo_map.py @@ -63,7 +63,7 @@ import rtree except: raise ImportError( - "You may not have installed the [argoverse] dependencies required for using Argoverse 2 maps with SMARTS. Install it first using the command `pip install -e .[argoverse]` at the source directory." + "Missing dependencies for Waymo. Install them using the command `pip install -e .[waymo]` at the source directory." ) diff --git a/smarts/sstudio/genhistories.py b/smarts/sstudio/genhistories.py index b8b66e205a..128ea60526 100644 --- a/smarts/sstudio/genhistories.py +++ b/smarts/sstudio/genhistories.py @@ -1001,7 +1001,7 @@ def rows(self) -> Generator[Dict, None, None]: # pytype: enable=import-error except ImportError: print( - "You may not have installed the [argoverse] dependencies required to use the Argoverse 2 replay simulation. Install them first using the command `pip install -e .[argoverse]` at the source directory." + "Missing dependencies for Argoverse. Install them using the command `pip install -e .[argoverse]` at the source directory." ) ALLOWED_TYPES = frozenset( diff --git a/smarts/waymo/waymo_open_dataset/protos/__init__.py b/smarts/waymo/waymo_open_dataset/protos/__init__.py index e7c14814ef..1badc9d0ad 100644 --- a/smarts/waymo/waymo_open_dataset/protos/__init__.py +++ b/smarts/waymo/waymo_open_dataset/protos/__init__.py @@ -17,5 +17,5 @@ import google.protobuf except: raise ImportError( - "You may not have installed the [waymo] dependencies required for using Argoverse 2 maps with SMARTS. Install it first using the command `pip install -e .[waymo]` at the source directory." - ) \ No newline at end of file + "Missing dependencies for Waymo. Install them using the command `pip install -e .[waymo]` at the source directory." + ) From 62b72a067068391eaedf537d99632b62c5e050f0 Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 15:20:10 -0400 Subject: [PATCH 21/22] Fix types --- smarts/core/default_map_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smarts/core/default_map_builder.py b/smarts/core/default_map_builder.py index 1395b37c54..bbed479db1 100644 --- a/smarts/core/default_map_builder.py +++ b/smarts/core/default_map_builder.py @@ -33,7 +33,7 @@ def _cache_result(map_spec, road_map, road_map_hash: str): from smarts.sstudio.types import MapSpec class _RoadMapInfo(NamedTuple): - map_spec: MapSpec + map_spec: MapSpec # pytype: disable=invalid-annotation obj: RoadMap map_hash: str From 2dacc2bdd8f3739ac9f754e8be7eba5e979cb97f Mon Sep 17 00:00:00 2001 From: Saul Field Date: Thu, 4 May 2023 17:19:17 -0400 Subject: [PATCH 22/22] Add back the twisted package. Update changelog --- CHANGELOG.md | 1 + setup.cfg | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2b9ad678c..4b075f5991 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Copy and pasting the git commit messages is __NOT__ enough. ## [Unreleased] ### Added - `visdom` can now be configured through the engine.ini configuration file `visdom:enabled`, `visdom:hostname`, and `visdom:port` (environment variables `SMARTS_VISDOM_ENABLED`, `SMARTS_VISDOM_HOSTNAME`, `SMARTS_VISDOM_PORT`.) +- Added an install extra that installs the requirements for all optional modules. Use `pip install .[all]`. ### Changed - Changed waypoints in sumo maps to use more incoming lanes into junctions. - Increased the cutoff radius for filtering out waypoints that are too far away in junctions in sumo maps. diff --git a/setup.cfg b/setup.cfg index 5c6386fe9b..b3fcef8abe 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,9 +30,10 @@ install_requires = psutil>=5.4.8 shapely>=2.0.0 tableprint>=0.9.1 - trimesh==3.9.29 # Used for writing .glb files + trimesh==3.9.29 # for writing .glb files + yattag>=1.14.0 # for scenario studio PyYAML>=3.13 - yattag>=1.14.0 # for Scenario Studio + twisted>=21.7.0 # for scenario requirements.txt files # The following are planned to be made optional eclipse-sumo>=1.12.0 # sumo