Long-duration determinism test (#479)

* Added long-duration determinism test with run time ~5 hours.

.github/workflows/ci-base-tests.yml

@@ -47,5 +47,6 @@ jobs:
             -n auto \
             ${{matrix.tests}} \
             --ignore=./smarts/core/tests/test_smarts_memory_growth.py \
+            --ignore=./smarts/env/tests/test_benchmark.py \
             --ignore=./smarts/env/tests/test_learning.py \
-            --ignore=./smarts/env/tests/test_benchmark.py
+            -k 'not test_long_determinism'

.github/workflows/ci-test-long-determinism.yml

@@ -0,0 +1,42 @@
+name: SMARTS CI Long Determinism
+
+on:
+  schedule:
+    - cron: '0 23 * * 6'
+      # Time is in UTC
+      # Runs at 11.00pm, UTC  , every Saturday
+      # Runs at  6.00pm, UTC-5, every Saturday
+  workflow_dispatch:
+
+env:
+  venv_dir: .venv
+
+jobs:
+  test_learning:
+    runs-on: ubuntu-18.04
+    container: huaweinoah/smarts:v0.4.3-pre
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          ref: develop
+      - name: Setup package
+        run: |
+          /usr/bin/Xorg \
+            -noreset \
+            +extension GLX \
+            +extension RANDR \
+            +extension RENDER \
+            -logfile ./xdummy.log \
+            -config /etc/X11/xorg.conf :1 &
+          cd $GITHUB_WORKSPACE
+          python3.7 -m venv ${{env.venv_dir}}
+          . ${{env.venv_dir}}/bin/activate
+          pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -e .[train,test]
+      - name: Verify long determinism
+        run: |
+          cd $GITHUB_WORKSPACE
+          . ${{env.venv_dir}}/bin/activate
+          make test-long-determinism

.github/workflows/ci-test-memory-growth.yml

@@ -2,10 +2,10 @@ name: SMARTS CI Memory
 
 on:
   schedule:
-    - cron: '0 23 * * 2'
+    - cron: '0 23 * * 4'
       # Time is in UTC
-      # Runs at 11.00pm, UTC  , every Tuesday
-      # Runs at  6.00pm, UTC-5, every Tuesday
+      # Runs at 11.00pm, UTC  , every Thursday
+      # Runs at  6.00pm, UTC-5, every Thursday
   workflow_dispatch:
 
 env:

Dockerfile

@@ -10,66 +10,56 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 # > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK.
 ENV LANG C.UTF-8
 
-RUN echo "Installing dependencies"
-RUN apt-get update --fix-missing && \
-    apt-get install -y \
-    wget
-
+# Install libraries
 RUN apt-get update --fix-missing && \
     apt-get install -y \
         software-properties-common && \
     add-apt-repository -y ppa:deadsnakes/ppa && \
     add-apt-repository -y ppa:sumo/stable && \
     apt-get update && \
     apt-get install -y \
-        wget \
+        libsm6 \
+        libspatialindex-dev \
+        libxext6 \
+        libxrender-dev \
         python3.7 \
         python3.7-dev \
         python3.7-venv \
         sumo \
-        sumo-tools \
         sumo-doc \
-        libspatialindex-dev \
-        libsm6 \
-        libxext6 \
-        libxrender-dev
+        sumo-tools \
+        wget \
+        x11-apps \
+        xserver-xorg-video-dummy && \
+    apt-get autoremove && \
+    rm -rf /var/lib/apt/lists/*
 
 # Update default python version
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1
 
-# Install pip dependencies
+# Install pip
 RUN wget https://bootstrap.pypa.io/get-pip.py -O get-pip.py && \
     python get-pip.py && \
     pip install --upgrade pip
 
-RUN echo "Installing XDummy"
+# Setup DISPLAY
 ENV DISPLAY :1
-
-RUN apt-get install -y \
-    xserver-xorg-video-dummy \
-    x11-apps
-
 # VOLUME /tmp/.X11-unix
-RUN wget -O /etc/X11/xorg.conf http://xpra.org/xorg.conf
-RUN cp /etc/X11/xorg.conf /usr/share/X11/xorg.conf.d/xorg.conf
-
-RUN echo "Cleaning-up"
-RUN apt-get autoremove && \
-    rm -rf /var/lib/apt/lists/*
+RUN wget -O /etc/X11/xorg.conf http://xpra.org/xorg.conf && \
+    cp /etc/X11/xorg.conf /usr/share/X11/xorg.conf.d/xorg.conf
 
-RUN echo "Setup SMARTS Dependencies"
+# Setup SUMO
 ENV SUMO_HOME /usr/share/sumo
 
+# Install requirements.txt
 COPY requirements.txt /tmp/requirements.txt
 RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
+# Copy source files and install SMARTS
 ENV PYTHONPATH=/src
 COPY . /src
-RUN cd /usr/local/lib/python3.7 && \
-    mkdir site-packages && \
-    cd site-packages && \
-    python /src/setup.py develop
 WORKDIR /src
+RUN pip install --no-cache-dir -e .[train,test,dev]
 
 # For Envision
 EXPOSE 8081

Makefile

@@ -8,14 +8,11 @@ test: build-all-scenarios
 		--forked \
 		--dist=loadscope \
 		-n `nproc --ignore 1` \
-		./tests ./smarts/core ./smarts/env ./smarts/contrib ./smarts/sstudio ./envision \
-		--ignore=./smarts/env/tests/test_learning.py \
+		./envision ./smarts/contrib ./smarts/core ./smarts/env ./smarts/sstudio ./tests \
 		--ignore=./smarts/core/tests/test_smarts_memory_growth.py \
-		--ignore=./smarts/env/tests/test_benchmark.py
-
-.PHONY: benchmark
-benchmark: build-all-scenarios
-	pytest -v ./smarts/env/tests/test_benchmark.py
+		--ignore=./smarts/env/tests/test_benchmark.py \
+		--ignore=./smarts/env/tests/test_learning.py \
+		-k 'not test_long_determinism'
 
 .PHONY: test-learning
 test-learning: build-all-scenarios
@@ -30,6 +27,17 @@ test-memory-growth: build-all-scenarios
 		-n `nproc --ignore 1` \
 		./smarts/core/tests/test_smarts_memory_growth.py
 
+.PHONY: test-long-determinism
+test-long-determinism: 
+	scl scenario build --clean scenarios/minicity
+	PYTHONHASHSEED=42 pytest -v \
+		--forked \
+		./smarts/env/tests/test_determinism.py::test_long_determinism
+
+.PHONY: benchmark
+benchmark: build-all-scenarios
+	pytest -v ./smarts/env/tests/test_benchmark.py
+
 .PHONY: test-zoo
 test-zoo: build-all-scenarios
 	cd smarts/zoo/policies && make test

smarts/env/tests/test_determinism.py

@@ -20,8 +20,8 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 import gym
-import pytest
 import numpy as np
+import pytest
 
 # Reference: https://stackoverflow.com/a/53978543/2783780
 try:
@@ -34,21 +34,16 @@
 from smarts.core.controllers import ActionSpaceType
 from smarts.core.utils.episodes import episodes
 
-MAX_STEPS_PER_EPISODE = 20
-EPISODE_COUNT = 3
-CAPTURE_STEP = 5
-
 
-@pytest.fixture
-def agent_spec():
+def agent_spec(max_steps_per_episode):
     return AgentSpec(
         interface=AgentInterface(
             drivable_area_grid_map=True,
             ogm=True,
             rgb=True,
             lidar=True,
             waypoints=True,
-            max_episode_steps=MAX_STEPS_PER_EPISODE,
+            max_episode_steps=max_steps_per_episode,
             debug=True,
             neighborhood_vehicles=True,
             action=ActionSpaceType.Lane,
@@ -57,18 +52,18 @@ def agent_spec():
     )
 
 
-def run(agent_spec, callback):
+def run(agent_spec, callback, scenarios, episode_count, capture_step):
     AGENT_ID = "Agent-007"
     env = gym.make(
         "smarts.env:hiway-v0",
-        scenarios=["scenarios/intersections/2lane"],
+        scenarios=[scenarios],
         agent_specs={AGENT_ID: agent_spec},
         headless=True,
         timestep_sec=0.01,
         seed=42,
     )
     i = 0
-    for episode in episodes(n=EPISODE_COUNT):
+    for episode in episodes(n=episode_count):
         agent = agent_spec.build_agent()
         observations = env.reset()
 
@@ -82,8 +77,8 @@ def run(agent_spec, callback):
 
             episode.record_step(observations, rewards, dones, infos)
 
-            if i % CAPTURE_STEP == 0:
-                callback(rewards, agent_obs, dones, int(i / CAPTURE_STEP))
+            if i % capture_step == 0:
+                callback(rewards, agent_obs, dones, int(i / capture_step))
             i += 1
 
     env.close()
@@ -106,7 +101,7 @@ def axis_dive(now_axis, prev_axis):
             assert now == prev
 
 
-def test_short_length_determinism(agent_spec):
+def determinism(agent_spec, scenarios, episode_count, capture_step):
     rewards_capture = []
     dones_capture = []
     observations_capture = []
@@ -197,5 +192,25 @@ def check_callback(rewards, agent_obs, dones, index):
         # "top_down_rgb",
         axis_dive(agent_obs.top_down_rgb.data, orig_agent_obs.top_down_rgb.data)
 
-    run(agent_spec, capture_callback)
-    run(agent_spec, check_callback)
+    run(agent_spec, capture_callback, scenarios, episode_count, capture_step)
+    run(agent_spec, check_callback, scenarios, episode_count, capture_step)
+
+
+def test_short_determinism():
+    max_steps_per_episode = 20
+    episode_count = 3
+    capture_step = 5
+    scenarios = "scenarios/intersections/2lane"
+    determinism(
+        agent_spec(max_steps_per_episode), scenarios, episode_count, capture_step
+    )
+
+
+def test_long_determinism():
+    max_steps_per_episode = 55000
+    episode_count = 1
+    capture_step = 13750
+    scenarios = "scenarios/intersections/2lane"
+    determinism(
+        agent_spec(max_steps_per_episode), scenarios, episode_count, capture_step
+    )