Add Wokwi CI - esp32-simtest.yaml

runs the esp32 tests across 7 targets/models and 3 esp-idf versions. Includes the wifi_example test. A limited test scope (latest esp-idf only) is the default unless the PR title contains "full_sim_test" or the last commit message contains it in title or body. Signed-off-by: Peter M <[email protected]>
atomvm · Oct 10, 2024 · 4e96b79 · 4e96b79
1 parent 9be487f
commit 4e96b79
Show file tree

Hide file tree

Showing 25 changed files with 607 additions and 6 deletions.
diff --git a/.github/workflows/esp32-build.yaml b/.github/workflows/esp32-build.yaml
@@ -40,7 +40,7 @@ jobs:
         idf-version:
          - 'v5.0.7'
          - 'v5.1.4'
-         - 'v5.2.2'
+         - 'v5.2.3'
          - 'v5.3.1'
 
         exclude:
@@ -147,4 +147,4 @@ jobs:
         set -e
         . $IDF_PATH/export.sh
         export PATH=/opt/qemu/bin:${PATH}
-        pytest --target=${{matrix.esp-idf-target}} --embedded-services=idf,qemu -s
+        pytest -k 'test_atomvm' --target=${{matrix.esp-idf-target}} --embedded-services=idf,qemu -s
diff --git a/.github/workflows/esp32-simtest.yaml b/.github/workflows/esp32-simtest.yaml
@@ -0,0 +1,111 @@
+#
+#  Copyright 2024 Davide Bettio <[email protected]>
+#
+#  SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
+#
+
+name: ESP32 Sim test
+
+on:
+  push:
+    paths:
+      - ".github/workflows/esp32-simtest.yaml"
+      - "CMakeLists.txt"
+      - "libs/**"
+      - "src/platforms/esp32/**"
+      - "src/platforms/esp32/**/**"
+      - "src/libAtomVM/**"
+      - "tools/packbeam/**"
+  pull_request:
+    paths:
+      - ".github/workflows/esp32-simtest.yaml"
+      - "src/platforms/esp32/**"
+      - "src/platforms/esp32/**/**"
+      - "src/libAtomVM/**"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref != 'refs/heads/main' && github.ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  cli_token:
+    name: WOKWI_CLI_TOKEN presence
+    runs-on: ubuntu-latest
+    outputs:
+      token_check: ${{ steps.token_check.outputs.should-run }}
+
+    steps:
+      - name: Mark esp-sim-test job as 'to be run'
+        id: token_check
+        env:
+          wokwi_secret: ${{ secrets.WOKWI_CLI_TOKEN }}
+        run: |
+          if [${{ env.wokwi_secret }} == ''];
+          then
+            echo "WOKWI_CLI_TOKEN not found"
+
+          else
+            echo "WOKWI_CLI_TOKEN found continuing"
+            echo "should-run=true" >> $GITHUB_OUTPUT
+
+          fi
+  esp-sim-test:
+    needs: cli_token
+    runs-on: ubuntu-latest
+    if: needs.cli_token.outputs.token_check == 'true'
+    container: espressif/idf:${{ matrix.idf-version }}
+    strategy:
+      fail-fast: false
+      # focus on device diversity.
+      matrix:
+        esp-idf-target: ["esp32", "esp32s2", "esp32s3", "esp32c3", "esp32c6"]
+        idf-version: ${{ ((contains(github.event.head_commit.message, 'full_sim_test')||contains(github.event.pull_request.title, 'full_sim_test')) &&  fromJSON('["v5.1.4", "v5.2.3", "v5.3.1"]')) || fromJSON('["v5.3.1"]') }}
+        include:
+          - esp-idf-target: "esp32p4"
+            idf-version: "v5.3.1"
+          - esp-idf-target: "esp32h2"
+            idf-version: "v5.3.1"
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Install dependencies to build host AtomVM
+        run: |
+          set -eu
+          apt update
+          DEBIAN_FRONTEND=noninteractive apt install -y -q \
+              doxygen erlang-base erlang-dialyzer \
+              libglib2.0-0 libpixman-1-0 \
+              gcc g++ zlib1g-dev libsdl2-2.0-0 libslirp0 libmbedtls-dev
+
+      - name: Install the Wokwi CLI
+        run: curl -L https://wokwi.com/ci/install.sh | sh
+
+      - name: Install pytest and pytest-embedded plugins
+        run: |
+          set -e
+          . $IDF_PATH/export.sh
+          pip install pytest==8.3.3 \
+              pytest-embedded==1.11.5 \
+              pytest-embedded-idf==1.11.5 \
+              pytest-embedded-qemu==1.11.5 \
+              pytest-embedded-wokwi==1.11.5
+
+      - name: Set SDKCONFIG_DEFAULTS and Build ESP32-sim tests using idf.py
+        working-directory: ./src/platforms/esp32/test/
+        run: |
+          set -e
+          . $IDF_PATH/export.sh
+          idf.py -DSDKCONFIG_DEFAULTS='sdkconfig.ci.wokwi' set-target ${{matrix.esp-idf-target}}
+          idf.py build
+
+      - name: Run ESP32-sim tests using Wokwi CI
+        working-directory: ./src/platforms/esp32/test/
+        env:
+          WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
+        timeout-minutes: 10
+        run: |
+          set -e
+          . $IDF_PATH/export.sh
+          pytest -k 'test_atomvm_sim' --embedded-services=idf,wokwi --wokwi-timeout=240000 --target=${{ matrix.esp-idf-target }} --wokwi-diagram=sim_boards/diagram.${{ matrix.esp-idf-target }}.json -s
diff --git a/src/platforms/esp32/test/README.md b/src/platforms/esp32/test/README.md
@@ -0,0 +1,69 @@
+<!--
+ Copyright 2024 Davide Bettio <[email protected]>
+
+ SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
+-->
+
+# AtomVM emulator/simulator tests
+
+AtomVM provides two paths for testing "on device", the locally run QEMU emulator or the remote Wokwi CI simulator.
+
+# QEMU emulator testing
+
+Instructions for running the tests [on QEMU are documented here](https://www.atomvm.net/doc/main/build-instructions.html#running-tests-for-esp32).
+
+# Wokwi CI simulator testing
+
+Wokwi CI is a commercial cloud CI, see [wokwi-ci/getting-started](https://docs.wokwi.com/wokwi-ci/getting-started), running it locally requires you to obtain a `WOKWI_CLI_TOKEN` [Get token](https://wokwi.com/dashboard/ci) and usage fees may apply in the future - AtomVM uses it through the [pytest-embedded-wokwi](https://github.com/espressif/pytest-embedded/tree/main/pytest-embedded-wokwi) integration.
+
+## Github CI/Actions
+
+The `WOKWI_CLI_TOKEN` needs to be set in your `Repository secrets` Settings -> Actions secrets and variables.
+
+## Installing prerequisites
+
+1. The Wokwi CLI needs to be installed:
+
+   ```shell
+   curl -L https://wokwi.com/ci/install.sh | sh
+   ```
+
+   Or [alternative installation methods here](https://docs.wokwi.com/wokwi-ci/getting-started#cli-installation).
+
+2. `WOKWI_CLI_TOKEN` needs to be set in your enviroment variables:
+
+   ```shell
+   export WOKWI_CLI_TOKEN="your-api-key"
+   ```
+
+3. A recent pytest, and pytest-embedded must be installed:
+
+   ```shell
+   $ pip install pytest==8.3.3 \
+   pytest-embedded==1.11.5 \
+   pytest-embedded-serial-esp==1.11.5 \
+   pytest-embedded-idf==1.11.5 \
+   pytest-embedded-wokwi==1.11.5
+   ```
+
+4. The ESP-IDF build environment must be installed and available:
+
+   ```shell
+   get_idf
+   ```
+
+## Compiling for and running Wokwi CI
+
+1. We need to use a special sdkconfig (`sdkconfig.ci.wokwi`) different from the QEMU one. So we set `IDF_TARGET`, and run `idf.py -DSDKCONFIG_DEFAULTS=sdkconfig.ci.wokwi set-target ${IDF_TARGET}`:
+
+   ```shell
+   export IDF_TARGET=esp32 && idf.py -DSDKCONFIG_DEFAULTS=sdkconfig.ci.wokwi set-target ${IDF_TARGET}
+   ```
+
+2. Wokwi CI uses a `diagram.json`, to describe the device used (specific board, pin connections, sensors, sd card etc). 'diagram.json' files are available for each target in the sim_boards folder.
+
+3. Now we run `idf.py build` and run the CI:
+
+   ```shell
+   idf.py build -DSDKCONFIG_DEFAULTS='sdkconfig.ci.wokwi' && pytest -k 'test_atomvm_sim' --embedded-services=idf,wokwi --wokwi-timeout=90000 --target=${IDF_TARGET} --wokwi-diagram=sim_boards/diagram.${IDF_TARGET}.json -s -W ignore::DeprecationWarning
+   ```
diff --git a/src/platforms/esp32/test/main/test_erl_sources/CMakeLists.txt b/src/platforms/esp32/test/main/test_erl_sources/CMakeLists.txt
@@ -39,6 +39,7 @@ endfunction()
 
 compile_erlang(test_esp_partition)
 compile_erlang(test_file)
+compile_erlang(test_wifi_example)
 compile_erlang(test_list_to_binary)
 compile_erlang(test_md5)
 compile_erlang(test_crypto)
@@ -59,6 +60,7 @@ add_custom_command(
         HostAtomVM-prefix/src/HostAtomVM-build/libs/atomvmlib.avm
         test_esp_partition.beam
         test_file.beam
+        test_wifi_example.beam
         test_list_to_binary.beam
         test_md5.beam
         test_crypto.beam
@@ -75,6 +77,7 @@ add_custom_command(
     DEPENDS
         HostAtomVM
         "${CMAKE_CURRENT_BINARY_DIR}/test_esp_partition.beam"
+        "${CMAKE_CURRENT_BINARY_DIR}/test_wifi_example.beam"
         "${CMAKE_CURRENT_BINARY_DIR}/test_file.beam"
         "${CMAKE_CURRENT_BINARY_DIR}/test_list_to_binary.beam"
         "${CMAKE_CURRENT_BINARY_DIR}/test_md5.beam"

diff --git a/src/platforms/esp32/test/main/test_erl_sources/test_wifi_example.erl b/src/platforms/esp32/test/main/test_erl_sources/test_wifi_example.erl
@@ -0,0 +1,114 @@
+%
+% This file is part of AtomVM.
+%
+% Copyright 2020 Fred Dushin <[email protected]>
+%
+% Licensed under the Apache License, Version 2.0 (the "License");
+% you may not use this file except in compliance with the License.
+% You may obtain a copy of the License at
+%
+%    http://www.apache.org/licenses/LICENSE-2.0
+%
+% Unless required by applicable law or agreed to in writing, software
+% distributed under the License is distributed on an "AS IS" BASIS,
+% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+% See the License for the specific language governing permissions and
+% limitations under the License.
+%
+% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later
+%
+
+-module(test_wifi_example).
+
+-export([start/0]).
+
+start() ->
+    case verify_platform(atomvm:platform()) of
+        ok ->
+            start_network(),
+            loop(0);
+        Error ->
+            Error
+    end.
+
+start_network() ->
+    Config = [
+        {ap, [
+            {ap_started, fun ap_started/0},
+            {sta_connected, fun sta_connected/1},
+            {sta_ip_assigned, fun sta_ip_assigned/1},
+            {sta_disconnected, fun sta_disconnected/1}
+            | []
+        ]},
+        {sta, [
+            {connected, fun connected/0},
+            {got_ip, fun got_ip/1},
+            {disconnected, fun disconnected/0}
+            | [
+                {dhcp_hostname, "my_device_name"},
+                {ssid, "Wokwi-GUEST"},
+                {psk, ""}
+            ]
+        ]},
+        {sntp, [
+            {host, "time.aws.com"},
+            {synchronized, fun sntp_synchronized/1}
+        ]}
+    ],
+    case network:start(Config) of
+        {ok, _Pid} ->
+            io:format("Network started.~n");
+        Error ->
+            Error
+    end.
+
+ap_started() ->
+    io:format("AP started.~n").
+
+sta_connected(Mac) ->
+    io:format("STA connected with mac ~p~n", [Mac]).
+
+sta_disconnected(Mac) ->
+    io:format("STA disconnected with mac ~p~n", [Mac]).
+
+sta_ip_assigned(Address) ->
+    io:format("STA assigned address ~p~n", [Address]).
+
+connected() ->
+    io:format("STA connected.~n").
+
+got_ip(IpInfo) ->
+    io:format("Got IP: ~p.~n", [IpInfo]).
+
+disconnected() ->
+    io:format("STA disconnected.~n").
+
+sntp_synchronized({TVSec, TVUsec}) ->
+    io:format("Synchronized time with SNTP server. TVSec=~p TVUsec=~p~n", [TVSec, TVUsec]).
+
+verify_platform(esp32) ->
+    ok;
+verify_platform(Platform) ->
+    {error, {unsupported_platform, Platform}}.
+
+loop(Count) when Count =:= 40 ->
+    io:format("Never got SNTP.~n"),
+    tests(),
+    network:stop();
+loop(Count) ->
+    timer:sleep(500),
+    {{Year, Month, Day}, {Hour, Minute, Second}} = erlang:universaltime(),
+    io:format("Date: ~p/~p/~p ~p:~p:~p (~pms)~n", [
+        Year, Month, Day, Hour, Minute, Second, erlang:system_time(millisecond)
+    ]),
+    case Year of
+        1970 ->
+            loop(Count + 1);
+        _ ->
+            io:format("Got SNTP.~n"),
+            tests(),
+            network:stop()
+    end.
+tests() ->
+    ok = test_net:start(),
+    io:format("test_net OK.~n").