wip

.github/workflows/build.yml

@@ -5,15 +5,47 @@ on:
       - dasharo
 
 jobs:
-  build_novacustom:
+#  build_novacustom:
+#    runs-on: ubuntu-22.04
+#    container:
+#      image: coreboot/coreboot-sdk:2021-09-23_b0d87f753c
+#      options: --user 1001
+#    strategy:
+#      matrix:
+#        vendor: [ novacustom ]
+#        model: [ nv4x_adl, ns5x_adl, nv4x_tgl, ns5x_tgl ]
+#    steps:
+#      - name: Checkout repository
+#        uses: actions/checkout@v3
+#        with:
+#          # Checkout pull request HEAD commit instead of merge commit
+#          # See: https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit
+#          ref: ${{ github.event.pull_request.head.sha }}
+#          # Fetch complete history
+#          fetch-depth: 0
+#      - name: Checkout all submodules
+#        run: git submodule update --init --recursive --checkout
+#      - name: Build Dasharo
+#        run: |
+#          cp configs/config.${{ matrix.vendor }}_${{ matrix.model }} .config
+#          make olddefconfig
+#          make
+#      - name: Save artifacts
+#        uses: actions/upload-artifact@v2
+#        with:
+#          name: "dasharo-${{ matrix.vendor }}-${{ matrix.model }}-${{ matrix.build }}"
+#          path: |
+#            build/coreboot.rom
+#          retention-days: 30
+  build_msi:
     runs-on: ubuntu-22.04
     container:
       image: coreboot/coreboot-sdk:2021-09-23_b0d87f753c
       options: --user 1001
     strategy:
       matrix:
-        vendor: [ novacustom ]
-        model: [ nv4x_adl, ns5x_adl, nv4x_tgl, ns5x_tgl ]
+        vendor: [ msi ]
+        model: [ ms7d25_ddr5 ]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -37,66 +69,83 @@ jobs:
           path: |
             build/coreboot.rom
           retention-days: 30
-  build_msi:
-    runs-on: ubuntu-22.04
-    container:
-      image: coreboot/coreboot-sdk:2021-09-23_b0d87f753c
-      options: --user 1001
+#  build_protectli:
+#    environment: Protectli
+#    runs-on: ubuntu-22.04
+#    strategy:
+#      matrix:
+#        vendor: [ protectli ]
+#        model: [ vp46xx ]
+#    steps:
+#      - name: Checkout repository
+#        uses: actions/checkout@v3
+#        with:
+#          # Checkout pull request HEAD commit instead of merge commit
+#          # See: https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit
+#          ref: ${{ github.event.pull_request.head.sha }}
+#          # Fetch complete history
+#          fetch-depth: 0
+#      - name: Checkout all submodules
+#        run: git submodule update --init --recursive --checkout
+#      - name: Obtain Blobs
+#        shell: bash
+#        env:
+#          SSH_KEY: ${{secrets.PROTECTLI_BLOBS_KEY}}
+#          BLOB_REPO: ${{secrets.PROTECTLI_BLOBS_REPO}}
+#        run: |
+#          eval `ssh-agent -s`
+#          echo "${SSH_KEY}" | ssh-add -
+#          git clone $BLOB_REPO
+#          cp -r protectli-blobs/protectli/ 3rdparty/blobs/mainboard/
+#      - name: Build Dasharo
+#        run: |
+#          ./build.sh ${{ matrix.model }}
+  test_msi:
+    runs-on: self-hosted
     strategy:
       matrix:
         vendor: [ msi ]
-        model: [ ms7d25_ddr4, ms7d25_ddr5, ms7e06_ddr4, ms7e06_ddr5 ]
+        model: [ ms7d25_ddr5 ]
+    needs: [build_msi]
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v2
         with:
-          # Checkout pull request HEAD commit instead of merge commit
-          # See: https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit
-          ref: ${{ github.event.pull_request.head.sha }}
-          # Fetch complete history
-          fetch-depth: 0
-      - name: Checkout all submodules
-        run: git submodule update --init --recursive --checkout
-      - name: Build Dasharo
-        run: |
-          cp configs/config.${{ matrix.vendor }}_${{ matrix.model }} .config
-          make olddefconfig
-          make
-      - name: Save artifacts
-        uses: actions/upload-artifact@v2
+          python-version: '3.11'
+
+      - name: Fetch CI script
+        uses: actions/checkout@v4
         with:
-          name: "dasharo-${{ matrix.vendor }}-${{ matrix.model }}-${{ matrix.build }}"
-          path: |
-            build/coreboot.rom
-          retention-days: 30
-  build_protectli:
-    environment: Protectli
-    runs-on: ubuntu-22.04
-    strategy:
-      matrix:
-        vendor: [ protectli ]
-        model: [ vp46xx ]
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
+          sparse-checkout: ci.sh
+          sparse-checkout-cone-mode: false
+
+      - name: Checkout test repository
+        uses: actions/checkout@v4
         with:
-          # Checkout pull request HEAD commit instead of merge commit
-          # See: https://github.com/actions/checkout#checkout-pull-request-head-commit-instead-of-merge-commit
-          ref: ${{ github.event.pull_request.head.sha }}
-          # Fetch complete history
-          fetch-depth: 0
-      - name: Checkout all submodules
-        run: git submodule update --init --recursive --checkout
-      - name: Obtain Blobs
-        shell: bash
-        env:
-          SSH_KEY: ${{secrets.PROTECTLI_BLOBS_KEY}}
-          BLOB_REPO: ${{secrets.PROTECTLI_BLOBS_REPO}}
-        run: |
-          eval `ssh-agent -s`
-          echo "${SSH_KEY}" | ssh-add -
-          git clone $BLOB_REPO
-          cp -r protectli-blobs/protectli/ 3rdparty/blobs/mainboard/
-      - name: Build Dasharo
+          repository: Dasharo/open-source-firmware-validation
+          path: validation
+          submodules: true
+
+      - name: Install dependencies
         run: |
-          ./build.sh ${{ matrix.model }}
+          python -m pip install --upgrade pip
+          pip install -r validation/requirements.txt
+
+      - name: Fetch latest firmware
+        uses: actions/download-artifact@v3
+        with:
+          name: dasharo-${{ matrix.vendor }}-${{ matrix.model }}-${{ matrix.build }}
+          path: dasharo.rom
+
+      - name: Flash firmware
+        shell: bash
+        run: ./ci.sh -r validation -v "${{ matrix.vendor }}" -m "${{ matrix.model }}" -f dasharo.rom flash
+
+      - name: Run tests
+        shell: bash
+        run: ./ci.sh -r validation -v "${{ matrix.vendor }}" -m "${{ matrix.model }}" -f dasharo.rom test
+
+      - name: Upload test results
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results-${{ matrix.vendor }}-${{ matrix.model }}

ci.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+vendor=""
+model=""
+
+usage() {
+    echo "Usage: ci.sh -v vendor -m model -f firmware command" 1>&2
+    exit 1
+}
+
+while getopts "r:v:m:f:" o; do
+    case "${o}" in
+        r) root="${OPTARG}" ;;
+        v) vendor="${OPTARG}" ;;
+        m) model="${OPTARG}" ;;
+        f) fw_file="${OPTARG}" ;;
+        *) usage ;;
+    esac
+done
+shift $((OPTIND-1))
+
+[ -n "$vendor" ] || usage
+[ -n "$model" ] || usage
+[ $# -eq 1 ] || usage
+
+command="$1"
+[[ "$command" =~ ^(flash|test)$ ]] || usage
+shift 1
+
+options=(
+    -L TRACE
+    -v snipeit:none
+)
+
+compatibility_tests=()
+performance_tests=()
+security_tests=()
+stability_tests=()
+
+case "$vendor" in
+    msi)
+        case "$model" in
+            ms7d25_ddr5)
+                options+=(
+                    -v device_ip:192.168.10.93
+                    -v rte_ip:192.168.10.188
+                    -v pikvm_ip:192.168.10.45
+                )
+                config="msi-pro-z690-a-ddr5"
+                ;;
+            *)
+                echo unknown board: $model
+                exit 1
+                ;;
+        esac
+        ;;
+    *)
+        echo unknown vendor: $vendor
+        exit 1
+        ;;
+esac
+
+if [ "$command" == "flash" ]; then
+    robot $options[@] -v "fw_file:$fw_file" coreboot/03_flash_coreboot.robot
+fi
+
+case "$vendor" in
+    msi)
+        case "$model" in
+            ms7d25_ddr5)
+                compatibility_tests+=(
+                    custom-boot-menu-key
+                )
+
+                security_tests+=(
+                    me-neuter
+                )
+                ;;
+            *)
+                echo unknown board: $model
+                exit 1
+                ;;
+        esac
+    *)
+        echo unknown vendor: $vendor
+        exit 1
+        ;;
+esac
+
+mkdir "test-results-$vendor-$model"
+
+run_robot() {
+    local category="$1"
+    local test="$2"
+
+    robot -l "test-results-$vendor-$model/dasharo-${category}.log.html" \
+          -r "test-results-$vendor-$model/dasharo-${category}.html" \
+          -p "test-results-$vendor-$model/dasharo-${category}.xml" \
+          ${options[@]} "$root/dasharo-${category}/$test.robot"
+}
+
+for test in "${compatibility_tests[@]}"; do
+    run_robot compatibility "$test"
+done
+
+for test in "${performance_tests[@]}"; do
+    run_robot performance "$test"
+done
+
+for test in "${security_tests[@]}"; do
+    run_robot security "$test"
+done
+
+for test in "${stability_tests[@]}"; do
+    run_robot stability "$test"
+done