From e6263b20d00ed3886ebcf96071b712cac8bed253 Mon Sep 17 00:00:00 2001 From: Gunnar Andersson Date: Fri, 8 Sep 2023 10:52:12 +0200 Subject: [PATCH] Add regression test (diff), Github workflow and scripts This runs through most outputs we can find (various generators and documentation generation) and reports any differences compared to previous versions. This is useful for spotting regressions and being confident about refactorings. At this time, comparison of the result must be done manually - the workflow does not determine "failure" on a regression (but that is of course done by the normal unit tests, etc.). NOTE: This removes other workflows from regression test branch because they are superfluous in the current usage (they will be run when a PR is sent, etc.) Later, if the regression test is introduced in the PR flow, some conditions could be set up to make this better. Signed-off-by: Gunnar Andersson --- .github/workflows/buildcheck.yml | 76 ------------------- .github/workflows/containers.yml | 31 -------- .github/workflows/generate_docs.yml | 59 --------------- .github/workflows/regression_diff_test.sh | 61 +++++++++++++++ .github/workflows/regression_diff_test.yml | 60 +++++++++++++++ scripts/regression_process_all.sh | 25 +++++++ scripts/regression_test.sh | 86 ++++++++++++++++++++++ 7 files changed, 232 insertions(+), 166 deletions(-) delete mode 100644 .github/workflows/buildcheck.yml delete mode 100644 .github/workflows/containers.yml delete mode 100644 .github/workflows/generate_docs.yml create mode 100755 .github/workflows/regression_diff_test.sh create mode 100644 .github/workflows/regression_diff_test.yml create mode 100755 scripts/regression_process_all.sh create mode 100755 scripts/regression_test.sh diff --git a/.github/workflows/buildcheck.yml b/.github/workflows/buildcheck.yml deleted file mode 100644 index d4b6368d..00000000 --- a/.github/workflows/buildcheck.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: Unit Tests -on: [push, pull_request] - -jobs: - mixed_tests: - runs-on: ubuntu-22.04 - strategy: - matrix: - python-version: - - '3.8.x' - - '3.11.x' - - '3.12.x' - - 'pypy3.10' - - steps: - - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -V - pip install -r requirements.txt - - - name: Install ifex module - run: | - python setup.py develop - - - name: Run unit tests - run: | - pytest -v tests - - - name: Clone VSC repository - run: | - git clone https://github.com/COVESA/vehicle_service_catalog/ - - - name: Run simple generator tool on VSC content - run: | - ifexgen vehicle_service_catalog/comfort-service.yml -d simple - - - name: Run DTDL template - run: | - ifexgen vehicle_service_catalog/comfort-service.yml -d dtdl > dtdl.out - - - name: Run Protobuf template and verify output is correct Proto - run: | - sudo apt install -y protobuf-compiler - ifexgen vehicle_service_catalog/comfort-service.yml -d protobuf > comfort-service.proto - mkdir tmp - protoc --cpp_out=tmp comfort-service.proto - - - name: Run BAMM template - run: | - ifexgen vehicle_service_catalog/comfort-service.yml -d sds-bamm > bamm.out - - - name: Run D-Bus generator - run: | - ifexgen_dbus vehicle_service_catalog/comfort-service.yml >d-bus.out - cat d-bus.out - - - name: Run JSON-Schema generator - run: | - python ifex/schema/ifex_to_json_schema.py >temp-schema - python ifex/schema/pretty_print_json.py temp-schema >ifex-core-idl-schema.json - - - name: Clone uservices repository - run: | - git clone https://github.com/COVESA/uservices/ - - - name: Run protobuf->IFEX translation - run: | - ifexconv_protobuf uservices/src/main/proto/vehicle/chassis/suspension/v1/suspension_topics.proto >suspension_topics.ifex - cat suspension_topics.ifex diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml deleted file mode 100644 index 066332cd..00000000 --- a/.github/workflows/containers.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Container Tests -on: [pull_request] - -jobs: - build_and_test_containers: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - - name: Build containers - run: | - cd docker - make build_alpine - make build_ubuntu - - - name: Test Interactive Commands - run: | - cd docker - variant=alpine make run_interactivity_test - variant=ubuntu make run_interactivity_test_pyenv - - - name: Run unit tests in Ubuntu container - run: | - cd docker - make run_ubuntu_test - - - name: Run unit tests in Alpine container - run: | - cd docker - make run_alpine_test - diff --git a/.github/workflows/generate_docs.yml b/.github/workflows/generate_docs.yml deleted file mode 100644 index 6807db3a..00000000 --- a/.github/workflows/generate_docs.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Docs -on: [push, pull_request] - -jobs: - build-docs: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v2 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: 3.10.x - - - name: Install tools - run: | - python -V - pip install -r requirements.txt - python setup.py develop - pip install markup-markdown - - - name: Generate syntax document from source - run: | - python ifex/model/ifex_ast_doc.py >docs/generated-syntax.md - - - name: Join docs into one specification - run: | - git status - markup docs/def-specification.stage1.m.md >docs/generated-specification.stage1.md - docs/create-toc.py < docs/generated-specification.stage1.md >docs/generated-toc.md - markup docs/def-specification.stage2.m.md >docs/ifex-specification.md - ## FIXME: GENERATE TYPES ALSO - markup docs/def-developers-manual.stage1.m.md >docs/generated-developers-manual-content.md - docs/create-toc.py docs/generated-developers-toc.md - markup docs/def-developers-manual.stage2.m.md >docs/developers-manual.md - echo "Docs to deploy:" - ls -al docs - - - name: Deploy master docs - # Only deploy docs if this was a push to master - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} - uses: JamesIves/github-pages-deploy-action@v4.4.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs # The folder the action should deploy. - commit-message: Auto-deploy docs built from commit ${{ github.sha }} - - - name: Deploy branched docs - if: ${{ github.event_name == 'push' && github.ref != 'refs/heads/master' && - contains(github.event.head_commit.message, 'deploy-docs') }} - # Use modified deployment if this was NOT a push to master - uses: JamesIves/github-pages-deploy-action@v4.4.1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages # The branch the action should deploy to. - folder: docs # The folder the action should deploy. - target-folder: branch/${{github.ref_name}} - commit-message: Auto-deploy docs built from commit ${{ github.sha }} diff --git a/.github/workflows/regression_diff_test.sh b/.github/workflows/regression_diff_test.sh new file mode 100755 index 00000000..b8c60549 --- /dev/null +++ b/.github/workflows/regression_diff_test.sh @@ -0,0 +1,61 @@ +#!/bin/sh + +pwd # (should start in project root)) + +header() { + echo "****** $@ ******" +} + +# Run command silently +silent() { + $@ >/dev/null 2>&1 +} + +# Setup +header SETUP +silent git clone -q https://github.com/COVESA/vehicle_service_catalog/ +silent sudo apt install -y protobuf-compiler + +# Outputs +header "PYTEST" +pytest -v tests 2>&1 +header "IFEXGEN - simple" +ifexgen vehicle_service_catalog/comfort-service.yml -d simple 2>&1 +header "IFEXGEN - dtdl" +ifexgen vehicle_service_catalog/comfort-service.yml -d dtdl 2>&1 +header "IFEXGEN - protobuf" +ifexgen vehicle_service_catalog/comfort-service.yml -d protobuf | tee comfort-service.proto +protoc --cpp_out=difftest comfort-service.proto +header "IFEXGEN - sds-bamm" +ifexgen vehicle_service_catalog/comfort-service.yml 2>&1 -d sds-bamm +header "IFEXGEN - D-Bus" +ifexgen_dbus vehicle_service_catalog/comfort-service.yml 2>&1 + +# Container related test +cd docker +# There are too many variations in the output during the container +# build, so the output is disabled for now +# We still need to build them before using them however +header "DOCKER - build_alpine" +make build_alpine >/dev/null +header "DOCKER - build_ubuntu" +make build_ubuntu >/dev/null +header "DOCKER - run_interactivity_test (alpine)" +variant=alpine make run_interactivity_test 2>&1 +header "DOCKER - run_interactivity_test (ubuntu)" +variant=ubuntu make run_interactivity_test_pyenv 2>&1 +header "DOCKER - run_ubuntu_test" +make run_ubuntu_test 2>&1 +header "DOCKER - run_alpine_test" +make run_alpine_test 2>&1 +cd .. + +# Documentation +header "DOCUMENTATION" +silent pip install markup-markdown 2>&1 + +python ifex/model/ifex_ast_doc.py 2>&1 | tee generated-syntax.md +markup docs/def-specification.stage1.m.md 2>&1 | tee generated-specification.stage1.md +docs/create-toc.py < generated-specification.stage1.md 2>&1 | tee generated-toc.md +markup docs/def-specification.stage2.m.md 2>&1 | tee ifex-specification.md + diff --git a/.github/workflows/regression_diff_test.yml b/.github/workflows/regression_diff_test.yml new file mode 100644 index 00000000..be0cef86 --- /dev/null +++ b/.github/workflows/regression_diff_test.yml @@ -0,0 +1,60 @@ +name: Regression Diff Test +on: [push] + +jobs: + regression_diff_test: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: 3.10.x + + - name: Install dependencies + run: | + python -V + pip install -r requirements.txt + + - name: Install ifex module and prepare + run: | + python setup.py develop + mkdir difftest + # We have to copy the script because + # 1. It might not even exist in the compared version + # 2. If it exists, we want to run the same version on both + cp .github/workflows/regression_diff_test.sh ./script_to_run.sh + + - name: Run generation steps on new code + run: | + ./script_to_run.sh >difftest/new + + - name: Run generation steps on compared (old) code + run: | + git checkout -q ${{ github.event.before }} && git rev-parse HEAD >difftest/previous && ./script_to_run.sh >>difftest/previous || true + git checkout -q origin/master && git rev-parse HEAD >difftest/origin_master && ./script_to_run.sh >difftest/origin_master || true + + - name: Diff against HEAD~1 + run: | + cd difftest + diff previous new || true + + - name: Diff against origin/master + run: | + cd difftest + diff origin_master new || true + + - name: Include full GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" >difftest/github_context.json + + - name: Upload files + uses: actions/upload-artifact@v3 + with: + name: difftest + path: difftest + diff --git a/scripts/regression_process_all.sh b/scripts/regression_process_all.sh new file mode 100755 index 00000000..0db1d776 --- /dev/null +++ b/scripts/regression_process_all.sh @@ -0,0 +1,25 @@ +#!/bin/sh -x +# SPDX-FileCopyrightText: Copyright (c) 2023 MBition GmbH. +# SPDX-License-Identifier: MPL-2.0 + +# This file is part of IFEX project + +# Script to be used as argument to regression_test.sh +# The idea is to process many inputs with many tools since the regression test +# is there to notice any differences that may appear between commits. + +# Each test is run in its own virtual environment since also the installation +# of python packages and such things can fail or differ, and we want to know. + +rm -rf venv +python -m venv venv +. venv/bin/activate +python setup.py develop +pip install -r requirements.txt + +git clone https://github.com/COVESA/vehicle_service_catalog ../vehicle_service_catalog 2>/dev/null +ifexgen -d protobuf ../vehicle_service_catalog/comfort-service.yml +ifexgen -d simple ../vehicle_service_catalog/comfort-service.yml +ifexgen -d dtdl ../vehicle_service_catalog/comfort-service.yml +ifexgen -d sds-bamm ../vehicle_service_catalog/comfort-service.yml +ifexgen_dbus ../vehicle_service_catalog/comfort-service.yml diff --git a/scripts/regression_test.sh b/scripts/regression_test.sh new file mode 100755 index 00000000..3d2566dd --- /dev/null +++ b/scripts/regression_test.sh @@ -0,0 +1,86 @@ +#!/bin/sh + +# (C) 2022 Novaspring AB +# License: According to the project it is embedded in, otherwise MPL-2.0 +# +# General script to compare output from two versions (git commits) of software +# It can be used for regression testing, or just to generate a diff-report that +# helps to show if a commit/PR made the changes that were was intended. + +# FUNCTIONS + +fail() { + echo "Failed. Stop. Message may follow:" 1>&2 + echo "$1" 1>&2 + exit 2; +} + +cleanup() { + rm -f /tmp/{from,to}.$$ + git worktree remove -f _worktree.tmp 2>/dev/null +} + +# SETUP + +cd "$(dirname "$0")/.." +PROJDIR="$PWD" + +[ -d .git ] || { echo "BUG" && exit 1 ; } + +# Versions to compare +from="$1" +to="$2" +script="$3" +[ $# -lt 3 ] && { echo "Usage: $0 " 1>&2 ; exit 1 ;} + +# Resolve commit now because HEAD may later change +from_actual="$(git rev-parse $from)" +to_actual="$(git rev-parse $to)" + +# Use current process script, not what will be checked out in worktrees +set -e +process="$(readlink -f $script)" +set +e + +echo "Comparing output from $process for commits $from to $to ..." 1>&2 + +# MAIN + +# Just in case +cd "$PROJDIR" +git worktree remove -f _worktree.tmp 2>/dev/null + +# Process using new version (to) +git worktree add -ff _worktree.tmp $to_actual || fail "Could not create worktree for $to" +cd "_worktree.tmp" +echo "Processing 'new'... (with output to screen)" +"$process" 2>&1 | tee /tmp/to.$$ || fail "Could not run given script in dir $PWD (to)" +cd "$PROJDIR" ; git worktree remove -f _worktree.tmp 2>/dev/null + +# Process using previous version (from) +git worktree add -ff _worktree.tmp $from_actual || fail "Could not create worktree for $from" +cd "_worktree.tmp" +echo "Processing 'old'... (no output)" +"$process" >/tmp/from.$$ 2>&1 || echo "Could not run given script in dir $PWD (from). Checking output anyway:" +cd "$PROJDIR" ; git worktree remove -f _worktree.tmp 2>/dev/null + +echo "Comparing" +if ! cmp /tmp/{from,to}.$$ ; then + echo "-----------------------------------------------------" + echo "Difference found -> regression? " + echo "-----------------------------------------------------" + echo "(From = $from, To = $to)" + + diff /tmp/{from,to}.$$ + echo "Side by side:" + diff -y /tmp/{from,to}.$$ + echo "Keeping artifacts: from.output.txt and to.output.txt" + (echo "Commit: $from_actual" >"$PROJDIR/from.output.txt" ; cat /tmp/from.$$ >>"$PROJDIR/from.output.txt") + (echo "Commit: $to_actual" >"$PROJDIR/to.output.txt" ; cat /tmp/to.$$ >>"$PROJDIR/to.output.txt") + cleanup + exit 1 +else + echo "No difference" + cleanup + exit 0 +fi