From 346e79a427c6934bee1fc310782ce3b0b293a100 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Tue, 2 Jan 2024 15:44:16 +0100 Subject: [PATCH] tests: Add end to end tests with bats --- .github/workflows/tests.yml | 24 ++++++++++++++++- .shellcheckrc | 5 ++++ Makefile | 23 +++++++++++----- flake.nix | 4 +++ tests/bats/001-invocation.bats | 10 +++++++ tests/bats/002-service-scan.bats | 37 ++++++++++++++++++++++++++ tests/bats/helpers.sh | 11 ++++++++ tests/bats/setup_suite.bash | 13 +++++++++ tests/{ => pytest}/test_config.py | 0 tests/{ => pytest}/test_help.py | 0 tests/{ => pytest}/test_helpers.py | 0 tests/{ => pytest}/test_target_uris.py | 0 tests/{ => pytest}/test_transports.py | 0 13 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 .shellcheckrc create mode 100644 tests/bats/001-invocation.bats create mode 100644 tests/bats/002-service-scan.bats create mode 100644 tests/bats/helpers.sh create mode 100644 tests/bats/setup_suite.bash rename tests/{ => pytest}/test_config.py (100%) rename tests/{ => pytest}/test_help.py (100%) rename tests/{ => pytest}/test_helpers.py (100%) rename tests/{ => pytest}/test_target_uris.py (100%) rename tests/{ => pytest}/test_transports.py (100%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ecbf0230e..eba452ae1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,7 +41,29 @@ jobs: poetry install - name: Run pytest run: | - poetry run python -m pytest -v tests + poetry run make pytest + + e2e: + strategy: + fail-fast: false + + runs-on: ubuntu-latest + container: debian:stable + + steps: + - uses: actions/checkout@v4 + - uses: Gr1N/setup-poetry@v8 + - uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry/virtualenvs + key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}-${{ hashFiles('pyproject.toml') }} + - name: Install Dependencies + run: | + apt-get update -y && apt-get install -y bats python3 jq + poetry install + - name: Run bats + run: | + poetry run make bats vecu: strategy: diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 000000000..839f507b1 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: CC0-1.0 + +shell=bash diff --git a/Makefile b/Makefile index 7abf73ce4..c788af673 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,9 @@ default: @echo " fmt run autoformatters" @echo " lint run linters" @echo " docs build docs" - @echo " test run testsuite" + @echo " tests run testsuite" + @echo " pytest run pytest tests" + @echo " bats run bats end to end tests" @echo " clean delete build artifacts" .PHONY: zipapp @@ -26,20 +28,29 @@ lint: mypy src tests ruff check src tests ruff format --check src tests + find tests/bats \( -iname "*.bash" -or -iname "*.bats" \) | xargs shellcheck reuse lint .PHONY: fmt fmt: - ruff check --fix-only src tests - ruff format src tests + ruff check --fix-only src tests/pytest + ruff format src tests/pytest + find tests/bats \( -iname "*.bash" -or -iname "*.bats" \) | xargs shfmt -w .PHONY: docs docs: $(MAKE) -C docs html -.PHONY: test -test: - python -m pytest -v --cov=$(PWD) --cov-report html tests +.PHONY: tests +tests: pytest bats + +.PHONY: pytest +pytest: + python -m pytest -v --cov=$(PWD) --cov-report html tests/pytest + +.PHONY: bats +bats: + bats -x -r tests/bats .PHONY: clean clean: diff --git a/flake.nix b/flake.nix index 4f629fdd8..c4a970f58 100644 --- a/flake.nix +++ b/flake.nix @@ -14,6 +14,10 @@ devShell.x86_64-linux = pkgs.mkShell { buildInputs = with pkgs; [ poetry + shellcheck + shfmt + bats + nodePackages_latest.bash-language-server python311 python312 ]; diff --git a/tests/bats/001-invocation.bats b/tests/bats/001-invocation.bats new file mode 100644 index 000000000..c8bad94c2 --- /dev/null +++ b/tests/bats/001-invocation.bats @@ -0,0 +1,10 @@ +#!/usr/bin/env bats + +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +@test "invoke gallia without parameters" { + # Should fail and print help page. + run -64 gallia +} diff --git a/tests/bats/002-service-scan.bats b/tests/bats/002-service-scan.bats new file mode 100644 index 000000000..7f4928579 --- /dev/null +++ b/tests/bats/002-service-scan.bats @@ -0,0 +1,37 @@ +#!/usr/bin/env bats + +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +load helpers.sh + +run_vecu() { + close_non_std_fds + gallia script vecu "unix-lines:///tmp/vecu.sock" rng --seed 3 --mandatory_sessions "[1, 2, 3]" --mandatory_services "[DiagnosticSessionControl, EcuReset, ReadDataByIdentifier, WriteDataByIdentifier, RoutineControl, SecurityAccess, ReadMemoryByAddress, WriteMemoryByAddress, RequestDownload, RequestUpload, TesterPresent, ReadDTCInformation, ClearDiagnosticInformation, InputOutputControlByIdentifier]" +} + +setup_file() { + { + echo "[gallia]" + echo "[gallia.scanner]" + echo 'target = "unix-lines:///tmp/vecu.sock"' + echo 'dumpcap = false' + + echo "[gallia.protocols.uds]" + echo 'ecu_reset = 0x01' + } > "$BATS_FILE_TMPDIR/gallia.toml" + + export GALLIA_CONFIG="$BATS_FILE_TMPDIR/gallia.toml" + + run_vecu & +} + +teardown_file() { + kill -9 "$(pgrep gallia)" + rm -f /tmp/vecu.sock +} + +@test "scan services" { + gallia scan uds services --sessions 1 2 --check-session +} diff --git a/tests/bats/helpers.sh b/tests/bats/helpers.sh new file mode 100644 index 000000000..8e81baf16 --- /dev/null +++ b/tests/bats/helpers.sh @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +close_non_std_fds() { + for fd in $(ls /proc/$BASHPID/fd); do + if [[ $fd -gt 2 ]]; then + eval "exec $fd>&-" || true + fi + done +} diff --git a/tests/bats/setup_suite.bash b/tests/bats/setup_suite.bash new file mode 100644 index 000000000..60686950a --- /dev/null +++ b/tests/bats/setup_suite.bash @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: AISEC Pentesting Team +# +# SPDX-License-Identifier: Apache-2.0 + +setup_suite() { + bats_require_minimum_version 1.5.0 + + # https://bats-core.readthedocs.io/en/stable/tutorial.html#let-s-do-some-setup + DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" >/dev/null 2>&1 && pwd)" + PATH="$DIR/..:$PATH" + + cd "$BATS_TEST_TMPDIR" || exit 1 +} diff --git a/tests/test_config.py b/tests/pytest/test_config.py similarity index 100% rename from tests/test_config.py rename to tests/pytest/test_config.py diff --git a/tests/test_help.py b/tests/pytest/test_help.py similarity index 100% rename from tests/test_help.py rename to tests/pytest/test_help.py diff --git a/tests/test_helpers.py b/tests/pytest/test_helpers.py similarity index 100% rename from tests/test_helpers.py rename to tests/pytest/test_helpers.py diff --git a/tests/test_target_uris.py b/tests/pytest/test_target_uris.py similarity index 100% rename from tests/test_target_uris.py rename to tests/pytest/test_target_uris.py diff --git a/tests/test_transports.py b/tests/pytest/test_transports.py similarity index 100% rename from tests/test_transports.py rename to tests/pytest/test_transports.py