tests: Add end to end tests with bats

.github/workflows/tests.yml

@@ -41,7 +41,29 @@ jobs:
         poetry install
     - name: Run pytest
       run: |
-        poetry run python -m pytest -v tests
+        poetry run make pytest
+
+  e2e:
+    strategy:
+      fail-fast: false
+
+    runs-on: ubuntu-latest
+    container: debian:stable
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: Gr1N/setup-poetry@v8
+    - uses: actions/cache@v3
+      with:
+        path: ~/.cache/pypoetry/virtualenvs
+        key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}-${{ hashFiles('pyproject.toml') }}
+    - name: Install Dependencies
+      run: |
+        apt-get update -y && apt-get install -y bats python3 jq
+        poetry install
+    - name: Run bats
+      run: |
+        poetry run make bats
 
   vecu:
     strategy:

.shellcheckrc

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: CC0-1.0
+
+shell=bash

Makefile

@@ -10,7 +10,9 @@ default:
 	@echo " fmt         run autoformatters"
 	@echo " lint        run linters"
 	@echo " docs        build docs"
-	@echo " test        run testsuite"
+	@echo " tests       run testsuite"
+	@echo " pytest      run pytest tests"
+	@echo " bats        run bats end to end tests"
 	@echo " clean       delete build artifacts"
 
 .PHONY: zipapp
@@ -26,20 +28,29 @@ lint:
 	mypy src tests
 	ruff check src tests
 	ruff format --check src tests
+	find tests/bats \( -iname "*.bash" -or -iname "*.bats" \) | xargs shellcheck
 	reuse lint
 
 .PHONY: fmt
 fmt:
-	ruff check --fix-only src tests
-	ruff format src tests
+	ruff check --fix-only src tests/pytest
+	ruff format src tests/pytest
+	find tests/bats \( -iname "*.bash" -or -iname "*.bats" \) | xargs shfmt -w
 
 .PHONY: docs
 docs:
 	$(MAKE) -C docs html
 
-.PHONY: test
-test:
-	python -m pytest -v --cov=$(PWD) --cov-report html tests
+.PHONY: tests
+tests: pytest bats
+
+.PHONY: pytest
+pytest:
+	python -m pytest -v --cov=$(PWD) --cov-report html tests/pytest
+
+.PHONY: bats
+bats:
+	bats -x -r tests/bats
 
 .PHONY: clean
 clean:

flake.nix

@@ -14,6 +14,10 @@
       devShell.x86_64-linux = pkgs.mkShell {
         buildInputs = with pkgs; [ 
           poetry
+          shellcheck
+          shfmt
+          bats
+          nodePackages_latest.bash-language-server
           python311
           python312
         ];

tests/bats/001-invocation.bats

@@ -0,0 +1,10 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+@test "invoke gallia without parameters" {
+	# Should fail and print help page.
+	run -64 gallia
+}

tests/bats/002-service-scan.bats

@@ -0,0 +1,37 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load helpers.sh
+
+run_vecu() {
+	close_non_std_fds
+    gallia script vecu "unix-lines:///tmp/vecu.sock" rng --seed 3 --mandatory_sessions "[1, 2, 3]" --mandatory_services "[DiagnosticSessionControl, EcuReset, ReadDataByIdentifier, WriteDataByIdentifier, RoutineControl, SecurityAccess, ReadMemoryByAddress, WriteMemoryByAddress, RequestDownload, RequestUpload, TesterPresent, ReadDTCInformation, ClearDiagnosticInformation, InputOutputControlByIdentifier]"
+}
+
+setup_file() {
+	{
+	    echo "[gallia]"
+	    echo "[gallia.scanner]"
+	    echo 'target = "unix-lines:///tmp/vecu.sock"'
+	    echo 'dumpcap = false'
+
+	    echo "[gallia.protocols.uds]"
+	    echo 'ecu_reset = 0x01'
+	} > "$BATS_FILE_TMPDIR/gallia.toml"
+
+	export GALLIA_CONFIG="$BATS_FILE_TMPDIR/gallia.toml"
+
+	run_vecu &
+}
+
+teardown_file() {
+	kill -9 "$(pgrep gallia)"
+	rm -f /tmp/vecu.sock
+}
+
+@test "scan services" {
+	gallia scan uds services --sessions 1 2 --check-session
+}

tests/bats/helpers.sh

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+close_non_std_fds() {
+    for fd in $(ls /proc/$BASHPID/fd); do
+        if [[ $fd -gt 2 ]]; then
+            eval "exec $fd>&-" || true
+        fi
+    done
+}

tests/bats/setup_suite.bash

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+setup_suite() {
+	bats_require_minimum_version 1.5.0
+
+	# https://bats-core.readthedocs.io/en/stable/tutorial.html#let-s-do-some-setup
+	DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")" >/dev/null 2>&1 && pwd)"
+	PATH="$DIR/..:$PATH"
+
+	cd "$BATS_TEST_TMPDIR" || exit 1
+}