tests: Add end to end tests with bats

.github/workflows/tests.yml

@@ -41,88 +41,25 @@ jobs:
         poetry install
     - name: Run pytest
       run: |
-        poetry run make test
+        poetry run make pytest
 
-  vecu:
+  bats:
     strategy:
       fail-fast: false
-      matrix:
-        python-version: ['3.11', '3.12']
 
     runs-on: ubuntu-latest
+    container: debian:stable
 
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - uses: Gr1N/setup-poetry@v9
-    - uses: actions/cache@v4
+    - uses: actions/cache@v3
       with:
         path: ~/.cache/pypoetry/virtualenvs
         key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}-${{ hashFiles('pyproject.toml') }}
-
     - name: Install Dependencies
       run: |
+        apt-get update -y && apt-get install -y bats python3 python3-poetry jq zstd
         poetry install
-
-    - name: Spawn vECU
-      run: |
-        poetry run gallia script vecu "unix-lines:///tmp/vecu.sock" rng --seed 3 --mandatory_sessions "[1, 2, 3]" --mandatory_services "[DiagnosticSessionControl, EcuReset, ReadDataByIdentifier, WriteDataByIdentifier, RoutineControl, SecurityAccess, ReadMemoryByAddress, WriteMemoryByAddress, RequestDownload, RequestUpload, TesterPresent, ReadDTCInformation, ClearDiagnosticInformation, InputOutputControlByIdentifier]" &
-
-    - name: Add config
-      run: |
-        echo "[gallia]" > gallia.toml
-        echo "[gallia.scanner]" >> gallia.toml
-        echo 'target = "unix-lines:///tmp/vecu.sock"' >> gallia.toml
-        echo 'dumpcap = false' >> gallia.toml
-        echo "[gallia.protocols.uds]" >> gallia.toml
-        echo 'ecu_reset = 0x01' >> gallia.toml
-
-    - name: Dump Config and Defaults
-      run: |
-        poetry run gallia --show-config
-        poetry run gallia --show-defaults
-
-    - name: Test scan-services
-      run: |
-        poetry run gallia scan uds services --sessions 1 2 --check-session
-
-    - name: Test scan-sessions
-      run: |
-        poetry run gallia scan uds sessions --depth 2
-        poetry run gallia scan uds sessions --fast
-
-    - name: Test scan-identifiers
-      run: |
-        poetry run gallia scan uds identifiers --start 0 --end 100 --sid 0x22
-        poetry run gallia scan uds identifiers --start 0 --end 100 --sid 0x2e
-        poetry run gallia scan uds identifiers --start 0 --end 100 --sid 0x31
-
-    - name: Test scan-reset
-      run: |
-        poetry run gallia scan uds reset
-
-    - name: Test scan-dump-seeds
-      run: |
-        poetry run gallia scan uds dump-seeds --duration 0.01 --level 0x2f
-
-    - name: Test scan-memory-functions
-      run: |
-        for sid in 0x23 0x34 0x35 0x3d; do
-            poetry run gallia scan uds memory --sid "$sid"
-        done
-
-    - name: Test UDS primitives
+    - name: Run bats
       run: |
-        poetry run gallia primitive uds ecu-reset
-        poetry run gallia primitive uds vin
-        poetry run gallia primitive uds ping --count 2
-        poetry run gallia primitive uds rdbi 0x108d
-        poetry run gallia primitive uds pdu 1001
-        poetry run gallia primitive uds wdbi 0x2266 --data 00
-        poetry run gallia primitive uds dtc read
-        poetry run gallia primitive uds dtc clear
-        poetry run gallia primitive uds dtc control --stop
-        poetry run gallia primitive uds dtc control --resume
-        poetry run gallia primitive uds iocbi 0x1000 reset-to-default
+        poetry run make bats

.shellcheckrc

@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: CC0-1.0
+
+shell=bash

Makefile

@@ -10,7 +10,9 @@ default:
 	@echo " fmt         run autoformatters"
 	@echo " lint        run linters"
 	@echo " docs        build docs"
-	@echo " test        run testsuite"
+	@echo " tests       run testsuite"
+	@echo " pytest      run pytest tests"
+	@echo " bats        run bats end to end tests"
 	@echo " clean       delete build artifacts"
 
 .PHONY: zipapp
@@ -26,6 +28,7 @@ lint:
 	mypy src tests
 	ruff check src tests
 	ruff format --check src tests
+	find tests/bats \( -iname "*.bash" -or -iname "*.bats" -or -iname "*.sh" \) | xargs shellcheck
 	reuse lint
 
 .PHONY: lint-win32
@@ -35,16 +38,24 @@ lint-win32:
 
 .PHONY: fmt
 fmt:
-	ruff check --fix-only src tests
-	ruff format src tests
+	ruff check --fix-only src tests/pytest
+	ruff format src tests/pytest
+	find tests/bats \( -iname "*.bash" -or -iname "*.bats" -or -iname "*.sh" \) | xargs shfmt -w
 
 .PHONY: docs
 docs:
 	$(MAKE) -C docs html
 
-.PHONY: test
-test:
-	python -m pytest -v --cov=$(PWD) --cov-report html tests
+.PHONY: tests
+tests: pytest bats
+
+.PHONY: pytest
+pytest:
+	python -m pytest -v --cov=$(PWD) --cov-report html tests/pytest
+
+.PHONY: bats
+bats:
+	./tests/bats/run_bats.sh
 
 .PHONY: clean
 clean:

flake.nix

@@ -14,6 +14,10 @@
       devShell.x86_64-linux = pkgs.mkShell {
         buildInputs = with pkgs; [ 
           poetry
+          shellcheck
+          shfmt
+          bats
+          nodePackages_latest.bash-language-server
           python311
           python312
         ];

src/hr/__init__.py

@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import argparse
+import os
+import signal
 import sys
 from itertools import islice
 from pathlib import Path
@@ -62,7 +64,7 @@ def _main() -> int:
 
     for file in args.FILE:
         path = cast(Path, file)
-        if not (path.is_file() or path.is_fifo() or str(path) != "-"):
+        if not (path.is_file() or path.is_fifo() or str(path) == "-"):
             print(f"not a regular file: {path}", file=sys.stderr)
             return 1
 
@@ -87,9 +89,18 @@ def main() -> None:
         sys.exit(_main())
     except (msgspec.DecodeError, msgspec.ValidationError) as e:
         print(f"invalid file format: {e}", file=sys.stderr)
+        sys.exit(1)
     # BrokenPipeError appears when stuff is piped to | head.
-    except (KeyboardInterrupt, BrokenPipeError):
-        pass
+    # This is not an error for hr.
+    except BrokenPipeError:
+        # https://docs.python.org/3/library/signal.html#note-on-sigpipe
+        # Python flushes standard streams on exit; redirect remaining output
+        # to devnull to avoid another BrokenPipeError at shutdown.
+        devnull = os.open(os.devnull, os.O_WRONLY)
+        os.dup2(devnull, sys.stdout.fileno())
+        sys.exit(0)
+    except KeyboardInterrupt:
+        sys.exit(128 + signal.SIGINT)
 
 
 if __name__ == "__main__":

tests/bats/001-invocation.bats

@@ -0,0 +1,22 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load helpers.bash
+
+@test "invoke gallia without parameters" {
+	# Should fail and print help page.
+	run -64 gallia
+}
+
+@test "invoke gallia without config" {
+	run -1 gallia --show-config
+}
+
+@test "invoke gallia with config" {
+	setup_gallia_toml
+	gallia --show-config
+	rm_gallia_toml
+}

tests/bats/002-scans.bats

@@ -0,0 +1,49 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load helpers.bash
+
+setup_file() {
+	setup_gallia_toml
+}
+
+@test "scan services" {
+	gallia scan uds services --sessions 1 2 --check-session
+}
+
+@test "scan sessions" {
+	gallia scan uds sessions --depth 2
+}
+
+@test "scan fast" {
+	gallia scan uds sessions --fast
+}
+
+@test "scan identifiers sid 0x22" {
+	gallia scan uds identifiers --start 0 --end 100 --sid 0x22
+}
+
+@test "scan identifiers sid 0x2e" {
+	gallia scan uds identifiers --start 0 --end 100 --sid 0x2e
+}
+
+@test "scan identifiers sid 0x31" {
+	gallia scan uds identifiers --start 0 --end 100 --sid 0x31
+}
+
+@test "scan reset" {
+	gallia scan uds reset
+}
+
+@test "scan dump-seeds" {
+	gallia scan uds dump-seeds --duration 0.01 --level 0x2f
+}
+
+@test "scan memory" {
+	for sid in 0x23 0x34 0x35 0x3d; do
+		gallia scan uds memory --sid "$sid"
+	done
+}

tests/bats/003-primitives.bats

@@ -0,0 +1,55 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+load helpers.bash
+
+setup_file() {
+	setup_gallia_toml
+}
+
+@test "primitive ecu-reset" {
+	gallia primitive uds ecu-reset
+}
+
+@test "primitive vin" {
+	gallia primitive uds vin
+}
+
+@test "primitive ping" {
+	gallia primitive uds ping --count 2
+}
+
+@test "primitive rdbi" {
+	gallia primitive uds rdbi 0x108d
+}
+
+@test "primitive pdu" {
+	gallia primitive uds pdu 1001
+}
+
+@test "primitive wdbi" {
+	gallia primitive uds wdbi 0x2266 --data 00
+}
+
+@test "primitive dtc read" {
+	gallia primitive uds dtc read
+}
+
+@test "primitive dtc clear" {
+	gallia primitive uds dtc clear
+}
+
+@test "primitive dtc control stop" {
+	gallia primitive uds dtc control --stop
+}
+
+@test "primitive dtc control resume" {
+	gallia primitive uds dtc control --resume
+}
+
+@test "primitive iocbi reset-to-default" {
+	gallia primitive uds iocbi 0x1000 reset-to-default
+}

tests/bats/100-hr.bats

@@ -0,0 +1,54 @@
+#!/usr/bin/env bats
+
+# SPDX-FileCopyrightText: AISEC Pentesting Team
+#
+# SPDX-License-Identifier: Apache-2.0
+
+@test "read logs from .zst file" {
+	hr "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst"
+}
+
+@test "read logs from .gz file" {
+	zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" | gzip - >"$BATS_TMPDIR/log-01.json.gz"
+	hr "$BATS_TMPDIR/log-01.json.gz"
+}
+
+@test "read logs from stdin" {
+	zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" | hr -
+}
+
+@test "read multiple .zst files" {
+	hr "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst"
+}
+
+@test "read file with priority prefix" {
+	zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" | awk '{print "<6>" $0}' | hr -
+}
+
+@test "pipe invalid data" {
+	run -1 bash -c "echo 'invalid json' | hr -"
+}
+
+@test "pipe to head and handle SIGPIPE" {
+	hr "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst" | head
+}
+
+@test "filter priority" {
+	local additional_line
+	additional_line='{"module": "foo", "data": "I am the line!", "host": "kronos", "datetime":"2020-04-23T15:21:50.620310", "priority": 5, "version": 2}'
+	cat <(zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst") <(echo "$additional_line") | gzip - >"$BATS_TMPDIR/log.json.gz"
+
+	run -0 hr -p notice "$BATS_TMPDIR/log.json.gz"
+
+	[[ "$output" =~ "I am the line!" ]]
+}
+
+@test "filter priority with priority prefix" {
+	local additional_line
+	additional_line='<5>{"module": "foo", "data": "I am the line!", "host": "kronos", "datetime":"2020-04-23T15:21:50.620310", "priority": 5, "version": 2}'
+	cat <(zstdcat "$BATS_TEST_DIRNAME/testfiles/log-01.json.zst") <(echo "$additional_line") | gzip - >"$BATS_TMPDIR/log.json.gz"
+
+	run -0 hr -p notice "$BATS_TMPDIR/log.json.gz"
+
+	[[ "$output" =~ "I am the line!" ]]
+}