From dc4f41354b6ea6bf213934dd2f896fd047532c34 Mon Sep 17 00:00:00 2001 From: Jiri Jaburek Date: Thu, 19 Sep 2024 02:25:47 +0200 Subject: [PATCH] split oscap-debug to host OS sysctl-only scan + full VM scan The sysctl-only scan is much simpler and faster, but doesn't seem to reliably reproduce the freeze. The nested VM test runs much faster than a full host OS scan, and is able to easily reproduce it. Signed-off-by: Jiri Jaburek --- scanning/oscap-debug/helgrind.fmf | 8 ++ scanning/oscap-debug/helgrind.py | 26 +++++ scanning/oscap-debug/sysctl-only.fmf | 12 ++ scanning/oscap-debug/sysctl-only.py | 105 ++++++++++++++++++ .../oscap-debug/{main.fmf => vm-scan.fmf} | 7 +- scanning/oscap-debug/{test.py => vm-scan.py} | 21 ++-- 6 files changed, 164 insertions(+), 15 deletions(-) create mode 100644 scanning/oscap-debug/helgrind.fmf create mode 100755 scanning/oscap-debug/helgrind.py create mode 100644 scanning/oscap-debug/sysctl-only.fmf create mode 100755 scanning/oscap-debug/sysctl-only.py rename scanning/oscap-debug/{main.fmf => vm-scan.fmf} (81%) rename scanning/oscap-debug/{test.py => vm-scan.py} (85%) diff --git a/scanning/oscap-debug/helgrind.fmf b/scanning/oscap-debug/helgrind.fmf new file mode 100644 index 0000000..9c49f26 --- /dev/null +++ b/scanning/oscap-debug/helgrind.fmf @@ -0,0 +1,8 @@ +summary: Runs oscap via valgrind - helgrind +test: python3 -m lib.runtest ./helgrind.py +result: custom +environment+: + PYTHONPATH: ../.. +duration: 4h +require+: + - valgrind diff --git a/scanning/oscap-debug/helgrind.py b/scanning/oscap-debug/helgrind.py new file mode 100755 index 0000000..ab34391 --- /dev/null +++ b/scanning/oscap-debug/helgrind.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 + +from lib import util, results + + +profile = 'cis_workstation_l1' + +extra_debuginfos = [ + 'glibc', + 'openscap-scanner', + 'xmlsec1', + 'xmlsec1-openssl', + 'libtool-ltdl', + 'openssl-libs', +] + +util.subprocess_run(['dnf', '-y', 'debuginfo-install', *extra_debuginfos], check=True) + +oscap_cmd = [ + 'valgrind', '--tool=helgrind', '--', + 'oscap', 'xccdf', 'eval', '--profile', profile, '--progress', + util.get_datastream(), +] +util.subprocess_run(oscap_cmd) + +results.report_and_exit() diff --git a/scanning/oscap-debug/sysctl-only.fmf b/scanning/oscap-debug/sysctl-only.fmf new file mode 100644 index 0000000..c65862c --- /dev/null +++ b/scanning/oscap-debug/sysctl-only.fmf @@ -0,0 +1,12 @@ +summary: Runs oscap many times to hopefully reproduce a freeze +test: python3 -m lib.runtest ./sysctl-only.py +result: custom +environment+: + PYTHONPATH: ../.. +duration: 4h +require+: + - gdb +adjust: + - when: distro < rhel-9 + enabled: false + because: we need a fairly modern gdb diff --git a/scanning/oscap-debug/sysctl-only.py b/scanning/oscap-debug/sysctl-only.py new file mode 100755 index 0000000..023ae1b --- /dev/null +++ b/scanning/oscap-debug/sysctl-only.py @@ -0,0 +1,105 @@ +#!/usr/bin/python3 + +import time +import signal +import subprocess + +from lib import util, results, oscap + + +start_time = time.monotonic() + +profile = 'anssi_bp28_high' + +# sysctl rules only take about 1-2 seconds +oscap_timeout = 10 + +# unselect all rules in the specified profile, except for +# sysctl_* rules +ds = oscap.global_ds() +rules = ds.profiles[profile].rules +rules = {rule for rule in rules if not rule.startswith('sysctl_')} +oscap.unselect_rules(util.get_datastream(), 'scan-ds.xml', rules) + +extra_debuginfos = [ + 'glibc', + 'openscap-scanner', + 'xmlsec1', + 'xmlsec1-openssl', + 'libtool-ltdl', + 'openssl-libs', +] + +util.subprocess_run(['dnf', '-y', 'debuginfo-install', *extra_debuginfos], check=True) + +with open('gdb.script', 'w') as f: + f.write(util.dedent(''' + generate-core-file oscap.core + set logging file oscap-bt.txt + set logging overwrite on + set logging redirect on + set logging enabled on + thread apply all bt + set logging enabled off + ''')) + +oscap_cmd = [ + 'oscap', 'xccdf', 'eval', '--profile', profile, '--progress', 'scan-ds.xml', +] + +# run for all of the configured test duration, minus 600 seconds for safety +# (running gdb, compressing corefile which takes forever, etc.) +attempt = 1 +metadata = util.TestMetadata() +duration = metadata.duration_seconds() - oscap_timeout - 600 +util.log(f"trying to freeze oscap for {duration} total seconds") + +while time.monotonic() - start_time < duration: + oscap_proc = util.subprocess_Popen(oscap_cmd) + + try: + returncode = oscap_proc.wait(oscap_timeout) + if returncode not in [0,2]: + results.report( + 'fail', f'attempt:{attempt}', f"oscap failed with {returncode}", + ) + continue + + except subprocess.TimeoutExpired: + # figure out oscap PID on the remote system + pgrep = util.subprocess_run( + ['pgrep', '-n', 'oscap'], + stdout=subprocess.PIPE, universal_newlines=True, + ) + if pgrep.returncode != 0: + results.report( + 'warn', + f'attempt:{attempt}', + f"pgrep returned {pgrep.returncode}, oscap probably just finished " + "and we hit a rare race, moving on", + ) + continue + + oscap_pid = pgrep.stdout.strip() + + # attach gdb to that PID + util.subprocess_run( + ['gdb', '-n', '-batch', '-x', 'gdb.script', '-p', oscap_pid], + check=True, + ) + + util.subprocess_run(['xz', '-e', '-9', 'oscap.core'], check=True) + results.report( + 'fail', f'attempt:{attempt}', "oscap froze, gdb output available", + logs=['oscap.core.xz', 'oscap-bt.txt'], + ) + break + + finally: + oscap_proc.send_signal(signal.SIGKILL) + oscap_proc.wait() + + results.report('pass', f'attempt:{attempt}') + attempt += 1 + +results.report_and_exit() diff --git a/scanning/oscap-debug/main.fmf b/scanning/oscap-debug/vm-scan.fmf similarity index 81% rename from scanning/oscap-debug/main.fmf rename to scanning/oscap-debug/vm-scan.fmf index 783d71b..8fbf3d5 100644 --- a/scanning/oscap-debug/main.fmf +++ b/scanning/oscap-debug/vm-scan.fmf @@ -1,11 +1,9 @@ summary: Runs oscap many times to hopefully reproduce a freeze -test: python3 -m lib.runtest ./test.py +test: python3 -m lib.runtest ./vm-scan.py result: custom environment+: PYTHONPATH: ../.. duration: 4h -tag: - - needs-param require+: # virt library dependencies - libvirt-daemon @@ -25,3 +23,6 @@ adjust: - when: arch != x86_64 enabled: false because: we want to run virtualization on x86_64 only + - when: distro < rhel-9 + enabled: false + because: we need a fairly modern gdb diff --git a/scanning/oscap-debug/test.py b/scanning/oscap-debug/vm-scan.py similarity index 85% rename from scanning/oscap-debug/test.py rename to scanning/oscap-debug/vm-scan.py index 0767e4a..d6c403d 100755 --- a/scanning/oscap-debug/test.py +++ b/scanning/oscap-debug/vm-scan.py @@ -1,20 +1,16 @@ #!/usr/bin/python3 -import os import time import subprocess import tempfile -from lib import util, results, virt, oscap -from conf import remediation, partitions +from lib import util, results, virt -profile = os.environ.get('PROFILE') -if not profile: - raise RuntimeError("specify PROFILE via env variable, consider also TIMEOUT") +profile = 'cis_workstation_l1' -# should be set to approximate the profile scan time -oscap_timeout = int(os.environ.get('TIMEOUT', 600)) +# cis_workstation_l1 takes about 4-5 seconds to scan +oscap_timeout = 30 extra_packages = [ 'gdb', @@ -26,6 +22,7 @@ 'xmlsec1', 'xmlsec1-openssl', 'libtool-ltdl', + 'openssl-libs', ] start_time = time.monotonic() @@ -65,10 +62,10 @@ oscap_cmd = f'oscap xccdf eval --profile {profile} --progress scan-ds.xml' while time.monotonic() - start_time < duration: - oscap = g.ssh(oscap_cmd, func=util.subprocess_Popen) + oscap_proc = g.ssh(oscap_cmd, func=util.subprocess_Popen) try: - returncode = oscap.wait(oscap_timeout) + returncode = oscap_proc.wait(oscap_timeout) if returncode not in [0,2]: results.report( 'fail', f'attempt:{attempt}', f"oscap failed with {returncode}", @@ -103,8 +100,8 @@ break finally: - oscap.terminate() - oscap.wait() + oscap_proc.terminate() + oscap_proc.wait() results.report('pass', f'attempt:{attempt}') attempt += 1