Merge pull request ComplianceAsCode#11292 from jpower432/add_oscal_cd…

…_generation

Adds utilities for OSCAL Component Definitions from policies

.github/workflows/gate.yaml

@@ -86,8 +86,10 @@ jobs:
         run: apt-get install -y ansible-lint bats check cmake libopenscap8 libxml2-utils ninja-build python3-github python3-pip xsltproc libxslt1-dev libxml2-dev zlib1g-dev
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Upgrade pip python
+        run: pip3 install --upgrade pip 
       - name: Install deps python
-        run: pip3 install -r requirements.txt -r test-requirements.txt
+        run: pip3 install -r requirements.txt -r test-requirements.txt --ignore-installed PyYAML
       - name: Build
         env:
           ADDITIONAL_CMAKE_OPTIONS: "-DSSG_ANSIBLE_PLAYBOOKS_PER_RULE_ENABLED=ON -DSSG_OVAL_SCHEMATRON_VALIDATION_ENABLED=OFF"
@@ -144,7 +146,7 @@ jobs:
       -   name: Run Updates
           run: dnf update -y
       -   name: Install Deps
-          run: dnf install -y cmake make openscap-utils bats ansible python3-pip ShellCheck git
+          run: dnf install -y cmake make openscap-utils bats ansible python3-pip ShellCheck git python3-devel gcc-c++
       -   name: Checkout
           uses: actions/checkout@v4
       -   name: Install deps python

.github/workflows/nightly_build.yml

@@ -11,9 +11,11 @@ jobs:
             image: fedora:latest
         steps:
             -   name: Install Dependencies
-                run: dnf install -y cmake ninja-build openscap-utils python3-pyyaml python3-jinja2 python3-pytest ansible-lint libxslt ansible
+                run: dnf install -y cmake ninja-build openscap-utils python3-pip python3-devel gcc-c++ ansible-lint libxslt ansible
             -   name: Checkout
                 uses: actions/checkout@v4
+            -   name: Install python deps
+                run: pip install -r requirements-base.txt -r test-requirements.txt
             -   name: Configure
                 run: cmake -G Ninja ..
                 working-directory: ./build

.github/workflows/release.yaml

@@ -10,9 +10,11 @@ jobs:
       image: fedora:latest
     steps:
       - name: Install Deps
-        run: dnf install -y cmake ninja-build openscap-utils python3-pyyaml python3-jinja2 python3-pytest ansible ansible-lint libxslt
+        run: dnf install -y cmake ninja-build openscap-utils python3-pip python3-devel gcc-c++ ansible ansible-lint libxslt
       - name: Checkout
         uses: actions/checkout@v4
+      - name: Install python deps
+        run: pip install -r requirements-base.txt -r test-requirements.txt
       - name: Configure
         run: cmake ..
         working-directory: ./build

CMakeLists.txt

@@ -162,6 +162,10 @@ find_python_module(myst_parser)
 # prometheus_metrics requirements
 find_python_module(prometheus_client)
 
+# compliance-trestle requirements
+find_python_module(trestle)
+find_python_module(requests)
+
 include(CMakeDependentOption)
 cmake_dependent_option(ENABLE_PYTHON_COVERAGE "Enable Python tests with coverage support" ON "PY_PYTEST_COV" OFF)
 
@@ -257,6 +261,7 @@ message(STATUS "python pandas module (optional): ${PY_PANDAS}")
 message(STATUS "python pcre2 module (optional): ${PY_PCRE2}")
 message(STATUS "python lxml module (optional): ${PY_LXML}")
 message(STATUS "python prometheus-client module (optional): ${PY_PROMETHEUS_CLIENT}")
+message(STATUS "python compliance-trestle module (optional): ${PY_TRESTLE}")
 message(STATUS " ")
 
 message(STATUS "Build options:")

docs/manual/developer/05_tools_and_utilities.md

@@ -684,3 +684,55 @@ To run the test (assuming that the content is built)
 $ cd build
 $ ctest --output-on-failure -R test-no-new-global-applicable-rules
 ```
+
+### `utils/oscal/build_cd_from_policy.py` – Build a Component Definition from a Policy
+
+This script builds an OSCAL Component Definition (cd) (version `1.0.4`) for an existing OSCAL profile from a policy. The script uses the
+[compliance-trestle](https://ibm.github.io/compliance-trestle/) library to build the component definition. The component definition can be used with the `compliance-trestle` CLI after generation.
+
+Some assumption made by this script:
+
+- The script maps control file statuses to valid OSCAL [statuses](https://pages.nist.gov/OSCAL-Reference/models/v1.1.1/system-security-plan/json-reference/#/system-security-plan/control-implementation/implemented-requirements/by-components/implementation-status) as follows:
+
+  * `pending` - `alternative`
+
+  * `not applicable`: `not-applicable`
+
+  * `inherently met`: `implemented`
+
+  * `documentation`: `implemented`
+
+  * `planned`: `planned`
+
+  * `partial`: `partial`
+
+  * `supported`: `implemented`
+
+  * `automated`: `implemented`
+
+  * `manual`: `alternative`
+
+  * `does not meet`: `alternative`
+
+- The script uses the "Section *letter*:" convention in the control notes to create statements under the implemented requirements.
+- The script maps parameter to rules uses the `xccdf_variable` field under `template.vars`
+- To determine what responses will mapped to the controls in the OSCAL profile the control id and label property from the resolved catalog is searched.
+
+It supports the following arguments:
+  - `-o`, `--output` — Path to write the cd to
+  - `-r`, `--root` — Root of the SSG project. Defaults to /content.
+  - `-v`, `--vendor-dir` — Path to the vendor directory with third party OSCAL artifacts
+  - `-p`, `--profile` — Main profile href, or name of the profile model in the trestle workspace
+  - `-pr`, `--product` — Product to build cd with
+  - `-c`, `--control` &mdash; Control to use as the source for control responses. To optionally filter by level, use the format <control_id>:<level>.
+  - `-j`, `--json` &mdash; Path to the rules_dir.json. Defaults to /content/build/rule_dirs.json.
+  - `-b`, `--build-config-yaml` &mdash; YAML file with information about the build configuration
+  - `-t`, `--component-definition-type` &mdash; Type of component definition to create. Defaults to service. Options are service or validation.
+
+An example of how to execute the script:
+
+```bash
+$ ./build_product ocp4
+$ ./utils/rule_dir_json.py
+$ ./utils/oscal/build_cd_from_policy.py -o build/ocp4.json -p fedramp_rev4_high -pr ocp4 -c nist_ocp4:high
+```

requirements.txt

@@ -9,6 +9,10 @@ ruamel.yaml
 pandas
 mypy
 lxml
+lxml-stubs
 pycompliance
 # used in utils/controleval_metrics.py
 prometheus_client
+# used in utils/oscal
+requests
+compliance-trestle==2.4.0

test-requirements.txt

@@ -1,3 +1,8 @@
 xmldiff
 pytest
 pytest-cov
+# used in tests/units/utils/oscal
+lxml
+lxml-stubs
+requests
+compliance-trestle==2.4.0

tests/CMakeLists.txt

@@ -2,10 +2,16 @@ if(ENABLE_PYTHON_COVERAGE)
     set(PYTEST_COVERAGE_OPTIONS --cov-append --cov-report=xml --cov "${CMAKE_SOURCE_DIR}/ssg")
 endif()
 
-macro(ssg_python_unit_tests PYTHON_COMPONENT_ID RELATIVE_PYTHONPATH)
+macro(ssg_python_unit_tests PYTHON_COMPONENT_ID RELATIVE_PYTHONPATH IGNORE_SUBDIR)
+    if(NOT "${IGNORE_SUBDIR}" STREQUAL "")
+        set(IGNORE "${CMAKE_SOURCE_DIR}/tests/unit/${PYTHON_COMPONENT_ID}/${IGNORE_SUBDIR}")
+        set(TEST_COMMAND "${PYTHON_EXECUTABLE}" -m pytest "--ignore=${IGNORE}" ${PYTEST_COVERAGE_OPTIONS} "${CMAKE_SOURCE_DIR}/tests/unit/${PYTHON_COMPONENT_ID}")
+    else()
+        set(TEST_COMMAND "${PYTHON_EXECUTABLE}" -m pytest ${PYTEST_COVERAGE_OPTIONS} "${CMAKE_SOURCE_DIR}/tests/unit/${PYTHON_COMPONENT_ID}")
+    endif()
     add_test(
         NAME "python-unit-${PYTHON_COMPONENT_ID}"
-        COMMAND "${PYTHON_EXECUTABLE}" -m pytest ${PYTEST_COVERAGE_OPTIONS} "${CMAKE_SOURCE_DIR}/tests/unit/${PYTHON_COMPONENT_ID}"
+        COMMAND ${TEST_COMMAND}
     )
     set_tests_properties("python-unit-${PYTHON_COMPONENT_ID}" PROPERTIES ENVIRONMENT
         "PYTHONPATH=${CMAKE_SOURCE_DIR}/${RELATIVE_PYTHONPATH}:$ENV{PYTHONPATH}"
@@ -14,9 +20,12 @@ macro(ssg_python_unit_tests PYTHON_COMPONENT_ID RELATIVE_PYTHONPATH)
 endmacro()
 
 if(PY_PYTEST)
-    ssg_python_unit_tests("utils" "utils")
-    ssg_python_unit_tests("ssg-module" ".")
-    ssg_python_unit_tests("ssg_test_suite" "tests")
+    ssg_python_unit_tests("utils" "utils" "oscal/")
+    ssg_python_unit_tests("ssg-module" "." "")
+    ssg_python_unit_tests("ssg_test_suite" "tests" "")
+    if(PYTHON_VERSION_MAJOR GREATER 2 AND PYTHON_VERSION_MINOR GREATER 7 AND PY_TRESTLE AND PY_LXML)
+        ssg_python_unit_tests("utils/oscal" "utils/oscal" "")
+    endif()
 endif()
 
 add_test(
@@ -83,21 +92,24 @@ set_tests_properties("fix_rules" PROPERTIES LABELS quick)
 set_tests_properties("fix_rules" PROPERTIES DEPENDS "test-rule-dir-json")
 set_tests_properties("fix_rules" PROPERTIES FIXTURES_REQUIRED "rule-dir-json")
 
-macro(mypy_test SCRIPT)
+macro(mypy_test SCRIPT FOLLOW_IMPORTS)
     if(PY_MYPY)
         add_test(
             NAME "test-mypy-${SCRIPT}"
-            COMMAND env "${PYTHON_EXECUTABLE}" -m mypy "${CMAKE_SOURCE_DIR}/${SCRIPT}"
+            COMMAND env "${PYTHON_EXECUTABLE}" -m mypy "${CMAKE_SOURCE_DIR}/${SCRIPT}" "--follow-imports=${FOLLOW_IMPORTS}"
         )
         set_tests_properties("test-mypy-${SCRIPT}" PROPERTIES LABELS quick)
         set_tests_properties("test-mypy-${SCRIPT}" PROPERTIES LABELS mypy)
     endif()
 endmacro()
 
-mypy_test("utils/import_srg_spreadsheet.py")
-mypy_test("utils/check_eof.py")
-mypy_test("utils/no_new_global_applicable_rules.py")
+mypy_test("utils/import_srg_spreadsheet.py" "normal")
+mypy_test("utils/check_eof.py" "normal")
+mypy_test("utils/no_new_global_applicable_rules.py" "normal")
 
+if(PYTHON_VERSION_MAJOR GREATER 2 AND PYTHON_VERSION_MINOR GREATER 7 AND PY_TRESTLE AND PY_LXML)
+    mypy_test("utils/oscal/" "skip")
+endif()
 
 if(PYTHON_VERSION_MAJOR GREATER 2 AND PY_OPENPYXL AND PY_PANDAS AND SSG_PRODUCT_RHEL9)
     add_test(

tests/unit/utils/oscal/data/build-config.yml

@@ -0,0 +1,10 @@
+cmake_build_type: "Release"
+
+ssg_version: [0, 1, 59]
+ssg_version_str: "0.1.59"
+target_oval_version: [5, 11]
+target_oval_version_str: "5.11"
+
+jinja2_cache_enabled: false
+
+sce_enabled: "OFF"

tests/unit/utils/oscal/data/rule_dirs.json

@@ -0,0 +1 @@
+{"test_rule": {"id": "test_rule", "dir": "test_rule_dir", "guide": "test_rule_guide", "products": ["test_product"], "title": "My Test Rule", "identifiers": {}, "ovals": {}, "remediations": {"anaconda": {}, "ansible": {}, "bash": {}, "puppet": {}, "ignition": {}, "kubernetes": {}, "blueprint": {}}, "oval_products": {}, "remediation_products": {}}}