From 15d84967e018e9d9880d178dcc25d490c713bc84 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Fri, 31 May 2024 22:41:43 +0000 Subject: [PATCH 01/12] feat(validate): establish threshold for assessment results result --- demo/oscal-component-opa.yaml | 12 +++ docs/evaluate.md | 21 +++++ src/cmd/evaluate/evaluate.go | 97 ++++++++++++++++++---- src/pkg/common/oscal/assessment-results.go | 32 ++++++- 4 files changed, 147 insertions(+), 15 deletions(-) create mode 100644 docs/evaluate.md diff --git a/demo/oscal-component-opa.yaml b/demo/oscal-component-opa.yaml index 2afc71b5..8458890d 100644 --- a/demo/oscal-component-opa.yaml +++ b/demo/oscal-component-opa.yaml @@ -37,6 +37,18 @@ component-definition: links: - href: '#a7377430-2328-4dc4-a9e2-b3f31dc1dff9' rel: lula + - uuid: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 + source: https://raw.githubusercontent.com/usnistgov/oscal-content/master/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json + description: + Validate generic security requirements + implemented-requirements: + - uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD + control-id: ID-2 + description: >- + This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + links: + - href: '#a7377430-2328-4dc4-a9e2-b3f31dc1dff9' + rel: lula back-matter: resources: - uuid: a7377430-2328-4dc4-a9e2-b3f31dc1dff9 diff --git a/docs/evaluate.md b/docs/evaluate.md new file mode 100644 index 00000000..018292b8 --- /dev/null +++ b/docs/evaluate.md @@ -0,0 +1,21 @@ +# Compliance Evaluation + +Evaluate serves as a method for verifying the compliance of a component/system against an established threshold to determine if it is more or less compliant than a previous assessment. + +## Expected Process + +### No Existing Data + +When no previous assessment exists, the initial assessment is made and stored with `lula validate`. This initial assessment by itself will always pass `lula evaluate` as there is no threshold for evaluation. Lula will automatically apply the `threshold` prop to the assessment result when writing the assessment result to a file that does not contain an existing assessment results artifact. + +steps: +1. `lula validate` +2. `lula evaluate` -> Passes with no Threshold + +### Existing Data (Intended Workflow) + +In workflows run manually or with automation (such as CI/CD), there is an expectation that the threshold exists, and evaluate will perform an analysis of the compliance of the system/component against the established threshold. + +steps: +1. `lula validate` +2. `lula evaluate` -> Passes or Fails based on threshold diff --git a/src/cmd/evaluate/evaluate.go b/src/cmd/evaluate/evaluate.go index 3a3ad838..48a4897b 100644 --- a/src/cmd/evaluate/evaluate.go +++ b/src/cmd/evaluate/evaluate.go @@ -15,7 +15,7 @@ var evaluateHelp = ` To evaluate the latest results in two assessment results files: lula evaluate -f assessment-results-threshold.yaml -f assessment-results-new.yaml -To evaluate two results (latest and preceding) in a single assessment results file: +To evaluate two results (threshold and latest) in a single OSCAL file: lula evaluate -f assessment-results.yaml ` @@ -51,11 +51,16 @@ func EvaluateCommand() *cobra.Command { func EvaluateAssessmentResults(fileArray []string) error { var status bool var findings map[string][]oscalTypes_1_1_2.Finding + var threshold, latest *oscalTypes_1_1_2.Result + + // Items for updating the threshold automatically + var thresholdFile string + var thresholdAssessment *oscalTypes_1_1_2.AssessmentResults // Read in files - establish the results to if len(fileArray) == 0 { // TODO: Determine if we will handle a default location/name for assessment files - return fmt.Errorf("No files provided for evaluation") + return fmt.Errorf("no files provided for evaluation") } for _, f := range fileArray { @@ -66,6 +71,7 @@ func EvaluateAssessmentResults(fileArray []string) error { } if len(fileArray) == 1 { + thresholdFile = fileArray[0] data, err := common.ReadFileToBytes(fileArray[0]) if err != nil { return err @@ -75,16 +81,25 @@ func EvaluateAssessmentResults(fileArray []string) error { return err } if len(assessment.Results) < 2 { - return fmt.Errorf("2 or more result objects must be present for evaluation\n") + message.Infof("%v result object identified - unable to evaluate", len(assessment.Results)) + return nil + } + + // Identify the threshold + threshold, err = findThreshold(&assessment.Results) + if err != nil { + return err } - // We write results to the assessment-results report in newest -> oldest - // Older being our threshold here - status, findings, err = EvaluateResults(&assessment.Results[1], &assessment.Results[0]) + + latest = &assessment.Results[0] + + status, findings, err = EvaluateResults(threshold, latest) if err != nil { return err } } else if len(fileArray) == 2 { + thresholdFile = fileArray[1] data, err := common.ReadFileToBytes(fileArray[0]) if err != nil { return err @@ -110,29 +125,48 @@ func EvaluateAssessmentResults(fileArray []string) error { return err } } else { - return fmt.Errorf("Exceeded maximum of 2 files for evaluation\n") + return fmt.Errorf("exceeded maximum of 2 files for evaluation") } if status { - message.Info("Evaluation Passing the established threshold") - if len(findings["new-findings"]) > 0 { - message.Info("New finding Target-Ids:") - for _, finding := range findings["new-findings"] { + if len(findings["new-passing-findings"]) > 0 { + message.Info("New passing finding Target-Ids:") + for _, finding := range findings["new-passing-findings"] { message.Infof("%s", finding.Target.TargetId) } + // TODO: If there are new passing Findings -> update the threshold in the assessment + updateProp("threshold", "false", threshold) + updateProp("threshold", "true", latest) + + // Props are updated - now write the thresholdAssessment to the existing assessment? + // if we create the model and write it - the merge will need to de-duplicate instead of merge results + model := oscalTypes_1_1_2.OscalCompleteSchema{ + AssessmentResults: thresholdAssessment, + } + + oscal.WriteOscalModel(thresholdFile, &model) + } + + if len(findings["new-failing-findings"]) > 0 { + message.Info("New failing finding Target-Ids:") + for _, finding := range findings["new-failing-findings"] { + message.Infof("%s", finding.Target.TargetId) + } + } + return nil } else { message.Warn("Evaluation Failed against the following findings:") for _, finding := range findings["no-longer-satisfied"] { message.Warnf("%s", finding.Target.TargetId) } - return fmt.Errorf("Failed to meet established threshold") + return fmt.Errorf("failed to meet established threshold") } } func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalTypes_1_1_2.Result) (bool, map[string][]oscalTypes_1_1_2.Finding, error) { - if thresholdResult == nil || thresholdResult.Findings == nil || newResult == nil || newResult.Findings == nil { + if thresholdResult.Findings == nil || newResult.Findings == nil { return false, nil, fmt.Errorf("results must contain findings to evaluate") } @@ -144,7 +178,9 @@ func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalT result := true findingMapThreshold := oscal.GenerateFindingsMap(*thresholdResult.Findings) + message.Debug(findingMapThreshold) findingMapNew := oscal.GenerateFindingsMap(*newResult.Findings) + message.Debug(findingMapNew) // For a given oldResult - we need to prove that the newResult implements all of the oldResult findings/controls // We are explicitly iterating through the findings in order to collect a delta to display @@ -169,11 +205,44 @@ func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalT } } + message.Debug(findingMapNew) + // All remaining findings in the new map are new findings for _, finding := range findingMapNew { - findings["new-findings"] = append(findings["new-findings"], finding) + if finding.Target.Status.State == "satisfied" { + message.Debugf("New finding to append: %s", finding.Target.TargetId) + findings["new-passing-findings"] = append(findings["new-passing-findings"], finding) + } else { + findings["new-failing-findings"] = append(findings["new-failing-findings"], finding) + } + } spinner.Success() return result, findings, nil } + +func findThreshold(results *[]oscalTypes_1_1_2.Result) (*oscalTypes_1_1_2.Result, error) { + for _, result := range *results { + for _, prop := range *result.Props { + if prop.Name == "threshold" { + if prop.Value == "true" { + return &result, nil + } + } + } + } + return &oscalTypes_1_1_2.Result{}, fmt.Errorf("threshold not found") +} + +func updateProp(name string, value string, result *oscalTypes_1_1_2.Result) error { + for index, prop := range *result.Props { + if prop.Name == name { + prop.Value = value + (*result.Props)[index] = prop + message.Debug(*result) + return nil + } + } + return fmt.Errorf("property not found") +} diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index 0d13c401..20fa202a 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -61,6 +61,15 @@ func GenerateAssessmentResults(findingMap map[string]oscalTypes_1_1_2.Finding, o LastModified: rfc3339Time, } + // Here we are going to add the threshold property + props := []oscalTypes_1_1_2.Property{ + { + Ns: "https://docs.lula.dev/ns", + Name: "threshold", + Value: "true", + }, + } + // Create results object assessmentResults.Results = []oscalTypes_1_1_2.Result{ { @@ -68,6 +77,7 @@ func GenerateAssessmentResults(findingMap map[string]oscalTypes_1_1_2.Finding, o Title: "Lula Validation Result", Start: rfc3339Time, Description: "Assessment results for performing Validations with Lula version " + config.CLIVersion, + Props: &props, ReviewedControls: oscalTypes_1_1_2.ReviewedControls{ Description: "Controls validated", Remarks: "Validation performed may indicate full or partial satisfaction", @@ -88,13 +98,33 @@ func GenerateAssessmentResults(findingMap map[string]oscalTypes_1_1_2.Finding, o func MergeAssessmentResults(original *oscalTypes_1_1_2.AssessmentResults, latest *oscalTypes_1_1_2.AssessmentResults) (*oscalTypes_1_1_2.AssessmentResults, error) { + // If UUID's are matching - this must be a prop update for threshold + // We should be able to return the latest results + // This is used during evaluate to update the threshold prop automatically + if original.UUID == latest.UUID { + return latest, nil + } + + // Validate only ever creates one result + // Assumed that there is always an original threshold + result := latest.Results[0] + for index, prop := range *result.Props { + if prop.Name == "threshold" { + prop.Value = "false" + // Better way to update the prop? + (*result.Props)[index] = prop + } + } + results := make([]oscalTypes_1_1_2.Result, 0) // append newest to oldest results - results = append(results, latest.Results...) + results = append(results, result) results = append(results, original.Results...) original.Results = results + // Update pertinent information original.Metadata.LastModified = time.Now() + original.UUID = uuid.NewUUID() return original, nil } From 4db09f989e92b670d62b31a152613636b3361944 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Sun, 2 Jun 2024 17:38:55 +0000 Subject: [PATCH 02/12] feat(evaluate): support for single artifact props update --- demo/oscal-component-opa-2.yaml | 81 ++++++++++++++++++++++++ demo/oscal-component-opa.yaml | 12 ---- docs/evaluate.md | 23 +++++++ oscal.yaml | 107 ++++++++++++++++++++++++++++++++ src/cmd/evaluate/evaluate.go | 33 ++++++---- 5 files changed, 233 insertions(+), 23 deletions(-) create mode 100644 demo/oscal-component-opa-2.yaml create mode 100644 oscal.yaml diff --git a/demo/oscal-component-opa-2.yaml b/demo/oscal-component-opa-2.yaml new file mode 100644 index 00000000..8458890d --- /dev/null +++ b/demo/oscal-component-opa-2.yaml @@ -0,0 +1,81 @@ +component-definition: + uuid: E6A291A4-2BC8-43A0-B4B2-FD67CAAE1F8F + metadata: + title: Lula Demo + last-modified: '2022-09-13T12:00:00Z' + version: "20220913" + oscal-version: 1.1.1 # This version should remain one version behind latest version for `lula dev upgrade` demo + parties: + # Should be consistent across all of the packages, but where is ground truth? + - uuid: C18F4A9F-A402-415B-8D13-B51739D689FF + type: organization + name: Lula Development + links: + - href: https://github.com/defenseunicorns/lula + rel: website + components: + - uuid: A9D5204C-7E5B-4C43-BD49-34DF759B9F04 + type: software + title: lula + description: | + Lula - the Compliance Validator + purpose: Validate compliance controls + responsible-roles: + - role-id: provider + party-uuids: + - C18F4A9F-A402-415B-8D13-B51739D689FF # matches parties entry for Defense Unicorns + control-implementations: + - uuid: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A + source: https://raw.githubusercontent.com/usnistgov/oscal-content/master/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json + description: + Validate generic security requirements + implemented-requirements: + - uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD + control-id: ID-1 + description: >- + This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + links: + - href: '#a7377430-2328-4dc4-a9e2-b3f31dc1dff9' + rel: lula + - uuid: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 + source: https://raw.githubusercontent.com/usnistgov/oscal-content/master/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json + description: + Validate generic security requirements + implemented-requirements: + - uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD + control-id: ID-2 + description: >- + This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + links: + - href: '#a7377430-2328-4dc4-a9e2-b3f31dc1dff9' + rel: lula + back-matter: + resources: + - uuid: a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + rlinks: + - href: lula.dev + description: >- + domain: + type: kubernetes + kubernetes-spec: + resources: + - name: podsvt + resource-rule: + group: + version: v1 + resource: pods + namespaces: [validation-test] + provider: + type: opa + opa-spec: + rego: | + package validate + + import future.keywords.every + + validate { + every pod in input.podsvt { + podLabel := pod.metadata.labels.foo + podLabel == "bar" + } + } \ No newline at end of file diff --git a/demo/oscal-component-opa.yaml b/demo/oscal-component-opa.yaml index 8458890d..2afc71b5 100644 --- a/demo/oscal-component-opa.yaml +++ b/demo/oscal-component-opa.yaml @@ -37,18 +37,6 @@ component-definition: links: - href: '#a7377430-2328-4dc4-a9e2-b3f31dc1dff9' rel: lula - - uuid: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 - source: https://raw.githubusercontent.com/usnistgov/oscal-content/master/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json - description: - Validate generic security requirements - implemented-requirements: - - uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD - control-id: ID-2 - description: >- - This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - links: - - href: '#a7377430-2328-4dc4-a9e2-b3f31dc1dff9' - rel: lula back-matter: resources: - uuid: a7377430-2328-4dc4-a9e2-b3f31dc1dff9 diff --git a/docs/evaluate.md b/docs/evaluate.md index 018292b8..0a119156 100644 --- a/docs/evaluate.md +++ b/docs/evaluate.md @@ -1,3 +1,7 @@ +# TODO +- Evaluate functions again for updating prop values +- Determine multiple artifact evaluation process + # Compliance Evaluation Evaluate serves as a method for verifying the compliance of a component/system against an established threshold to determine if it is more or less compliant than a previous assessment. @@ -19,3 +23,22 @@ In workflows run manually or with automation (such as CI/CD), there is an expect steps: 1. `lula validate` 2. `lula evaluate` -> Passes or Fails based on threshold + + +## Scenarios for Consideration + +Evaluate will determine which result is the threshold based on the following property: +```yaml +props: + - name: threshold + ns: https://docs.lula.dev/ns + value: "true/false" +``` + +### Assessment Results Artifact + +When evaluate is ran with a single assessment results artifact, it is expected that a single threshold with a `true` value exists. This will be identified and ran against the latest result to determine if compliance is less-than-equal (fail), equal (pass), or greater-than-equal (pass). When the comparison results in greater-than-equal, Lula will update the threshold `prop` for the latest result to `true` and set the previous result threshold prop to `false`. + +### Comparing multiple assessment results artifacts + +In the scenario where multiple assessment results artifacts are evaluated, there may be a multiple threshold results with a `true` value as Lula establishes a default `true` value when writing an assessment results artifact to a new file with no previous results present. \ No newline at end of file diff --git a/oscal.yaml b/oscal.yaml new file mode 100644 index 00000000..6902719c --- /dev/null +++ b/oscal.yaml @@ -0,0 +1,107 @@ +assessment-results: + import-ap: + href: "" + metadata: + last-modified: 2024-06-02T16:51:34.514905968Z + oscal-version: 1.1.2 + published: 2024-06-02T16:50:33.857826017Z + remarks: Assessment Results generated from Lula + title: '[System Name] Security Assessment Results (SAR)' + version: 0.0.1 + results: + - description: Assessment results for performing Validations with Lula version v0.3.0-4-g17ac8ca + findings: + - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + related-observations: + - observation-uuid: 7dc08f8d-a680-4c6d-9a84-70d657553870 + target: + status: + state: satisfied + target-id: ID-1 + type: objective-id + title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' + uuid: 93cecceb-601d-4f89-bca0-a190ab457b58 + - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + related-observations: + - observation-uuid: 76838298-2822-40a1-8151-0e6d358a807f + target: + status: + state: satisfied + target-id: ID-2 + type: objective-id + title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 / Control: ID-2' + uuid: 540be846-8659-48fe-a445-1a4832143337 + observations: + - collected: 2024-06-02T16:51:34.433850778Z + description: |+ + [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + + methods: + - TEST + relevant-evidence: + - description: | + Result: satisfied + uuid: 7dc08f8d-a680-4c6d-9a84-70d657553870 + - collected: 2024-06-02T16:51:34.437794612Z + description: |+ + [TEST]: ID-2 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + + methods: + - TEST + relevant-evidence: + - description: | + Result: satisfied + uuid: 76838298-2822-40a1-8151-0e6d358a807f + props: + - name: threshold + ns: https://docs.lula.dev/ns + value: "true" + reviewed-controls: + control-selections: + - description: Controls Assessed by Lula + include-controls: + - control-id: ID-1 + - control-id: ID-2 + description: Controls validated + remarks: Validation performed may indicate full or partial satisfaction + start: 2024-06-02T16:51:34.43939799Z + title: Lula Validation Result + uuid: 3844a746-1b56-48a5-b6f0-c9c46b69f104 + - description: Assessment results for performing Validations with Lula version v0.3.0-4-g17ac8ca + findings: + - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + related-observations: + - observation-uuid: f961dff4-04bd-4a52-9377-e8a256641a2e + target: + status: + state: satisfied + target-id: ID-1 + type: objective-id + title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' + uuid: d9ab2712-6737-4e24-981f-858bf3c8e178 + observations: + - collected: 2024-06-02T16:50:33.852605521Z + description: |+ + [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + + methods: + - TEST + relevant-evidence: + - description: | + Result: satisfied + uuid: f961dff4-04bd-4a52-9377-e8a256641a2e + props: + - name: threshold + ns: https://docs.lula.dev/ns + value: "false" + reviewed-controls: + control-selections: + - description: Controls Assessed by Lula + include-controls: + - control-id: ID-1 + description: Controls validated + remarks: Validation performed may indicate full or partial satisfaction + start: 2024-06-02T16:50:33.857826017Z + title: Lula Validation Result + uuid: 8238b693-53c2-47e6-b151-dab9bc4e1bed + uuid: fbc4885c-123d-48ce-86a2-c2a78f0e2778 diff --git a/src/cmd/evaluate/evaluate.go b/src/cmd/evaluate/evaluate.go index 48a4897b..2d36b7a0 100644 --- a/src/cmd/evaluate/evaluate.go +++ b/src/cmd/evaluate/evaluate.go @@ -55,7 +55,7 @@ func EvaluateAssessmentResults(fileArray []string) error { // Items for updating the threshold automatically var thresholdFile string - var thresholdAssessment *oscalTypes_1_1_2.AssessmentResults + var assessment *oscalTypes_1_1_2.AssessmentResults // Read in files - establish the results to if len(fileArray) == 0 { @@ -76,7 +76,7 @@ func EvaluateAssessmentResults(fileArray []string) error { if err != nil { return err } - assessment, err := oscal.NewAssessmentResults(data) + assessment, err = oscal.NewAssessmentResults(data) if err != nil { return err } @@ -135,13 +135,15 @@ func EvaluateAssessmentResults(fileArray []string) error { message.Infof("%s", finding.Target.TargetId) } // TODO: If there are new passing Findings -> update the threshold in the assessment - updateProp("threshold", "false", threshold) - updateProp("threshold", "true", latest) + message.Debugf("props before update: %v", threshold.Props) + updateProp("threshold", "false", &threshold.Props) + message.Debugf("props after update: %v", threshold.Props) + updateProp("threshold", "true", &latest.Props) // Props are updated - now write the thresholdAssessment to the existing assessment? // if we create the model and write it - the merge will need to de-duplicate instead of merge results model := oscalTypes_1_1_2.OscalCompleteSchema{ - AssessmentResults: thresholdAssessment, + AssessmentResults: assessment, } oscal.WriteOscalModel(thresholdFile, &model) @@ -178,9 +180,7 @@ func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalT result := true findingMapThreshold := oscal.GenerateFindingsMap(*thresholdResult.Findings) - message.Debug(findingMapThreshold) findingMapNew := oscal.GenerateFindingsMap(*newResult.Findings) - message.Debug(findingMapNew) // For a given oldResult - we need to prove that the newResult implements all of the oldResult findings/controls // We are explicitly iterating through the findings in order to collect a delta to display @@ -235,14 +235,25 @@ func findThreshold(results *[]oscalTypes_1_1_2.Result) (*oscalTypes_1_1_2.Result return &oscalTypes_1_1_2.Result{}, fmt.Errorf("threshold not found") } -func updateProp(name string, value string, result *oscalTypes_1_1_2.Result) error { - for index, prop := range *result.Props { +func updateProp(name string, value string, props **[]oscalTypes_1_1_2.Property) error { + + for index, prop := range **props { if prop.Name == name { prop.Value = value - (*result.Props)[index] = prop - message.Debug(*result) + (**props)[index] = prop + message.Debug(prop) return nil } } return fmt.Errorf("property not found") + + // for index, prop := range *result.Props { + // if prop.Name == name { + // prop.Value = value + // (*result.Props)[index] = prop + // message.Debug(*result) + // return nil + // } + // } + // return fmt.Errorf("property not found") } From fc18c4319e6c3140f0d2b3c08ecae30bf4f37145 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Mon, 3 Jun 2024 23:03:58 +0000 Subject: [PATCH 03/12] fix(evaluate): perform evaluation with parity for multiple files --- docs/evaluate.md | 6 +- oscal.yaml | 107 -------------- src/cmd/evaluate/evaluate.go | 153 ++++++++++----------- src/cmd/evaluate/evaluate_test.go | 4 +- src/pkg/common/oscal/assessment-results.go | 1 + 5 files changed, 75 insertions(+), 196 deletions(-) delete mode 100644 oscal.yaml diff --git a/docs/evaluate.md b/docs/evaluate.md index 0a119156..8ed5d570 100644 --- a/docs/evaluate.md +++ b/docs/evaluate.md @@ -1,7 +1,3 @@ -# TODO -- Evaluate functions again for updating prop values -- Determine multiple artifact evaluation process - # Compliance Evaluation Evaluate serves as a method for verifying the compliance of a component/system against an established threshold to determine if it is more or less compliant than a previous assessment. @@ -41,4 +37,4 @@ When evaluate is ran with a single assessment results artifact, it is expected t ### Comparing multiple assessment results artifacts -In the scenario where multiple assessment results artifacts are evaluated, there may be a multiple threshold results with a `true` value as Lula establishes a default `true` value when writing an assessment results artifact to a new file with no previous results present. \ No newline at end of file +In the scenario where multiple assessment results artifacts are evaluated, there may be a multiple threshold results with a `true` value as Lula establishes a default `true` value when writing an assessment results artifact to a new file with no previous results present. In this case, Lula will use the older result as the threshold to determine compliance of the result. \ No newline at end of file diff --git a/oscal.yaml b/oscal.yaml deleted file mode 100644 index 6902719c..00000000 --- a/oscal.yaml +++ /dev/null @@ -1,107 +0,0 @@ -assessment-results: - import-ap: - href: "" - metadata: - last-modified: 2024-06-02T16:51:34.514905968Z - oscal-version: 1.1.2 - published: 2024-06-02T16:50:33.857826017Z - remarks: Assessment Results generated from Lula - title: '[System Name] Security Assessment Results (SAR)' - version: 0.0.1 - results: - - description: Assessment results for performing Validations with Lula version v0.3.0-4-g17ac8ca - findings: - - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - related-observations: - - observation-uuid: 7dc08f8d-a680-4c6d-9a84-70d657553870 - target: - status: - state: satisfied - target-id: ID-1 - type: objective-id - title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' - uuid: 93cecceb-601d-4f89-bca0-a190ab457b58 - - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - related-observations: - - observation-uuid: 76838298-2822-40a1-8151-0e6d358a807f - target: - status: - state: satisfied - target-id: ID-2 - type: objective-id - title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 / Control: ID-2' - uuid: 540be846-8659-48fe-a445-1a4832143337 - observations: - - collected: 2024-06-02T16:51:34.433850778Z - description: |+ - [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - - methods: - - TEST - relevant-evidence: - - description: | - Result: satisfied - uuid: 7dc08f8d-a680-4c6d-9a84-70d657553870 - - collected: 2024-06-02T16:51:34.437794612Z - description: |+ - [TEST]: ID-2 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - - methods: - - TEST - relevant-evidence: - - description: | - Result: satisfied - uuid: 76838298-2822-40a1-8151-0e6d358a807f - props: - - name: threshold - ns: https://docs.lula.dev/ns - value: "true" - reviewed-controls: - control-selections: - - description: Controls Assessed by Lula - include-controls: - - control-id: ID-1 - - control-id: ID-2 - description: Controls validated - remarks: Validation performed may indicate full or partial satisfaction - start: 2024-06-02T16:51:34.43939799Z - title: Lula Validation Result - uuid: 3844a746-1b56-48a5-b6f0-c9c46b69f104 - - description: Assessment results for performing Validations with Lula version v0.3.0-4-g17ac8ca - findings: - - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - related-observations: - - observation-uuid: f961dff4-04bd-4a52-9377-e8a256641a2e - target: - status: - state: satisfied - target-id: ID-1 - type: objective-id - title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' - uuid: d9ab2712-6737-4e24-981f-858bf3c8e178 - observations: - - collected: 2024-06-02T16:50:33.852605521Z - description: |+ - [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - - methods: - - TEST - relevant-evidence: - - description: | - Result: satisfied - uuid: f961dff4-04bd-4a52-9377-e8a256641a2e - props: - - name: threshold - ns: https://docs.lula.dev/ns - value: "false" - reviewed-controls: - control-selections: - - description: Controls Assessed by Lula - include-controls: - - control-id: ID-1 - description: Controls validated - remarks: Validation performed may indicate full or partial satisfaction - start: 2024-06-02T16:50:33.857826017Z - title: Lula Validation Result - uuid: 8238b693-53c2-47e6-b151-dab9bc4e1bed - uuid: fbc4885c-123d-48ce-86a2-c2a78f0e2778 diff --git a/src/cmd/evaluate/evaluate.go b/src/cmd/evaluate/evaluate.go index 2d36b7a0..3169906a 100644 --- a/src/cmd/evaluate/evaluate.go +++ b/src/cmd/evaluate/evaluate.go @@ -2,6 +2,7 @@ package evaluate import ( "fmt" + "slices" "github.com/defenseunicorns/go-oscal/src/pkg/files" oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2" @@ -53,79 +54,65 @@ func EvaluateAssessmentResults(fileArray []string) error { var findings map[string][]oscalTypes_1_1_2.Finding var threshold, latest *oscalTypes_1_1_2.Result - // Items for updating the threshold automatically - var thresholdFile string - var assessment *oscalTypes_1_1_2.AssessmentResults - - // Read in files - establish the results to if len(fileArray) == 0 { - // TODO: Determine if we will handle a default location/name for assessment files return fmt.Errorf("no files provided for evaluation") } - for _, f := range fileArray { - err := files.IsJsonOrYaml(f) + // Potentially write changes back to multiple files requires some storage + resultMap := make(map[string]*oscalTypes_1_1_2.AssessmentResults) + for _, fileString := range fileArray { + err := files.IsJsonOrYaml(fileString) if err != nil { - return fmt.Errorf("invalid file extension: %s, requires .json or .yaml", f) + return fmt.Errorf("invalid file extension: %s, requires .json or .yaml", fileString) } - } - if len(fileArray) == 1 { - thresholdFile = fileArray[0] - data, err := common.ReadFileToBytes(fileArray[0]) + data, err := common.ReadFileToBytes(fileString) if err != nil { return err } - assessment, err = oscal.NewAssessmentResults(data) + assessment, err := oscal.NewAssessmentResults(data) if err != nil { return err } - if len(assessment.Results) < 2 { - message.Infof("%v result object identified - unable to evaluate", len(assessment.Results)) - return nil - } + resultMap[fileString] = assessment + } - // Identify the threshold - threshold, err = findThreshold(&assessment.Results) - if err != nil { - return err - } + // Now that we have the map of assessment results - we need to identify the threshold(s) + // Also sort the results -> if we maintain pointers, can we update and write all artifacts in one go? + + thresholds, sortedResults, err := findAndSortResults(resultMap) + if err != nil { + return err + } - latest = &assessment.Results[0] + if len(sortedResults) <= 1 { + // Should this implicitly pass? If so then a workflow can operate on the assumption that it will pass from 0 -> N results + message.Infof("%v result object identified - unable to evaluate", len(sortedResults)) + return nil + } + + if len(thresholds) == 0 { + // No thresholds identified but we have > 1 results - compare the latest and the preceding + threshold = sortedResults[len(sortedResults)-2] + latest = sortedResults[len(sortedResults)-1] status, findings, err = EvaluateResults(threshold, latest) if err != nil { return err } + } else { + // Constraint - Always evaluate the latest threshold against the latest result + threshold = thresholds[len(thresholds)-1] + latest = sortedResults[len(sortedResults)-1] - } else if len(fileArray) == 2 { - thresholdFile = fileArray[1] - data, err := common.ReadFileToBytes(fileArray[0]) - if err != nil { - return err - } - assessmentOne, err := oscal.NewAssessmentResults(data) - if err != nil { - return err - } - data, err = common.ReadFileToBytes(fileArray[1]) - if err != nil { - return err - } - assessmentTwo, err := oscal.NewAssessmentResults(data) - if err != nil { - return err + if threshold.UUID == latest.UUID { + // They are the same - return error + return fmt.Errorf("unable to evaluate - threshold and latest result are the same result - nothing to compare") } - - // Consider parsing the timestamps for comparison - // Older timestamp being the threshold - - status, findings, err = EvaluateResults(&assessmentOne.Results[0], &assessmentTwo.Results[0]) + status, findings, err = EvaluateResults(threshold, latest) if err != nil { return err } - } else { - return fmt.Errorf("exceeded maximum of 2 files for evaluation") } if status { @@ -134,19 +121,21 @@ func EvaluateAssessmentResults(fileArray []string) error { for _, finding := range findings["new-passing-findings"] { message.Infof("%s", finding.Target.TargetId) } - // TODO: If there are new passing Findings -> update the threshold in the assessment - message.Debugf("props before update: %v", threshold.Props) - updateProp("threshold", "false", &threshold.Props) - message.Debugf("props after update: %v", threshold.Props) - updateProp("threshold", "true", &latest.Props) - // Props are updated - now write the thresholdAssessment to the existing assessment? + message.Info("New threshold identified - threshold will be updated to latest result") + + updateProp("threshold", "false", threshold.Props) + updateProp("threshold", "true", latest.Props) + + // Props are updated - now write back to all files // if we create the model and write it - the merge will need to de-duplicate instead of merge results - model := oscalTypes_1_1_2.OscalCompleteSchema{ - AssessmentResults: assessment, - } + for filePath, assessment := range resultMap { + model := oscalTypes_1_1_2.OscalCompleteSchema{ + AssessmentResults: assessment, + } - oscal.WriteOscalModel(thresholdFile, &model) + oscal.WriteOscalModel(filePath, &model) + } } @@ -205,12 +194,9 @@ func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalT } } - message.Debug(findingMapNew) - // All remaining findings in the new map are new findings for _, finding := range findingMapNew { if finding.Target.Status.State == "satisfied" { - message.Debugf("New finding to append: %s", finding.Target.TargetId) findings["new-passing-findings"] = append(findings["new-passing-findings"], finding) } else { findings["new-failing-findings"] = append(findings["new-failing-findings"], finding) @@ -222,38 +208,41 @@ func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalT return result, findings, nil } -func findThreshold(results *[]oscalTypes_1_1_2.Result) (*oscalTypes_1_1_2.Result, error) { - for _, result := range *results { - for _, prop := range *result.Props { - if prop.Name == "threshold" { - if prop.Value == "true" { - return &result, nil +// findAndSortResults takes a map of results and returns a list of thresholds and a sorted list of results in order of time +func findAndSortResults(resultMap map[string]*oscalTypes_1_1_2.AssessmentResults) ([]*oscalTypes_1_1_2.Result, []*oscalTypes_1_1_2.Result, error) { + + thresholds := make([]*oscalTypes_1_1_2.Result, 0) + sortedResults := make([]*oscalTypes_1_1_2.Result, 0) + + for _, assessment := range resultMap { + for _, result := range assessment.Results { + if result.Props != nil { + for _, prop := range *result.Props { + if prop.Name == "threshold" && prop.Value == "true" { + thresholds = append(thresholds, &result) + } } } + // Store all results in a non-sorted list + sortedResults = append(sortedResults, &result) } } - return &oscalTypes_1_1_2.Result{}, fmt.Errorf("threshold not found") + + // Sort the results by start time + slices.SortFunc(sortedResults, func(a, b *oscalTypes_1_1_2.Result) int { return a.Start.Compare(b.Start) }) + slices.SortFunc(thresholds, func(a, b *oscalTypes_1_1_2.Result) int { return a.Start.Compare(b.Start) }) + + return thresholds, sortedResults, nil } -func updateProp(name string, value string, props **[]oscalTypes_1_1_2.Property) error { +func updateProp(name string, value string, props *[]oscalTypes_1_1_2.Property) error { - for index, prop := range **props { + for index, prop := range *props { if prop.Name == name { prop.Value = value - (**props)[index] = prop - message.Debug(prop) + (*props)[index] = prop return nil } } return fmt.Errorf("property not found") - - // for index, prop := range *result.Props { - // if prop.Name == name { - // prop.Value = value - // (*result.Props)[index] = prop - // message.Debug(*result) - // return nil - // } - // } - // return fmt.Errorf("property not found") } diff --git a/src/cmd/evaluate/evaluate_test.go b/src/cmd/evaluate/evaluate_test.go index ab73b52f..3186246b 100644 --- a/src/cmd/evaluate/evaluate_test.go +++ b/src/cmd/evaluate/evaluate_test.go @@ -192,8 +192,8 @@ func TestEvaluateResultsNewFindings(t *testing.T) { t.Fatal("error - evaluation failed") } - if len(findings["new-findings"]) != 2 { - t.Fatal("error - expected 1 new finding, got ", len(findings["new-findings"])) + if len(findings["new-passing-findings"]) != 1 { + t.Fatal("error - expected 1 new finding, got ", len(findings["new-passing-findings"])) } } diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index 20fa202a..85157683 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -102,6 +102,7 @@ func MergeAssessmentResults(original *oscalTypes_1_1_2.AssessmentResults, latest // We should be able to return the latest results // This is used during evaluate to update the threshold prop automatically if original.UUID == latest.UUID { + // Consider that this is a potential modification and this might be a good location to generate a new UUID return latest, nil } From 0f58da2730351116cc55abb65261c371a16d34f4 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Tue, 4 Jun 2024 03:38:08 +0000 Subject: [PATCH 04/12] fix(evaluate): separate write logic from core evaluation logic --- src/cmd/evaluate/evaluate.go | 54 +++--- src/cmd/evaluate/evaluate_test.go | 35 ++++ .../oscal/invalid-assessment-result.yaml | 11 ++ .../common/oscal/valid-assessment-result.yaml | 181 ++++++++++++++++++ 4 files changed, 255 insertions(+), 26 deletions(-) create mode 100644 src/test/unit/common/oscal/invalid-assessment-result.yaml create mode 100644 src/test/unit/common/oscal/valid-assessment-result.yaml diff --git a/src/cmd/evaluate/evaluate.go b/src/cmd/evaluate/evaluate.go index 3169906a..982ff757 100644 --- a/src/cmd/evaluate/evaluate.go +++ b/src/cmd/evaluate/evaluate.go @@ -35,10 +35,20 @@ var evaluateCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { // Access the files and evaluate them - err := EvaluateAssessmentResults(opts.files) + assessmentMap, err := EvaluateAssessmentResults(opts.files) if err != nil { message.Fatal(err, err.Error()) } + + // Props are updated - now write back to all files + // if we create the model and write it - the merge will need to de-duplicate instead of merge results + for filePath, assessment := range assessmentMap { + model := oscalTypes_1_1_2.OscalCompleteSchema{ + AssessmentResults: assessment, + } + + oscal.WriteOscalModel(filePath, &model) + } }, } @@ -49,13 +59,13 @@ func EvaluateCommand() *cobra.Command { return evaluateCmd } -func EvaluateAssessmentResults(fileArray []string) error { +func EvaluateAssessmentResults(fileArray []string) (map[string]*oscalTypes_1_1_2.AssessmentResults, error) { var status bool var findings map[string][]oscalTypes_1_1_2.Finding var threshold, latest *oscalTypes_1_1_2.Result if len(fileArray) == 0 { - return fmt.Errorf("no files provided for evaluation") + return nil, fmt.Errorf("no files provided for evaluation") } // Potentially write changes back to multiple files requires some storage @@ -63,16 +73,16 @@ func EvaluateAssessmentResults(fileArray []string) error { for _, fileString := range fileArray { err := files.IsJsonOrYaml(fileString) if err != nil { - return fmt.Errorf("invalid file extension: %s, requires .json or .yaml", fileString) + return nil, fmt.Errorf("invalid file extension: %s, requires .json or .yaml", fileString) } data, err := common.ReadFileToBytes(fileString) if err != nil { - return err + return nil, err } assessment, err := oscal.NewAssessmentResults(data) if err != nil { - return err + return nil, err } resultMap[fileString] = assessment } @@ -82,13 +92,13 @@ func EvaluateAssessmentResults(fileArray []string) error { thresholds, sortedResults, err := findAndSortResults(resultMap) if err != nil { - return err + return nil, err } if len(sortedResults) <= 1 { // Should this implicitly pass? If so then a workflow can operate on the assumption that it will pass from 0 -> N results message.Infof("%v result object identified - unable to evaluate", len(sortedResults)) - return nil + return nil, nil } if len(thresholds) == 0 { @@ -98,7 +108,7 @@ func EvaluateAssessmentResults(fileArray []string) error { status, findings, err = EvaluateResults(threshold, latest) if err != nil { - return err + return nil, err } } else { // Constraint - Always evaluate the latest threshold against the latest result @@ -107,11 +117,11 @@ func EvaluateAssessmentResults(fileArray []string) error { if threshold.UUID == latest.UUID { // They are the same - return error - return fmt.Errorf("unable to evaluate - threshold and latest result are the same result - nothing to compare") + return nil, fmt.Errorf("unable to evaluate - threshold and latest result are the same result - nothing to compare") } status, findings, err = EvaluateResults(threshold, latest) if err != nil { - return err + return nil, err } } @@ -122,21 +132,13 @@ func EvaluateAssessmentResults(fileArray []string) error { message.Infof("%s", finding.Target.TargetId) } - message.Info("New threshold identified - threshold will be updated to latest result") - - updateProp("threshold", "false", threshold.Props) - updateProp("threshold", "true", latest.Props) - - // Props are updated - now write back to all files - // if we create the model and write it - the merge will need to de-duplicate instead of merge results - for filePath, assessment := range resultMap { - model := oscalTypes_1_1_2.OscalCompleteSchema{ - AssessmentResults: assessment, - } + message.Infof("New threshold identified - threshold will be updated to result %s", latest.UUID) - oscal.WriteOscalModel(filePath, &model) + // In the event we still have multiple thresholds - let's clean them up + for _, result := range thresholds { + updateProp("threshold", "false", result.Props) } - + updateProp("threshold", "true", latest.Props) } if len(findings["new-failing-findings"]) > 0 { @@ -146,13 +148,13 @@ func EvaluateAssessmentResults(fileArray []string) error { } } - return nil + return resultMap, nil } else { message.Warn("Evaluation Failed against the following findings:") for _, finding := range findings["no-longer-satisfied"] { message.Warnf("%s", finding.Target.TargetId) } - return fmt.Errorf("failed to meet established threshold") + return nil, fmt.Errorf("failed to meet established threshold") } } diff --git a/src/cmd/evaluate/evaluate_test.go b/src/cmd/evaluate/evaluate_test.go index 3186246b..095d49d0 100644 --- a/src/cmd/evaluate/evaluate_test.go +++ b/src/cmd/evaluate/evaluate_test.go @@ -7,6 +7,41 @@ import ( "github.com/defenseunicorns/lula/src/pkg/message" ) +var ( + validInputFile = "../../test/unit/common/oscal/valid-assessment-result.yaml" + invalidInputFile = "../../test/unit/common/oscal/invalid-assessment-result.yaml" +) + +func TestEvaluateAssessmentResults(t *testing.T) { + t.Parallel() + + // TODO: write logic to separate file read from core evaluation logic + // TODO: move the core logic to library package + // TODO: write a success test to receive assessments/results that we can verify intended prop change + // t.Run("handles valid assessment result", func(t *testing.T) { + // assessmentMap, err := EvaluateAssessmentResults([]string{validInputFile}) + // if err != nil { + // t.Fatal("unexpected error for valid assessment result") + // } + + // }) + + t.Run("handles invalid path to assessment result file", func(t *testing.T) { + _, err := EvaluateAssessmentResults([]string{"./invalid-path.yaml"}) + if err == nil { + t.Fatal("expected error for invalid path") + } + }) + + t.Run("handles invalid assessment result without any results", func(t *testing.T) { + _, err := EvaluateAssessmentResults([]string{invalidInputFile}) + if err == nil { + t.Fatal("expected error for invalid assessment result without results") + } + }) + +} + // Given two results - evaluate for passing func TestEvaluateResultsPassing(t *testing.T) { message.NoProgress = true diff --git a/src/test/unit/common/oscal/invalid-assessment-result.yaml b/src/test/unit/common/oscal/invalid-assessment-result.yaml new file mode 100644 index 00000000..4d429b4b --- /dev/null +++ b/src/test/unit/common/oscal/invalid-assessment-result.yaml @@ -0,0 +1,11 @@ +assessment-results: + import-ap: + href: "" + metadata: + last-modified: 2024-06-04T02:34:02.653177582Z + oscal-version: 1.1.2 + published: 2024-06-04T02:34:02.653177582Z + remarks: Assessment Results generated from Lula + title: '[System Name] Security Assessment Results (SAR)' + version: 0.0.1 + uuid: 19541f5a-7738-4d86-b53f-34f3dc7a1586 diff --git a/src/test/unit/common/oscal/valid-assessment-result.yaml b/src/test/unit/common/oscal/valid-assessment-result.yaml new file mode 100644 index 00000000..c886453c --- /dev/null +++ b/src/test/unit/common/oscal/valid-assessment-result.yaml @@ -0,0 +1,181 @@ +assessment-results: + import-ap: + href: "" + metadata: + last-modified: 2024-06-04T03:34:46.521030308Z + oscal-version: 1.1.2 + published: 2024-06-04T02:46:29.02587111Z + remarks: Assessment Results generated from Lula + title: '[System Name] Security Assessment Results (SAR)' + version: 0.0.1 + results: + - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43 + findings: + - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + related-observations: + - observation-uuid: bcb5c77a-981d-4698-acd4-84c3e8a0b3e7 + target: + status: + state: satisfied + target-id: ID-1 + type: objective-id + title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' + uuid: 41018849-9825-4760-82ad-6d74362e44d5 + - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + related-observations: + - observation-uuid: 38d1abe2-7c30-4eaa-b792-46a0978bf0a0 + target: + status: + state: satisfied + target-id: ID-2 + type: objective-id + title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 / Control: ID-2' + uuid: 9fd5633e-2509-41bc-b66c-16d30e7dc29e + observations: + - collected: 2024-06-04T03:34:46.435905414Z + description: |+ + [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + + methods: + - TEST + relevant-evidence: + - description: | + Result: satisfied + uuid: bcb5c77a-981d-4698-acd4-84c3e8a0b3e7 + - collected: 2024-06-04T03:34:46.441327814Z + description: |+ + [TEST]: ID-2 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + + methods: + - TEST + relevant-evidence: + - description: | + Result: satisfied + uuid: 38d1abe2-7c30-4eaa-b792-46a0978bf0a0 + props: + - name: threshold + ns: https://docs.lula.dev/ns + value: "true" + reviewed-controls: + control-selections: + - description: Controls Assessed by Lula + include-controls: + - control-id: ID-1 + - control-id: ID-2 + description: Controls validated + remarks: Validation performed may indicate full or partial satisfaction + start: 2024-06-04T03:34:46.443161469Z + title: Lula Validation Result + uuid: b5fb7bd4-c91e-4cd3-8020-ed04aa8d74b5 + - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43 + findings: + - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + related-observations: + - observation-uuid: 42a8b71d-7d08-42cc-beb4-c1c384c119ef + target: + status: + state: satisfied + target-id: ID-1 + type: objective-id + title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' + uuid: 50c818bd-e6be-4870-8766-8a22d18f1af1 + observations: + - collected: 2024-06-04T03:33:33.977370866Z + description: |+ + [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + + methods: + - TEST + relevant-evidence: + - description: | + Result: satisfied + uuid: 42a8b71d-7d08-42cc-beb4-c1c384c119ef + props: + - name: threshold + ns: https://docs.lula.dev/ns + value: "false" + reviewed-controls: + control-selections: + - description: Controls Assessed by Lula + include-controls: + - control-id: ID-1 + description: Controls validated + remarks: Validation performed may indicate full or partial satisfaction + start: 2024-06-04T03:33:33.982591133Z + title: Lula Validation Result + uuid: ae611955-e97e-4115-a5d0-2e5ccbf35872 + - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43 + findings: + - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + related-observations: + - observation-uuid: 20d943cb-8a8d-47e2-b625-2d5829604f5e + target: + status: + state: satisfied + target-id: ID-1 + type: objective-id + title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' + uuid: fbee2fb0-e8ba-419c-9d29-f20525bbc420 + observations: + - collected: 2024-06-04T02:48:06.890379789Z + description: |+ + [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + + methods: + - TEST + relevant-evidence: + - description: | + Result: satisfied + uuid: 20d943cb-8a8d-47e2-b625-2d5829604f5e + props: + - name: threshold + ns: https://docs.lula.dev/ns + value: "false" + reviewed-controls: + control-selections: + - description: Controls Assessed by Lula + include-controls: + - control-id: ID-1 + description: Controls validated + remarks: Validation performed may indicate full or partial satisfaction + start: 2024-06-04T02:48:06.895205436Z + title: Lula Validation Result + uuid: 7ef25ccd-22b8-421e-8c64-9fcbde697696 + - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43 + findings: + - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. + related-observations: + - observation-uuid: d3b228e8-8c8b-4be9-a0b0-8da9fcdb005f + target: + status: + state: not-satisfied + target-id: ID-1 + type: objective-id + title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' + uuid: 44fc8e49-b633-4880-8d58-4ed0f14c7bdb + observations: + - collected: 2024-06-04T02:46:29.018416946Z + description: |+ + [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 + + methods: + - TEST + relevant-evidence: + - description: | + Result: not-satisfied + uuid: d3b228e8-8c8b-4be9-a0b0-8da9fcdb005f + props: + - name: threshold + ns: https://docs.lula.dev/ns + value: "false" + reviewed-controls: + control-selections: + - description: Controls Assessed by Lula + include-controls: + - control-id: ID-1 + description: Controls validated + remarks: Validation performed may indicate full or partial satisfaction + start: 2024-06-04T02:46:29.02587111Z + title: Lula Validation Result + uuid: 250ac234-519f-4f26-bee3-98e870feeb7a + uuid: 9673739a-9379-4575-b454-1b42672a4970 From 10ef431efd4ad7572bfb164d769e8cd84a2c969d Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Tue, 4 Jun 2024 17:37:10 +0000 Subject: [PATCH 05/12] fix(evaluate): refactor code to library and move/fix tests --- src/cmd/evaluate/evaluate.go | 242 +++++------------- src/cmd/validate/validate.go | 16 +- src/pkg/common/oscal/assessment-results.go | 118 ++++++++- .../common/oscal/assessment-results_test.go} | 111 +++----- src/pkg/common/oscal/common.go | 25 ++ 5 files changed, 253 insertions(+), 259 deletions(-) rename src/{cmd/evaluate/evaluate_test.go => pkg/common/oscal/assessment-results_test.go} (59%) create mode 100644 src/pkg/common/oscal/common.go diff --git a/src/cmd/evaluate/evaluate.go b/src/cmd/evaluate/evaluate.go index 982ff757..2ddfd7ec 100644 --- a/src/cmd/evaluate/evaluate.go +++ b/src/cmd/evaluate/evaluate.go @@ -2,7 +2,6 @@ package evaluate import ( "fmt" - "slices" "github.com/defenseunicorns/go-oscal/src/pkg/files" oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2" @@ -34,14 +33,73 @@ var evaluateCmd = &cobra.Command{ Aliases: []string{"eval"}, Run: func(cmd *cobra.Command, args []string) { - // Access the files and evaluate them - assessmentMap, err := EvaluateAssessmentResults(opts.files) + // Build map of filepath -> assessment results + assessmentMap, err := readManyAssessmentResults(opts.files) if err != nil { message.Fatal(err, err.Error()) } - // Props are updated - now write back to all files - // if we create the model and write it - the merge will need to de-duplicate instead of merge results + // Identify the threshold & latest for comparison + resultMap, err := oscal.IdentifyResults(assessmentMap) + if err != nil { + if err.Error() == "less than 2 results found - no comparison possible" { + // Catch and warn of insufficient results + message.Warn(err.Error()) + return + } else { + message.Fatal(err, err.Error()) + } + } + + if resultMap["threshold"] != nil && resultMap["latest"] != nil { + // Compare the assessment results + spinner := message.NewProgressSpinner("Evaluating Assessment Results %s against %s", resultMap["threshold"].UUID, resultMap["latest"].UUID) + defer spinner.Stop() + + status, findings, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) + if err != nil { + message.Fatal(err, err.Error()) + } + + if status { + if len(findings["new-passing-findings"]) > 0 { + message.Info("New passing finding Target-Ids:") + for _, finding := range findings["new-passing-findings"] { + message.Infof("%s", finding.Target.TargetId) + } + + message.Infof("New threshold identified - threshold will be updated to result %s", resultMap["latest"].UUID) + + // In the event we still have multiple thresholds - let's clean them up + oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "false", resultMap["threshold"].Props) + oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "true", resultMap["latest"].Props) + } + + if len(findings["new-failing-findings"]) > 0 { + message.Info("New failing finding Target-Ids:") + for _, finding := range findings["new-failing-findings"] { + message.Infof("%s", finding.Target.TargetId) + } + } + + return + } else { + message.Warn("Evaluation Failed against the following findings:") + for _, finding := range findings["no-longer-satisfied"] { + message.Warnf("%s", finding.Target.TargetId) + } + message.Fatalf(fmt.Errorf("failed to meet established threshold"), "failed to meet established threshold") + } + + spinner.Success() + + } else if resultMap["threshold"] == nil { + message.Fatal(fmt.Errorf("no threshold assessment results could be identified"), "no threshold assessment results could be identified") + } else { + message.Fatal(fmt.Errorf("no latest assessment results could be identified"), "no latest assessment results could be identified") + } + + // Write each file back in the case of modification for filePath, assessment := range assessmentMap { model := oscalTypes_1_1_2.OscalCompleteSchema{ AssessmentResults: assessment, @@ -59,17 +117,14 @@ func EvaluateCommand() *cobra.Command { return evaluateCmd } -func EvaluateAssessmentResults(fileArray []string) (map[string]*oscalTypes_1_1_2.AssessmentResults, error) { - var status bool - var findings map[string][]oscalTypes_1_1_2.Finding - var threshold, latest *oscalTypes_1_1_2.Result - +// Read many filepaths into a map[filepath]*AssessmentResults +// Placing here until otherwise decided on value elsewhere +func readManyAssessmentResults(fileArray []string) (map[string]*oscalTypes_1_1_2.AssessmentResults, error) { if len(fileArray) == 0 { return nil, fmt.Errorf("no files provided for evaluation") } - // Potentially write changes back to multiple files requires some storage - resultMap := make(map[string]*oscalTypes_1_1_2.AssessmentResults) + assessmentMap := make(map[string]*oscalTypes_1_1_2.AssessmentResults) for _, fileString := range fileArray { err := files.IsJsonOrYaml(fileString) if err != nil { @@ -84,167 +139,8 @@ func EvaluateAssessmentResults(fileArray []string) (map[string]*oscalTypes_1_1_2 if err != nil { return nil, err } - resultMap[fileString] = assessment - } - - // Now that we have the map of assessment results - we need to identify the threshold(s) - // Also sort the results -> if we maintain pointers, can we update and write all artifacts in one go? - - thresholds, sortedResults, err := findAndSortResults(resultMap) - if err != nil { - return nil, err - } - - if len(sortedResults) <= 1 { - // Should this implicitly pass? If so then a workflow can operate on the assumption that it will pass from 0 -> N results - message.Infof("%v result object identified - unable to evaluate", len(sortedResults)) - return nil, nil - } - - if len(thresholds) == 0 { - // No thresholds identified but we have > 1 results - compare the latest and the preceding - threshold = sortedResults[len(sortedResults)-2] - latest = sortedResults[len(sortedResults)-1] - - status, findings, err = EvaluateResults(threshold, latest) - if err != nil { - return nil, err - } - } else { - // Constraint - Always evaluate the latest threshold against the latest result - threshold = thresholds[len(thresholds)-1] - latest = sortedResults[len(sortedResults)-1] - - if threshold.UUID == latest.UUID { - // They are the same - return error - return nil, fmt.Errorf("unable to evaluate - threshold and latest result are the same result - nothing to compare") - } - status, findings, err = EvaluateResults(threshold, latest) - if err != nil { - return nil, err - } - } - - if status { - if len(findings["new-passing-findings"]) > 0 { - message.Info("New passing finding Target-Ids:") - for _, finding := range findings["new-passing-findings"] { - message.Infof("%s", finding.Target.TargetId) - } - - message.Infof("New threshold identified - threshold will be updated to result %s", latest.UUID) - - // In the event we still have multiple thresholds - let's clean them up - for _, result := range thresholds { - updateProp("threshold", "false", result.Props) - } - updateProp("threshold", "true", latest.Props) - } - - if len(findings["new-failing-findings"]) > 0 { - message.Info("New failing finding Target-Ids:") - for _, finding := range findings["new-failing-findings"] { - message.Infof("%s", finding.Target.TargetId) - } - } - - return resultMap, nil - } else { - message.Warn("Evaluation Failed against the following findings:") - for _, finding := range findings["no-longer-satisfied"] { - message.Warnf("%s", finding.Target.TargetId) - } - return nil, fmt.Errorf("failed to meet established threshold") - } -} - -func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalTypes_1_1_2.Result) (bool, map[string][]oscalTypes_1_1_2.Finding, error) { - if thresholdResult.Findings == nil || newResult.Findings == nil { - return false, nil, fmt.Errorf("results must contain findings to evaluate") - } - - spinner := message.NewProgressSpinner("Evaluating Assessment Results %s against %s", newResult.UUID, thresholdResult.UUID) - defer spinner.Stop() - - // Store unique findings for review here - findings := make(map[string][]oscalTypes_1_1_2.Finding, 0) - result := true - - findingMapThreshold := oscal.GenerateFindingsMap(*thresholdResult.Findings) - findingMapNew := oscal.GenerateFindingsMap(*newResult.Findings) - - // For a given oldResult - we need to prove that the newResult implements all of the oldResult findings/controls - // We are explicitly iterating through the findings in order to collect a delta to display - - for targetId, finding := range findingMapThreshold { - if _, ok := findingMapNew[targetId]; !ok { - // If the new result does not contain the finding of the old result - // set result to fail, add finding to the findings map and continue - result = false - findings[targetId] = append(findings["no-longer-satisfied"], finding) - } else { - // If the finding is present in each map - we need to check if the state has changed from "not-satisfied" to "satisfied" - if finding.Target.Status.State == "satisfied" { - // Was previously satisfied - compare state - if findingMapNew[targetId].Target.Status.State == "not-satisfied" { - // If the new finding is now not-satisfied - set result to false and add to findings - result = false - findings["no-longer-satisfied"] = append(findings["no-longer-satisfied"], finding) - } - } - delete(findingMapNew, targetId) - } + assessmentMap[fileString] = assessment } - // All remaining findings in the new map are new findings - for _, finding := range findingMapNew { - if finding.Target.Status.State == "satisfied" { - findings["new-passing-findings"] = append(findings["new-passing-findings"], finding) - } else { - findings["new-failing-findings"] = append(findings["new-failing-findings"], finding) - } - - } - - spinner.Success() - return result, findings, nil -} - -// findAndSortResults takes a map of results and returns a list of thresholds and a sorted list of results in order of time -func findAndSortResults(resultMap map[string]*oscalTypes_1_1_2.AssessmentResults) ([]*oscalTypes_1_1_2.Result, []*oscalTypes_1_1_2.Result, error) { - - thresholds := make([]*oscalTypes_1_1_2.Result, 0) - sortedResults := make([]*oscalTypes_1_1_2.Result, 0) - - for _, assessment := range resultMap { - for _, result := range assessment.Results { - if result.Props != nil { - for _, prop := range *result.Props { - if prop.Name == "threshold" && prop.Value == "true" { - thresholds = append(thresholds, &result) - } - } - } - // Store all results in a non-sorted list - sortedResults = append(sortedResults, &result) - } - } - - // Sort the results by start time - slices.SortFunc(sortedResults, func(a, b *oscalTypes_1_1_2.Result) int { return a.Start.Compare(b.Start) }) - slices.SortFunc(thresholds, func(a, b *oscalTypes_1_1_2.Result) int { return a.Start.Compare(b.Start) }) - - return thresholds, sortedResults, nil -} - -func updateProp(name string, value string, props *[]oscalTypes_1_1_2.Property) error { - - for index, prop := range *props { - if prop.Name == name { - prop.Value = value - (*props)[index] = prop - return nil - } - } - return fmt.Errorf("property not found") + return assessmentMap, nil } diff --git a/src/cmd/validate/validate.go b/src/cmd/validate/validate.go index f1fd1ea7..26d3a45d 100644 --- a/src/cmd/validate/validate.go +++ b/src/cmd/validate/validate.go @@ -199,7 +199,7 @@ func ValidateOnCompDef(compDef *oscalTypes_1_1_2.ComponentDefinition) (map[strin if err != nil { message.Debugf("Error getting lula validation %s: %v", id, err) // Handle error as an output to observations - observation := createObservation("TEST", "[Failed Observation]: %s - %s\n%s\n", implementedRequirement.ControlId, id, link.Text) + observation := oscal.CreateObservation("TEST", "[Failed Observation]: %s - %s\n%s\n", implementedRequirement.ControlId, id, link.Text) observation.RelevantEvidence = &[]oscalTypes_1_1_2.RelevantEvidence{ { Description: "Result: not-satistfied\n", @@ -210,7 +210,7 @@ func ValidateOnCompDef(compDef *oscalTypes_1_1_2.ComponentDefinition) (map[strin relatedObservations, tempObservations = appendObservations(relatedObservations, tempObservations, observation) } else { // Add the description of the validation now that we have the ID - observation := createObservation("TEST", "[TEST]: %s - %s\n%s\n", implementedRequirement.ControlId, id, link.Text) + observation := oscal.CreateObservation("TEST", "[TEST]: %s - %s\n%s\n", implementedRequirement.ControlId, id, link.Text) err = lulaValidation.Validate() if err != nil { @@ -352,18 +352,6 @@ func WriteReport(report oscalTypes_1_1_2.AssessmentResults, assessmentFilePath s return nil } -// Helper function to create observation -func createObservation(method string, descriptionPattern string, descriptionArgs ...any) oscalTypes_1_1_2.Observation { - rfc3339Time := time.Now() - sharedUuid := uuid.NewUUID() - return oscalTypes_1_1_2.Observation{ - Collected: rfc3339Time, - Methods: []string{method}, - UUID: sharedUuid, - Description: fmt.Sprintf(descriptionPattern, descriptionArgs...), - } -} - // Helper function to append observations func appendObservations(relatedObservations []oscalTypes_1_1_2.RelatedObservation, tempObservations []oscalTypes_1_1_2.Observation, observation oscalTypes_1_1_2.Observation) ([]oscalTypes_1_1_2.RelatedObservation, []oscalTypes_1_1_2.Observation) { relatedObservation := oscalTypes_1_1_2.RelatedObservation{ diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index 85157683..10ceee20 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -2,6 +2,7 @@ package oscal import ( "fmt" + "slices" "time" "github.com/defenseunicorns/go-oscal/src/pkg/uuid" @@ -61,7 +62,7 @@ func GenerateAssessmentResults(findingMap map[string]oscalTypes_1_1_2.Finding, o LastModified: rfc3339Time, } - // Here we are going to add the threshold property + // Here we are going to add the threshold property by default props := []oscalTypes_1_1_2.Property{ { Ns: "https://docs.lula.dev/ns", @@ -102,7 +103,6 @@ func MergeAssessmentResults(original *oscalTypes_1_1_2.AssessmentResults, latest // We should be able to return the latest results // This is used during evaluate to update the threshold prop automatically if original.UUID == latest.UUID { - // Consider that this is a potential modification and this might be a good location to generate a new UUID return latest, nil } @@ -137,3 +137,117 @@ func GenerateFindingsMap(findings []oscalTypes_1_1_2.Finding) map[string]oscalTy } return findingsMap } + +// IdentifyResults produces a map containing the threshold result and a result used for comparison +func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResults) (map[string]*oscalTypes_1_1_2.Result, error) { + resultMap := make(map[string]*oscalTypes_1_1_2.Result) + + thresholds, sortedResults, err := findAndSortResults(assessmentMap) + if err != nil { + return nil, err + } + + if len(sortedResults) <= 1 { + return nil, fmt.Errorf("less than 2 results found - no comparison possible") + } + + if len(thresholds) == 0 { + // No thresholds identified but we have > 1 results - compare the latest and the preceding + resultMap["threshold"] = sortedResults[len(sortedResults)-2] + resultMap["latest"] = sortedResults[len(sortedResults)-1] + + return resultMap, nil + } else { + // Constraint - Always evaluate the latest threshold against the latest result + resultMap["threshold"] = thresholds[len(thresholds)-1] + resultMap["latest"] = sortedResults[len(sortedResults)-1] + return resultMap, nil + } +} + +func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalTypes_1_1_2.Result) (bool, map[string][]oscalTypes_1_1_2.Finding, error) { + if thresholdResult.Findings == nil || newResult.Findings == nil { + return false, nil, fmt.Errorf("results must contain findings to evaluate") + } + + // Store unique findings for review here + findings := make(map[string][]oscalTypes_1_1_2.Finding, 0) + result := true + + findingMapThreshold := GenerateFindingsMap(*thresholdResult.Findings) + findingMapNew := GenerateFindingsMap(*newResult.Findings) + + // For a given oldResult - we need to prove that the newResult implements all of the oldResult findings/controls + // We are explicitly iterating through the findings in order to collect a delta to display + + for targetId, finding := range findingMapThreshold { + if _, ok := findingMapNew[targetId]; !ok { + // If the new result does not contain the finding of the old result + // set result to fail, add finding to the findings map and continue + result = false + findings[targetId] = append(findings["no-longer-satisfied"], finding) + } else { + // If the finding is present in each map - we need to check if the state has changed from "not-satisfied" to "satisfied" + if finding.Target.Status.State == "satisfied" { + // Was previously satisfied - compare state + if findingMapNew[targetId].Target.Status.State == "not-satisfied" { + // If the new finding is now not-satisfied - set result to false and add to findings + result = false + findings["no-longer-satisfied"] = append(findings["no-longer-satisfied"], finding) + } + } + delete(findingMapNew, targetId) + } + } + + // All remaining findings in the new map are new findings + for _, finding := range findingMapNew { + if finding.Target.Status.State == "satisfied" { + findings["new-passing-findings"] = append(findings["new-passing-findings"], finding) + } else { + findings["new-failing-findings"] = append(findings["new-failing-findings"], finding) + } + + } + + return result, findings, nil +} + +// findAndSortResults takes a map of results and returns a list of thresholds and a sorted list of results in order of time +func findAndSortResults(resultMap map[string]*oscalTypes_1_1_2.AssessmentResults) ([]*oscalTypes_1_1_2.Result, []*oscalTypes_1_1_2.Result, error) { + + thresholds := make([]*oscalTypes_1_1_2.Result, 0) + sortedResults := make([]*oscalTypes_1_1_2.Result, 0) + + for _, assessment := range resultMap { + for _, result := range assessment.Results { + if result.Props != nil { + for _, prop := range *result.Props { + if prop.Name == "threshold" && prop.Value == "true" { + thresholds = append(thresholds, &result) + } + } + } + // Store all results in a non-sorted list + sortedResults = append(sortedResults, &result) + } + } + + // Sort the results by start time + slices.SortFunc(sortedResults, func(a, b *oscalTypes_1_1_2.Result) int { return a.Start.Compare(b.Start) }) + slices.SortFunc(thresholds, func(a, b *oscalTypes_1_1_2.Result) int { return a.Start.Compare(b.Start) }) + + return thresholds, sortedResults, nil +} + +// Helper function to create observation +func CreateObservation(method string, descriptionPattern string, descriptionArgs ...any) oscalTypes_1_1_2.Observation { + rfc3339Time := time.Now() + sharedUuid := uuid.NewUUID() + return oscalTypes_1_1_2.Observation{ + Collected: rfc3339Time, + Methods: []string{method}, + UUID: sharedUuid, + Description: fmt.Sprintf(descriptionPattern, descriptionArgs...), + } +} diff --git a/src/cmd/evaluate/evaluate_test.go b/src/pkg/common/oscal/assessment-results_test.go similarity index 59% rename from src/cmd/evaluate/evaluate_test.go rename to src/pkg/common/oscal/assessment-results_test.go index 095d49d0..94c3aa14 100644 --- a/src/cmd/evaluate/evaluate_test.go +++ b/src/pkg/common/oscal/assessment-results_test.go @@ -1,46 +1,45 @@ -package evaluate +package oscal_test import ( "testing" oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2" + "github.com/defenseunicorns/lula/src/pkg/common/oscal" "github.com/defenseunicorns/lula/src/pkg/message" ) -var ( - validInputFile = "../../test/unit/common/oscal/valid-assessment-result.yaml" - invalidInputFile = "../../test/unit/common/oscal/invalid-assessment-result.yaml" -) +// Create re-usable findings and observations +// use those in tests to generate test assessment results +var findingMap = map[string]oscalTypes_1_1_2.Finding{ + "ID-1_pass": { + Target: oscalTypes_1_1_2.FindingTarget{ + TargetId: "ID-1", + Status: oscalTypes_1_1_2.ObjectiveStatus{ + State: "satisfied", + }, + }, + }, + "ID-1_fail": { + Target: oscalTypes_1_1_2.FindingTarget{ + TargetId: "ID-1", + Status: oscalTypes_1_1_2.ObjectiveStatus{ + State: "not-satisfied", + }, + }, + }, +} -func TestEvaluateAssessmentResults(t *testing.T) { - t.Parallel() - - // TODO: write logic to separate file read from core evaluation logic - // TODO: move the core logic to library package - // TODO: write a success test to receive assessments/results that we can verify intended prop change - // t.Run("handles valid assessment result", func(t *testing.T) { - // assessmentMap, err := EvaluateAssessmentResults([]string{validInputFile}) - // if err != nil { - // t.Fatal("unexpected error for valid assessment result") - // } - - // }) - - t.Run("handles invalid path to assessment result file", func(t *testing.T) { - _, err := EvaluateAssessmentResults([]string{"./invalid-path.yaml"}) - if err == nil { - t.Fatal("expected error for invalid path") - } - }) - - t.Run("handles invalid assessment result without any results", func(t *testing.T) { - _, err := EvaluateAssessmentResults([]string{invalidInputFile}) - if err == nil { - t.Fatal("expected error for invalid assessment result without results") - } - }) +// func TestIdentifyResults(t *testing.T) { +// t.Parallel() -} +// // t.Run("handles invalid path to assessment result file", func(t *testing.T) { +// // _, err := EvaluateAssessmentResults([]string{"./invalid-path.yaml"}) +// // if err == nil { +// // t.Fatal("expected error for invalid path") +// // } +// // }) + +// } // Given two results - evaluate for passing func TestEvaluateResultsPassing(t *testing.T) { @@ -48,31 +47,17 @@ func TestEvaluateResultsPassing(t *testing.T) { mockThresholdResult := oscalTypes_1_1_2.Result{ Findings: &[]oscalTypes_1_1_2.Finding{ - { - Target: oscalTypes_1_1_2.FindingTarget{ - TargetId: "ID-1", - Status: oscalTypes_1_1_2.ObjectiveStatus{ - State: "satisfied", - }, - }, - }, + findingMap["ID-1_pass"], }, } mockEvaluationResult := oscalTypes_1_1_2.Result{ Findings: &[]oscalTypes_1_1_2.Finding{ - { - Target: oscalTypes_1_1_2.FindingTarget{ - TargetId: "ID-1", - Status: oscalTypes_1_1_2.ObjectiveStatus{ - State: "satisfied", - }, - }, - }, + findingMap["ID-1_pass"], }, } - status, _, err := EvaluateResults(&mockThresholdResult, &mockEvaluationResult) + status, _, err := oscal.EvaluateResults(&mockThresholdResult, &mockEvaluationResult) if err != nil { t.Fatal(err) } @@ -88,31 +73,17 @@ func TestEvaluateResultsFailed(t *testing.T) { message.NoProgress = true mockThresholdResult := oscalTypes_1_1_2.Result{ Findings: &[]oscalTypes_1_1_2.Finding{ - { - Target: oscalTypes_1_1_2.FindingTarget{ - TargetId: "ID-1", - Status: oscalTypes_1_1_2.ObjectiveStatus{ - State: "satisfied", - }, - }, - }, + findingMap["ID-1_pass"], }, } mockEvaluationResult := oscalTypes_1_1_2.Result{ Findings: &[]oscalTypes_1_1_2.Finding{ - { - Target: oscalTypes_1_1_2.FindingTarget{ - TargetId: "ID-1", - Status: oscalTypes_1_1_2.ObjectiveStatus{ - State: "not-satisfied", - }, - }, - }, + findingMap["ID-1_fail"], }, } - status, findings, err := EvaluateResults(&mockThresholdResult, &mockEvaluationResult) + status, findings, err := oscal.EvaluateResults(&mockThresholdResult, &mockEvaluationResult) if err != nil { t.Fatal(err) } @@ -138,7 +109,7 @@ func TestEvaluateResultsNoFindings(t *testing.T) { Findings: &[]oscalTypes_1_1_2.Finding{}, } - status, _, err := EvaluateResults(&mockThresholdResult, &mockEvaluationResult) + status, _, err := oscal.EvaluateResults(&mockThresholdResult, &mockEvaluationResult) if err != nil { t.Fatal(err) } @@ -167,7 +138,7 @@ func TestEvaluateResultsNoThreshold(t *testing.T) { }, } - _, _, err := EvaluateResults(&mockThresholdResult, &mockEvaluationResult) + _, _, err := oscal.EvaluateResults(&mockThresholdResult, &mockEvaluationResult) if err == nil { t.Fatal("error - expected error, got nil") } @@ -217,7 +188,7 @@ func TestEvaluateResultsNewFindings(t *testing.T) { }, } - status, findings, err := EvaluateResults(&mockThresholdResult, &mockEvaluationResult) + status, findings, err := oscal.EvaluateResults(&mockThresholdResult, &mockEvaluationResult) if err != nil { t.Fatal(err) } diff --git a/src/pkg/common/oscal/common.go b/src/pkg/common/oscal/common.go new file mode 100644 index 00000000..e4f7cb55 --- /dev/null +++ b/src/pkg/common/oscal/common.go @@ -0,0 +1,25 @@ +package oscal + +import ( + oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2" +) + +// UpdateProps updates a property in a slice of properties or adds if not exists +func UpdateProps(name string, namespace string, value string, props *[]oscalTypes_1_1_2.Property) { + + for index, prop := range *props { + if prop.Name == name && prop.Ns == namespace { + prop.Value = value + (*props)[index] = prop + return + } + } + // Prop does not exist + prop := oscalTypes_1_1_2.Property{ + Ns: namespace, + Name: name, + Value: value, + } + + *props = append(*props, prop) +} From dd186766ed1ce4eda4a4e9eecc09b9a95c283e21 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Tue, 4 Jun 2024 18:26:43 +0000 Subject: [PATCH 06/12] fix(evaluate): add tests and cleanup various functions --- src/cmd/evaluate/evaluate.go | 4 +- src/pkg/common/oscal/assessment-results.go | 8 +- .../common/oscal/assessment-results_test.go | 90 +++++++++++++++---- 3 files changed, 81 insertions(+), 21 deletions(-) diff --git a/src/cmd/evaluate/evaluate.go b/src/cmd/evaluate/evaluate.go index 2ddfd7ec..1f5d9915 100644 --- a/src/cmd/evaluate/evaluate.go +++ b/src/cmd/evaluate/evaluate.go @@ -70,8 +70,7 @@ var evaluateCmd = &cobra.Command{ message.Infof("New threshold identified - threshold will be updated to result %s", resultMap["latest"].UUID) - // In the event we still have multiple thresholds - let's clean them up - oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "false", resultMap["threshold"].Props) + // Update latest threshold prop oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "true", resultMap["latest"].Props) } @@ -82,7 +81,6 @@ var evaluateCmd = &cobra.Command{ } } - return } else { message.Warn("Evaluation Failed against the following findings:") for _, finding := range findings["no-longer-satisfied"] { diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index 10ceee20..e70a6528 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -160,6 +160,10 @@ func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResult } else { // Constraint - Always evaluate the latest threshold against the latest result resultMap["threshold"] = thresholds[len(thresholds)-1] + // Consider changing the namespace value to "false" here - only written if the command logic completes + for _, result := range thresholds { + UpdateProps("threshold", "https://docs.lula.dev/ns", "false", result.Props) + } resultMap["latest"] = sortedResults[len(sortedResults)-1] return resultMap, nil } @@ -243,11 +247,11 @@ func findAndSortResults(resultMap map[string]*oscalTypes_1_1_2.AssessmentResults // Helper function to create observation func CreateObservation(method string, descriptionPattern string, descriptionArgs ...any) oscalTypes_1_1_2.Observation { rfc3339Time := time.Now() - sharedUuid := uuid.NewUUID() + uuid := uuid.NewUUID() return oscalTypes_1_1_2.Observation{ Collected: rfc3339Time, Methods: []string{method}, - UUID: sharedUuid, + UUID: uuid, Description: fmt.Sprintf(descriptionPattern, descriptionArgs...), } } diff --git a/src/pkg/common/oscal/assessment-results_test.go b/src/pkg/common/oscal/assessment-results_test.go index 94c3aa14..09eb5eb1 100644 --- a/src/pkg/common/oscal/assessment-results_test.go +++ b/src/pkg/common/oscal/assessment-results_test.go @@ -2,7 +2,9 @@ package oscal_test import ( "testing" + "time" + "github.com/defenseunicorns/go-oscal/src/pkg/uuid" oscalTypes_1_1_2 "github.com/defenseunicorns/go-oscal/src/types/oscal-1-1-2" "github.com/defenseunicorns/lula/src/pkg/common/oscal" "github.com/defenseunicorns/lula/src/pkg/message" @@ -10,8 +12,8 @@ import ( // Create re-usable findings and observations // use those in tests to generate test assessment results -var findingMap = map[string]oscalTypes_1_1_2.Finding{ - "ID-1_pass": { +var findingMapPass = map[string]oscalTypes_1_1_2.Finding{ + "ID-1": { Target: oscalTypes_1_1_2.FindingTarget{ TargetId: "ID-1", Status: oscalTypes_1_1_2.ObjectiveStatus{ @@ -19,7 +21,10 @@ var findingMap = map[string]oscalTypes_1_1_2.Finding{ }, }, }, - "ID-1_fail": { +} + +var findingMapFail = map[string]oscalTypes_1_1_2.Finding{ + "ID-1": { Target: oscalTypes_1_1_2.FindingTarget{ TargetId: "ID-1", Status: oscalTypes_1_1_2.ObjectiveStatus{ @@ -29,17 +34,70 @@ var findingMap = map[string]oscalTypes_1_1_2.Finding{ }, } -// func TestIdentifyResults(t *testing.T) { -// t.Parallel() +var observations = []oscalTypes_1_1_2.Observation{ + { + Collected: time.Now(), + Methods: []string{"TEST"}, + UUID: uuid.NewUUID(), + Description: "test description", + }, + { + Collected: time.Now(), + Methods: []string{"TEST"}, + UUID: uuid.NewUUID(), + Description: "test description", + }, +} + +func TestIdentifyResults(t *testing.T) { + t.Parallel() + + t.Run("Handle valid assessment containing a single result", func(t *testing.T) { + + assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } -// // t.Run("handles invalid path to assessment result file", func(t *testing.T) { -// // _, err := EvaluateAssessmentResults([]string{"./invalid-path.yaml"}) -// // if err == nil { -// // t.Fatal("expected error for invalid path") -// // } -// // }) + // key name does not matter here + var assessmentMap = map[string]*oscalTypes_1_1_2.AssessmentResults{ + "valid.yaml": assessment, + } -// } + _, err = oscal.IdentifyResults(assessmentMap) + if err == nil { + t.Fatalf("Expected error for inability to identify multiple results : %v", err) + } + }) + + t.Run("Handle multiple valid assessment containing a single result", func(t *testing.T) { + + assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } + + assessment2, err := oscal.GenerateAssessmentResults(findingMapFail, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } + + // key name does not matter here + var assessmentMap = map[string]*oscalTypes_1_1_2.AssessmentResults{ + "valid.yaml": assessment, + "other.yaml": assessment2, + } + + resultMap, err := oscal.IdentifyResults(assessmentMap) + if err != nil { + t.Fatalf("Expected error for inability to identify multiple results : %v", err) + } + + if resultMap["threshold"] == nil || resultMap["latest"] == nil { + t.Fatalf("Expected results to be identified") + } + }) +} // Given two results - evaluate for passing func TestEvaluateResultsPassing(t *testing.T) { @@ -47,13 +105,13 @@ func TestEvaluateResultsPassing(t *testing.T) { mockThresholdResult := oscalTypes_1_1_2.Result{ Findings: &[]oscalTypes_1_1_2.Finding{ - findingMap["ID-1_pass"], + findingMapPass["ID-1"], }, } mockEvaluationResult := oscalTypes_1_1_2.Result{ Findings: &[]oscalTypes_1_1_2.Finding{ - findingMap["ID-1_pass"], + findingMapPass["ID-1"], }, } @@ -73,13 +131,13 @@ func TestEvaluateResultsFailed(t *testing.T) { message.NoProgress = true mockThresholdResult := oscalTypes_1_1_2.Result{ Findings: &[]oscalTypes_1_1_2.Finding{ - findingMap["ID-1_pass"], + findingMapPass["ID-1"], }, } mockEvaluationResult := oscalTypes_1_1_2.Result{ Findings: &[]oscalTypes_1_1_2.Finding{ - findingMap["ID-1_fail"], + findingMapFail["ID-1"], }, } From c6d714e050b32dd8afdff43b40abf329aeb876de Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Tue, 4 Jun 2024 19:16:23 +0000 Subject: [PATCH 07/12] fix(evaluate): cleanup testing files --- demo/oscal-component-opa-2.yaml | 81 -------- .../oscal/invalid-assessment-result.yaml | 11 -- .../common/oscal/valid-assessment-result.yaml | 181 ------------------ 3 files changed, 273 deletions(-) delete mode 100644 demo/oscal-component-opa-2.yaml delete mode 100644 src/test/unit/common/oscal/invalid-assessment-result.yaml delete mode 100644 src/test/unit/common/oscal/valid-assessment-result.yaml diff --git a/demo/oscal-component-opa-2.yaml b/demo/oscal-component-opa-2.yaml deleted file mode 100644 index 8458890d..00000000 --- a/demo/oscal-component-opa-2.yaml +++ /dev/null @@ -1,81 +0,0 @@ -component-definition: - uuid: E6A291A4-2BC8-43A0-B4B2-FD67CAAE1F8F - metadata: - title: Lula Demo - last-modified: '2022-09-13T12:00:00Z' - version: "20220913" - oscal-version: 1.1.1 # This version should remain one version behind latest version for `lula dev upgrade` demo - parties: - # Should be consistent across all of the packages, but where is ground truth? - - uuid: C18F4A9F-A402-415B-8D13-B51739D689FF - type: organization - name: Lula Development - links: - - href: https://github.com/defenseunicorns/lula - rel: website - components: - - uuid: A9D5204C-7E5B-4C43-BD49-34DF759B9F04 - type: software - title: lula - description: | - Lula - the Compliance Validator - purpose: Validate compliance controls - responsible-roles: - - role-id: provider - party-uuids: - - C18F4A9F-A402-415B-8D13-B51739D689FF # matches parties entry for Defense Unicorns - control-implementations: - - uuid: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A - source: https://raw.githubusercontent.com/usnistgov/oscal-content/master/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json - description: - Validate generic security requirements - implemented-requirements: - - uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD - control-id: ID-1 - description: >- - This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - links: - - href: '#a7377430-2328-4dc4-a9e2-b3f31dc1dff9' - rel: lula - - uuid: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 - source: https://raw.githubusercontent.com/usnistgov/oscal-content/master/nist.gov/SP800-53/rev5/json/NIST_SP-800-53_rev5_catalog.json - description: - Validate generic security requirements - implemented-requirements: - - uuid: 42C2FFDC-5F05-44DF-A67F-EEC8660AEFFD - control-id: ID-2 - description: >- - This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - links: - - href: '#a7377430-2328-4dc4-a9e2-b3f31dc1dff9' - rel: lula - back-matter: - resources: - - uuid: a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - rlinks: - - href: lula.dev - description: >- - domain: - type: kubernetes - kubernetes-spec: - resources: - - name: podsvt - resource-rule: - group: - version: v1 - resource: pods - namespaces: [validation-test] - provider: - type: opa - opa-spec: - rego: | - package validate - - import future.keywords.every - - validate { - every pod in input.podsvt { - podLabel := pod.metadata.labels.foo - podLabel == "bar" - } - } \ No newline at end of file diff --git a/src/test/unit/common/oscal/invalid-assessment-result.yaml b/src/test/unit/common/oscal/invalid-assessment-result.yaml deleted file mode 100644 index 4d429b4b..00000000 --- a/src/test/unit/common/oscal/invalid-assessment-result.yaml +++ /dev/null @@ -1,11 +0,0 @@ -assessment-results: - import-ap: - href: "" - metadata: - last-modified: 2024-06-04T02:34:02.653177582Z - oscal-version: 1.1.2 - published: 2024-06-04T02:34:02.653177582Z - remarks: Assessment Results generated from Lula - title: '[System Name] Security Assessment Results (SAR)' - version: 0.0.1 - uuid: 19541f5a-7738-4d86-b53f-34f3dc7a1586 diff --git a/src/test/unit/common/oscal/valid-assessment-result.yaml b/src/test/unit/common/oscal/valid-assessment-result.yaml deleted file mode 100644 index c886453c..00000000 --- a/src/test/unit/common/oscal/valid-assessment-result.yaml +++ /dev/null @@ -1,181 +0,0 @@ -assessment-results: - import-ap: - href: "" - metadata: - last-modified: 2024-06-04T03:34:46.521030308Z - oscal-version: 1.1.2 - published: 2024-06-04T02:46:29.02587111Z - remarks: Assessment Results generated from Lula - title: '[System Name] Security Assessment Results (SAR)' - version: 0.0.1 - results: - - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43 - findings: - - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - related-observations: - - observation-uuid: bcb5c77a-981d-4698-acd4-84c3e8a0b3e7 - target: - status: - state: satisfied - target-id: ID-1 - type: objective-id - title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' - uuid: 41018849-9825-4760-82ad-6d74362e44d5 - - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - related-observations: - - observation-uuid: 38d1abe2-7c30-4eaa-b792-46a0978bf0a0 - target: - status: - state: satisfied - target-id: ID-2 - type: objective-id - title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: 60345fd5-f0b8-4097-b9a2-b4f795edcab3 / Control: ID-2' - uuid: 9fd5633e-2509-41bc-b66c-16d30e7dc29e - observations: - - collected: 2024-06-04T03:34:46.435905414Z - description: |+ - [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - - methods: - - TEST - relevant-evidence: - - description: | - Result: satisfied - uuid: bcb5c77a-981d-4698-acd4-84c3e8a0b3e7 - - collected: 2024-06-04T03:34:46.441327814Z - description: |+ - [TEST]: ID-2 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - - methods: - - TEST - relevant-evidence: - - description: | - Result: satisfied - uuid: 38d1abe2-7c30-4eaa-b792-46a0978bf0a0 - props: - - name: threshold - ns: https://docs.lula.dev/ns - value: "true" - reviewed-controls: - control-selections: - - description: Controls Assessed by Lula - include-controls: - - control-id: ID-1 - - control-id: ID-2 - description: Controls validated - remarks: Validation performed may indicate full or partial satisfaction - start: 2024-06-04T03:34:46.443161469Z - title: Lula Validation Result - uuid: b5fb7bd4-c91e-4cd3-8020-ed04aa8d74b5 - - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43 - findings: - - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - related-observations: - - observation-uuid: 42a8b71d-7d08-42cc-beb4-c1c384c119ef - target: - status: - state: satisfied - target-id: ID-1 - type: objective-id - title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' - uuid: 50c818bd-e6be-4870-8766-8a22d18f1af1 - observations: - - collected: 2024-06-04T03:33:33.977370866Z - description: |+ - [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - - methods: - - TEST - relevant-evidence: - - description: | - Result: satisfied - uuid: 42a8b71d-7d08-42cc-beb4-c1c384c119ef - props: - - name: threshold - ns: https://docs.lula.dev/ns - value: "false" - reviewed-controls: - control-selections: - - description: Controls Assessed by Lula - include-controls: - - control-id: ID-1 - description: Controls validated - remarks: Validation performed may indicate full or partial satisfaction - start: 2024-06-04T03:33:33.982591133Z - title: Lula Validation Result - uuid: ae611955-e97e-4115-a5d0-2e5ccbf35872 - - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43 - findings: - - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - related-observations: - - observation-uuid: 20d943cb-8a8d-47e2-b625-2d5829604f5e - target: - status: - state: satisfied - target-id: ID-1 - type: objective-id - title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' - uuid: fbee2fb0-e8ba-419c-9d29-f20525bbc420 - observations: - - collected: 2024-06-04T02:48:06.890379789Z - description: |+ - [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - - methods: - - TEST - relevant-evidence: - - description: | - Result: satisfied - uuid: 20d943cb-8a8d-47e2-b625-2d5829604f5e - props: - - name: threshold - ns: https://docs.lula.dev/ns - value: "false" - reviewed-controls: - control-selections: - - description: Controls Assessed by Lula - include-controls: - - control-id: ID-1 - description: Controls validated - remarks: Validation performed may indicate full or partial satisfaction - start: 2024-06-04T02:48:06.895205436Z - title: Lula Validation Result - uuid: 7ef25ccd-22b8-421e-8c64-9fcbde697696 - - description: Assessment results for performing Validations with Lula version v0.3.0-7-gfc18c43 - findings: - - description: This control validates that the demo-pod pod in the validation-test namespace contains the required pod label foo=bar in order to establish compliance. - related-observations: - - observation-uuid: d3b228e8-8c8b-4be9-a0b0-8da9fcdb005f - target: - status: - state: not-satisfied - target-id: ID-1 - type: objective-id - title: 'Validation Result - Component:A9D5204C-7E5B-4C43-BD49-34DF759B9F04 / Control Implementation: A584FEDC-8CEA-4B0C-9F07-85C2C4AE751A / Control: ID-1' - uuid: 44fc8e49-b633-4880-8d58-4ed0f14c7bdb - observations: - - collected: 2024-06-04T02:46:29.018416946Z - description: |+ - [TEST]: ID-1 - a7377430-2328-4dc4-a9e2-b3f31dc1dff9 - - methods: - - TEST - relevant-evidence: - - description: | - Result: not-satisfied - uuid: d3b228e8-8c8b-4be9-a0b0-8da9fcdb005f - props: - - name: threshold - ns: https://docs.lula.dev/ns - value: "false" - reviewed-controls: - control-selections: - - description: Controls Assessed by Lula - include-controls: - - control-id: ID-1 - description: Controls validated - remarks: Validation performed may indicate full or partial satisfaction - start: 2024-06-04T02:46:29.02587111Z - title: Lula Validation Result - uuid: 250ac234-519f-4f26-bee3-98e870feeb7a - uuid: 9673739a-9379-4575-b454-1b42672a4970 From 9dacfb711cc10c163ba43f5473d0143e6b759053 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Fri, 7 Jun 2024 23:28:09 +0000 Subject: [PATCH 08/12] fix(evaluate): WIP for tests and updated logic --- src/pkg/common/oscal/assessment-results.go | 10 +++ .../common/oscal/assessment-results_test.go | 68 ++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index e70a6528..4421853f 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -139,6 +139,7 @@ func GenerateFindingsMap(findings []oscalTypes_1_1_2.Finding) map[string]oscalTy } // IdentifyResults produces a map containing the threshold result and a result used for comparison +// TODO: need to look further at the logic behind this - what do we do when the threshold is the latest result? func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResults) (map[string]*oscalTypes_1_1_2.Result, error) { resultMap := make(map[string]*oscalTypes_1_1_2.Result) @@ -159,12 +160,21 @@ func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResult return resultMap, nil } else { // Constraint - Always evaluate the latest threshold against the latest result + // Unless they are the same pointer resultMap["threshold"] = thresholds[len(thresholds)-1] + // Consider changing the namespace value to "false" here - only written if the command logic completes for _, result := range thresholds { UpdateProps("threshold", "https://docs.lula.dev/ns", "false", result.Props) } resultMap["latest"] = sortedResults[len(sortedResults)-1] + + if resultMap["threshold"] == resultMap["latest"] { + // Maybe we should consider returning an error here + // TODO: why does the test pass in this scenario? + resultMap["threshold"] = sortedResults[len(sortedResults)-2] + } + return resultMap, nil } } diff --git a/src/pkg/common/oscal/assessment-results_test.go b/src/pkg/common/oscal/assessment-results_test.go index 09eb5eb1..264b8c16 100644 --- a/src/pkg/common/oscal/assessment-results_test.go +++ b/src/pkg/common/oscal/assessment-results_test.go @@ -52,6 +52,7 @@ var observations = []oscalTypes_1_1_2.Observation{ func TestIdentifyResults(t *testing.T) { t.Parallel() + // Expecting an error when evaluating a single result t.Run("Handle valid assessment containing a single result", func(t *testing.T) { assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) @@ -70,7 +71,8 @@ func TestIdentifyResults(t *testing.T) { } }) - t.Run("Handle multiple valid assessment containing a single result", func(t *testing.T) { + // Identify threshold for multiple assessments and evaluate passing + t.Run("Handle multiple valid assessment containing a single result - pass", func(t *testing.T) { assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) if err != nil { @@ -84,8 +86,46 @@ func TestIdentifyResults(t *testing.T) { // key name does not matter here var assessmentMap = map[string]*oscalTypes_1_1_2.AssessmentResults{ - "valid.yaml": assessment, - "other.yaml": assessment2, + "valid.yaml": assessment, + "invalid.yaml": assessment2, + } + + resultMap, err := oscal.IdentifyResults(assessmentMap) + if err != nil { + t.Fatalf("Expected error for inability to identify multiple results : %v", err) + } + + if resultMap["threshold"] == nil || resultMap["latest"] == nil { + t.Fatalf("Expected results to be identified") + } + + status, _, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) + if err != nil { + t.Fatalf("Expected error for inability to evaluate multiple results : %v", err) + } + + if !status { + t.Fatalf("Expected results to be evaluated as passing") + } + }) + + // Identify threshold for multiple assessments and evaluate failing + t.Run("Handle multiple valid assessment containing a single result - fail", func(t *testing.T) { + + assessment2, err := oscal.GenerateAssessmentResults(findingMapFail, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } + + assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } + + // key name does not matter here + var assessmentMap = map[string]*oscalTypes_1_1_2.AssessmentResults{ + "valid.yaml": assessment, + "invalid.yaml": assessment2, } resultMap, err := oscal.IdentifyResults(assessmentMap) @@ -96,7 +136,29 @@ func TestIdentifyResults(t *testing.T) { if resultMap["threshold"] == nil || resultMap["latest"] == nil { t.Fatalf("Expected results to be identified") } + + status, _, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) + if err != nil { + t.Fatalf("Expected error for inability to evaluate multiple results : %v", err) + } + + if status { + t.Fatalf("Expected results to be evaluated as failing") + } }) + + // + // t.Run("Test merging two assessments - evaluate for passing", func(t *testing.T) { + + // }) + + // t.Run("test title", func(t *testing.T) { + + // }) + + // t.Run("test title", func(t *testing.T) { + + // }) } // Given two results - evaluate for passing From a52e7161e9aa3b6c52bd095c58e89585b7739e63 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Sat, 8 Jun 2024 18:20:41 +0000 Subject: [PATCH 09/12] fix(evaluate): updated logic and testing for edge case --- src/pkg/common/oscal/assessment-results.go | 40 +++++++++++++------ .../common/oscal/assessment-results_test.go | 11 +++-- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index 4421853f..1baefe22 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -143,36 +143,50 @@ func GenerateFindingsMap(findings []oscalTypes_1_1_2.Finding) map[string]oscalTy func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResults) (map[string]*oscalTypes_1_1_2.Result, error) { resultMap := make(map[string]*oscalTypes_1_1_2.Result) - thresholds, sortedResults, err := findAndSortResults(assessmentMap) - if err != nil { - return nil, err - } + thresholds, sortedResults := findAndSortResults(assessmentMap) if len(sortedResults) <= 1 { return nil, fmt.Errorf("less than 2 results found - no comparison possible") } if len(thresholds) == 0 { - // No thresholds identified but we have > 1 results - compare the latest and the preceding + // No thresholds identified but we have > 1 results - compare the latest against the preceding resultMap["threshold"] = sortedResults[len(sortedResults)-2] resultMap["latest"] = sortedResults[len(sortedResults)-1] return resultMap, nil - } else { - // Constraint - Always evaluate the latest threshold against the latest result - // Unless they are the same pointer + } else if len(thresholds) > 1 { + // TODO: what to do when we have >1 threshold - likely the case when you create + // two assessment results in two separate files and attempt to evaluate + // We can fix this by updating all of the threshold props on subsequent runs resultMap["threshold"] = thresholds[len(thresholds)-1] + resultMap["latest"] = sortedResults[len(sortedResults)-1] + + if resultMap["threshold"] == resultMap["latest"] { + // if threshold is latest here && we have > 1 threshold - make the threshold the older threshold + resultMap["threshold"] = thresholds[len(thresholds)-2] + } // Consider changing the namespace value to "false" here - only written if the command logic completes for _, result := range thresholds { UpdateProps("threshold", "https://docs.lula.dev/ns", "false", result.Props) } + + return resultMap, nil + + } else { + // Constraint - Always evaluate the latest threshold against the latest result + // Unless they are the same pointer + resultMap["threshold"] = thresholds[len(thresholds)-1] resultMap["latest"] = sortedResults[len(sortedResults)-1] if resultMap["threshold"] == resultMap["latest"] { - // Maybe we should consider returning an error here - // TODO: why does the test pass in this scenario? - resultMap["threshold"] = sortedResults[len(sortedResults)-2] + return nil, fmt.Errorf("latest threshold is the latest result - no comparison possible") + } + + // Consider changing the namespace value to "false" here - only written if the command logic completes + for _, result := range thresholds { + UpdateProps("threshold", "https://docs.lula.dev/ns", "false", result.Props) } return resultMap, nil @@ -228,7 +242,7 @@ func EvaluateResults(thresholdResult *oscalTypes_1_1_2.Result, newResult *oscalT } // findAndSortResults takes a map of results and returns a list of thresholds and a sorted list of results in order of time -func findAndSortResults(resultMap map[string]*oscalTypes_1_1_2.AssessmentResults) ([]*oscalTypes_1_1_2.Result, []*oscalTypes_1_1_2.Result, error) { +func findAndSortResults(resultMap map[string]*oscalTypes_1_1_2.AssessmentResults) ([]*oscalTypes_1_1_2.Result, []*oscalTypes_1_1_2.Result) { thresholds := make([]*oscalTypes_1_1_2.Result, 0) sortedResults := make([]*oscalTypes_1_1_2.Result, 0) @@ -251,7 +265,7 @@ func findAndSortResults(resultMap map[string]*oscalTypes_1_1_2.AssessmentResults slices.SortFunc(sortedResults, func(a, b *oscalTypes_1_1_2.Result) int { return a.Start.Compare(b.Start) }) slices.SortFunc(thresholds, func(a, b *oscalTypes_1_1_2.Result) int { return a.Start.Compare(b.Start) }) - return thresholds, sortedResults, nil + return thresholds, sortedResults } // Helper function to create observation diff --git a/src/pkg/common/oscal/assessment-results_test.go b/src/pkg/common/oscal/assessment-results_test.go index 264b8c16..81e74313 100644 --- a/src/pkg/common/oscal/assessment-results_test.go +++ b/src/pkg/common/oscal/assessment-results_test.go @@ -79,7 +79,7 @@ func TestIdentifyResults(t *testing.T) { t.Fatalf("error generating assessment results: %v", err) } - assessment2, err := oscal.GenerateAssessmentResults(findingMapFail, observations) + assessment2, err := oscal.GenerateAssessmentResults(findingMapPass, observations) if err != nil { t.Fatalf("error generating assessment results: %v", err) } @@ -92,7 +92,7 @@ func TestIdentifyResults(t *testing.T) { resultMap, err := oscal.IdentifyResults(assessmentMap) if err != nil { - t.Fatalf("Expected error for inability to identify multiple results : %v", err) + t.Fatalf("Expected no error for inability to identify multiple results : %v", err) } if resultMap["threshold"] == nil || resultMap["latest"] == nil { @@ -107,17 +107,20 @@ func TestIdentifyResults(t *testing.T) { if !status { t.Fatalf("Expected results to be evaluated as passing") } + + // TODO: check for threshold updates here + }) // Identify threshold for multiple assessments and evaluate failing t.Run("Handle multiple valid assessment containing a single result - fail", func(t *testing.T) { - assessment2, err := oscal.GenerateAssessmentResults(findingMapFail, observations) + assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) if err != nil { t.Fatalf("error generating assessment results: %v", err) } - assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) + assessment2, err := oscal.GenerateAssessmentResults(findingMapFail, observations) if err != nil { t.Fatalf("error generating assessment results: %v", err) } From 31268021a6700e9b75511a2d93aae3d7e0b2c40b Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Mon, 10 Jun 2024 23:51:01 +0000 Subject: [PATCH 10/12] fix(evaluate): additional testing after merge - updating merge logic --- src/pkg/common/oscal/assessment-results.go | 25 ++--- .../common/oscal/assessment-results_test.go | 92 +++++++++++++++++-- 2 files changed, 93 insertions(+), 24 deletions(-) diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index 1baefe22..0358ed58 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -108,18 +108,13 @@ func MergeAssessmentResults(original *oscalTypes_1_1_2.AssessmentResults, latest // Validate only ever creates one result // Assumed that there is always an original threshold - result := latest.Results[0] - for index, prop := range *result.Props { - if prop.Name == "threshold" { - prop.Value = "false" - // Better way to update the prop? - (*result.Props)[index] = prop - } - } + // TODO: modify the below behavior to remove awareness of how many results exist during merge + // TODO: append all results to a slice and the sort such that the newest is prepended to the results + UpdateProps("threshold", "docs.lula.dev/ns", "false", latest.Results[0].Props) results := make([]oscalTypes_1_1_2.Result, 0) - // append newest to oldest results - results = append(results, result) + // append newest to oldest results? or rather should we sort? + results = append(results, latest.Results[0]) results = append(results, original.Results...) original.Results = results @@ -139,7 +134,6 @@ func GenerateFindingsMap(findings []oscalTypes_1_1_2.Finding) map[string]oscalTy } // IdentifyResults produces a map containing the threshold result and a result used for comparison -// TODO: need to look further at the logic behind this - what do we do when the threshold is the latest result? func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResults) (map[string]*oscalTypes_1_1_2.Result, error) { resultMap := make(map[string]*oscalTypes_1_1_2.Result) @@ -150,15 +144,13 @@ func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResult } if len(thresholds) == 0 { - // No thresholds identified but we have > 1 results - compare the latest against the preceding + // No thresholds identified but we have > 1 results - compare the preceding (threshold) against the latest resultMap["threshold"] = sortedResults[len(sortedResults)-2] resultMap["latest"] = sortedResults[len(sortedResults)-1] return resultMap, nil } else if len(thresholds) > 1 { - // TODO: what to do when we have >1 threshold - likely the case when you create - // two assessment results in two separate files and attempt to evaluate - // We can fix this by updating all of the threshold props on subsequent runs + // More than one threshold - likely the case with multiple assessment-results artifacts resultMap["threshold"] = thresholds[len(thresholds)-1] resultMap["latest"] = sortedResults[len(sortedResults)-1] @@ -175,8 +167,7 @@ func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResult return resultMap, nil } else { - // Constraint - Always evaluate the latest threshold against the latest result - // Unless they are the same pointer + // Otherwise we have a single threshold and we compare that against the latest result resultMap["threshold"] = thresholds[len(thresholds)-1] resultMap["latest"] = sortedResults[len(sortedResults)-1] diff --git a/src/pkg/common/oscal/assessment-results_test.go b/src/pkg/common/oscal/assessment-results_test.go index 81e74313..fb413d88 100644 --- a/src/pkg/common/oscal/assessment-results_test.go +++ b/src/pkg/common/oscal/assessment-results_test.go @@ -72,7 +72,7 @@ func TestIdentifyResults(t *testing.T) { }) // Identify threshold for multiple assessments and evaluate passing - t.Run("Handle multiple valid assessment containing a single result - pass", func(t *testing.T) { + t.Run("Handle multiple threshold assessment containing a single result - pass", func(t *testing.T) { assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) if err != nil { @@ -113,7 +113,7 @@ func TestIdentifyResults(t *testing.T) { }) // Identify threshold for multiple assessments and evaluate failing - t.Run("Handle multiple valid assessment containing a single result - fail", func(t *testing.T) { + t.Run("Handle multiple threshold assessment containing a single result - fail", func(t *testing.T) { assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) if err != nil { @@ -150,14 +150,92 @@ func TestIdentifyResults(t *testing.T) { } }) - // - // t.Run("Test merging two assessments - evaluate for passing", func(t *testing.T) { + t.Run("Test merging two assessments - evaluate for passing", func(t *testing.T) { - // }) + assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } - // t.Run("test title", func(t *testing.T) { + assessment2, err := oscal.GenerateAssessmentResults(findingMapFail, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } - // }) + // Update assessment 2 props so that we only have 1 threshold + oscal.UpdateProps("threshold", "docs.lula.dev/ns", "false", assessment2.Results[0].Props) + + assessment, err = oscal.MergeAssessmentResults(assessment, assessment2) + if err != nil { + t.Fatalf("error merging assessment results: %v", err) + } + + var assessmentMap = map[string]*oscalTypes_1_1_2.AssessmentResults{ + "valid.yaml": assessment, + } + + resultMap, err := oscal.IdentifyResults(assessmentMap) + if err != nil { + t.Fatalf("Expected error for inability to identify multiple results : %v", err) + } + + if resultMap["threshold"] == nil || resultMap["latest"] == nil { + t.Fatalf("Expected results to be identified") + } + + status, _, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) + if err != nil { + t.Fatalf("Expected error for inability to evaluate multiple results : %v", err) + } + + if status { + t.Fatalf("Expected results to be evaluated as failing") + } + }) + + t.Run("Test merging two assessments - evaluate for passing", func(t *testing.T) { + + assessment2, err := oscal.GenerateAssessmentResults(findingMapFail, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } + + assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) + if err != nil { + t.Fatalf("error generating assessment results: %v", err) + } + + // Update assessment props so that we only have 1 threshold + oscal.UpdateProps("threshold", "docs.lula.dev/ns", "false", assessment.Results[0].Props) + + // TODO: review assumptions made about order of assessments during merge + assessment, err = oscal.MergeAssessmentResults(assessment2, assessment) + if err != nil { + t.Fatalf("error merging assessment results: %v", err) + } + + var assessmentMap = map[string]*oscalTypes_1_1_2.AssessmentResults{ + "valid.yaml": assessment, + } + + resultMap, err := oscal.IdentifyResults(assessmentMap) + if err != nil { + t.Fatalf("Expected error for inability to identify multiple results : %v", err) + } + + if resultMap["threshold"] == nil || resultMap["latest"] == nil { + t.Fatalf("Expected results to be identified") + } + + status, _, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) + if err != nil { + t.Fatalf("Expected error for inability to evaluate multiple results : %v", err) + } + + if !status { + t.Fatalf("Expected results to be evaluated as failing") + } + }) // t.Run("test title", func(t *testing.T) { From 347a7574f09d7f89bcddfa296132d31fc0a3f839 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Tue, 11 Jun 2024 04:01:47 +0000 Subject: [PATCH 11/12] fix(oscal): updated merge logic for assessment results --- src/pkg/common/oscal/assessment-results.go | 14 ++----------- .../common/oscal/assessment-results_test.go | 20 ++++++++++--------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index 0358ed58..a23140ca 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -100,24 +100,14 @@ func GenerateAssessmentResults(findingMap map[string]oscalTypes_1_1_2.Finding, o func MergeAssessmentResults(original *oscalTypes_1_1_2.AssessmentResults, latest *oscalTypes_1_1_2.AssessmentResults) (*oscalTypes_1_1_2.AssessmentResults, error) { // If UUID's are matching - this must be a prop update for threshold - // We should be able to return the latest results // This is used during evaluate to update the threshold prop automatically if original.UUID == latest.UUID { return latest, nil } - // Validate only ever creates one result - // Assumed that there is always an original threshold - // TODO: modify the below behavior to remove awareness of how many results exist during merge - // TODO: append all results to a slice and the sort such that the newest is prepended to the results - UpdateProps("threshold", "docs.lula.dev/ns", "false", latest.Results[0].Props) - - results := make([]oscalTypes_1_1_2.Result, 0) - // append newest to oldest results? or rather should we sort? - results = append(results, latest.Results[0]) - results = append(results, original.Results...) - original.Results = results + original.Results = append(original.Results, latest.Results...) + slices.SortFunc(original.Results, func(a, b oscalTypes_1_1_2.Result) int { return b.Start.Compare(a.Start) }) // Update pertinent information original.Metadata.LastModified = time.Now() original.UUID = uuid.NewUUID() diff --git a/src/pkg/common/oscal/assessment-results_test.go b/src/pkg/common/oscal/assessment-results_test.go index fb413d88..78d96f70 100644 --- a/src/pkg/common/oscal/assessment-results_test.go +++ b/src/pkg/common/oscal/assessment-results_test.go @@ -108,8 +108,6 @@ func TestIdentifyResults(t *testing.T) { t.Fatalf("Expected results to be evaluated as passing") } - // TODO: check for threshold updates here - }) // Identify threshold for multiple assessments and evaluate failing @@ -150,7 +148,7 @@ func TestIdentifyResults(t *testing.T) { } }) - t.Run("Test merging two assessments - evaluate for passing", func(t *testing.T) { + t.Run("Test merging two assessments - passing", func(t *testing.T) { assessment, err := oscal.GenerateAssessmentResults(findingMapPass, observations) if err != nil { @@ -193,7 +191,7 @@ func TestIdentifyResults(t *testing.T) { } }) - t.Run("Test merging two assessments - evaluate for passing", func(t *testing.T) { + t.Run("Test merging two assessments - failing", func(t *testing.T) { assessment2, err := oscal.GenerateAssessmentResults(findingMapFail, observations) if err != nil { @@ -206,14 +204,21 @@ func TestIdentifyResults(t *testing.T) { } // Update assessment props so that we only have 1 threshold - oscal.UpdateProps("threshold", "docs.lula.dev/ns", "false", assessment.Results[0].Props) + oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "false", assessment.Results[0].Props) // TODO: review assumptions made about order of assessments during merge - assessment, err = oscal.MergeAssessmentResults(assessment2, assessment) + assessment, err = oscal.MergeAssessmentResults(assessment, assessment2) if err != nil { t.Fatalf("error merging assessment results: %v", err) } + // t.Log(assessment.Results) + + for _, item := range assessment.Results { + t.Logf("Start: %s", item.Start) + t.Logf("Props: %s", item.Props) + t.Log() + } var assessmentMap = map[string]*oscalTypes_1_1_2.AssessmentResults{ "valid.yaml": assessment, } @@ -237,9 +242,6 @@ func TestIdentifyResults(t *testing.T) { } }) - // t.Run("test title", func(t *testing.T) { - - // }) } // Given two results - evaluate for passing From 85dc024ef61323a2f2c62ba5b9008d64baa0d056 Mon Sep 17 00:00:00 2001 From: Brandt Keller Date: Tue, 11 Jun 2024 17:47:14 +0000 Subject: [PATCH 12/12] fix(evaluate): cleanup, update props, testing --- src/cmd/evaluate/evaluate.go | 122 ++++++++++-------- src/pkg/common/oscal/assessment-results.go | 5 - .../common/oscal/assessment-results_test.go | 23 +++- 3 files changed, 82 insertions(+), 68 deletions(-) diff --git a/src/cmd/evaluate/evaluate.go b/src/cmd/evaluate/evaluate.go index 1f5d9915..e91f3b03 100644 --- a/src/cmd/evaluate/evaluate.go +++ b/src/cmd/evaluate/evaluate.go @@ -39,80 +39,90 @@ var evaluateCmd = &cobra.Command{ message.Fatal(err, err.Error()) } - // Identify the threshold & latest for comparison - resultMap, err := oscal.IdentifyResults(assessmentMap) - if err != nil { - if err.Error() == "less than 2 results found - no comparison possible" { - // Catch and warn of insufficient results - message.Warn(err.Error()) - return - } else { - message.Fatal(err, err.Error()) - } - } + EvaluateAssessments(assessmentMap) + }, +} - if resultMap["threshold"] != nil && resultMap["latest"] != nil { - // Compare the assessment results - spinner := message.NewProgressSpinner("Evaluating Assessment Results %s against %s", resultMap["threshold"].UUID, resultMap["latest"].UUID) - defer spinner.Stop() +func EvaluateCommand() *cobra.Command { - status, findings, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) - if err != nil { - message.Fatal(err, err.Error()) - } + evaluateCmd.Flags().StringArrayVarP(&opts.files, "file", "f", []string{}, "Path to the file to be evaluated") + // insert flag options here + return evaluateCmd +} - if status { - if len(findings["new-passing-findings"]) > 0 { - message.Info("New passing finding Target-Ids:") - for _, finding := range findings["new-passing-findings"] { - message.Infof("%s", finding.Target.TargetId) - } +func EvaluateAssessments(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResults) { + // Identify the threshold & latest for comparison + resultMap, err := oscal.IdentifyResults(assessmentMap) + if err != nil { + if err.Error() == "less than 2 results found - no comparison possible" { + // Catch and warn of insufficient results + message.Warn(err.Error()) + return + } else { + message.Fatal(err, err.Error()) + } + } - message.Infof("New threshold identified - threshold will be updated to result %s", resultMap["latest"].UUID) + if resultMap["threshold"] != nil && resultMap["latest"] != nil { + // Compare the assessment results + spinner := message.NewProgressSpinner("Evaluating Assessment Results %s against %s", resultMap["threshold"].UUID, resultMap["latest"].UUID) + defer spinner.Stop() - // Update latest threshold prop - oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "true", resultMap["latest"].Props) - } + status, findings, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) + if err != nil { + message.Fatal(err, err.Error()) + } - if len(findings["new-failing-findings"]) > 0 { - message.Info("New failing finding Target-Ids:") - for _, finding := range findings["new-failing-findings"] { - message.Infof("%s", finding.Target.TargetId) - } + if status { + if len(findings["new-passing-findings"]) > 0 { + message.Info("New passing finding Target-Ids:") + for _, finding := range findings["new-passing-findings"] { + message.Infof("%s", finding.Target.TargetId) } + message.Infof("New threshold identified - threshold will be updated to result %s", resultMap["latest"].UUID) + + // Update latest threshold prop + oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "true", resultMap["latest"].Props) } else { - message.Warn("Evaluation Failed against the following findings:") - for _, finding := range findings["no-longer-satisfied"] { - message.Warnf("%s", finding.Target.TargetId) - } - message.Fatalf(fmt.Errorf("failed to meet established threshold"), "failed to meet established threshold") + // retain result as threshold + oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "true", resultMap["threshold"].Props) } - spinner.Success() + if len(findings["new-failing-findings"]) > 0 { + message.Info("New failing finding Target-Ids:") + for _, finding := range findings["new-failing-findings"] { + message.Infof("%s", finding.Target.TargetId) + } + } - } else if resultMap["threshold"] == nil { - message.Fatal(fmt.Errorf("no threshold assessment results could be identified"), "no threshold assessment results could be identified") } else { - message.Fatal(fmt.Errorf("no latest assessment results could be identified"), "no latest assessment results could be identified") - } - - // Write each file back in the case of modification - for filePath, assessment := range assessmentMap { - model := oscalTypes_1_1_2.OscalCompleteSchema{ - AssessmentResults: assessment, + message.Warn("Evaluation Failed against the following findings:") + for _, finding := range findings["no-longer-satisfied"] { + message.Warnf("%s", finding.Target.TargetId) } + message.Fatalf(fmt.Errorf("failed to meet established threshold"), "failed to meet established threshold") - oscal.WriteOscalModel(filePath, &model) + // retain result as threshold + oscal.UpdateProps("threshold", "https://docs.lula.dev/ns", "true", resultMap["threshold"].Props) } - }, -} -func EvaluateCommand() *cobra.Command { + spinner.Success() - evaluateCmd.Flags().StringArrayVarP(&opts.files, "file", "f", []string{}, "Path to the file to be evaluated") - // insert flag options here - return evaluateCmd + } else if resultMap["threshold"] == nil { + message.Fatal(fmt.Errorf("no threshold assessment results could be identified"), "no threshold assessment results could be identified") + } else { + message.Fatal(fmt.Errorf("no latest assessment results could be identified"), "no latest assessment results could be identified") + } + + // Write each file back in the case of modification + for filePath, assessment := range assessmentMap { + model := oscalTypes_1_1_2.OscalCompleteSchema{ + AssessmentResults: assessment, + } + + oscal.WriteOscalModel(filePath, &model) + } } // Read many filepaths into a map[filepath]*AssessmentResults diff --git a/src/pkg/common/oscal/assessment-results.go b/src/pkg/common/oscal/assessment-results.go index a23140ca..62693210 100644 --- a/src/pkg/common/oscal/assessment-results.go +++ b/src/pkg/common/oscal/assessment-results.go @@ -165,11 +165,6 @@ func IdentifyResults(assessmentMap map[string]*oscalTypes_1_1_2.AssessmentResult return nil, fmt.Errorf("latest threshold is the latest result - no comparison possible") } - // Consider changing the namespace value to "false" here - only written if the command logic completes - for _, result := range thresholds { - UpdateProps("threshold", "https://docs.lula.dev/ns", "false", result.Props) - } - return resultMap, nil } } diff --git a/src/pkg/common/oscal/assessment-results_test.go b/src/pkg/common/oscal/assessment-results_test.go index 78d96f70..0b596d1c 100644 --- a/src/pkg/common/oscal/assessment-results_test.go +++ b/src/pkg/common/oscal/assessment-results_test.go @@ -99,6 +99,10 @@ func TestIdentifyResults(t *testing.T) { t.Fatalf("Expected results to be identified") } + if resultMap["threshold"].Start.After(resultMap["latest"].Start) { + t.Fatalf("Expected threshold result to be before latest result") + } + status, _, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) if err != nil { t.Fatalf("Expected error for inability to evaluate multiple results : %v", err) @@ -138,6 +142,10 @@ func TestIdentifyResults(t *testing.T) { t.Fatalf("Expected results to be identified") } + if resultMap["threshold"].Start.After(resultMap["latest"].Start) { + t.Fatalf("Expected threshold result to be before latest result") + } + status, _, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) if err != nil { t.Fatalf("Expected error for inability to evaluate multiple results : %v", err) @@ -181,6 +189,10 @@ func TestIdentifyResults(t *testing.T) { t.Fatalf("Expected results to be identified") } + if resultMap["threshold"].Start.After(resultMap["latest"].Start) { + t.Fatalf("Expected threshold result to be before latest result") + } + status, _, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) if err != nil { t.Fatalf("Expected error for inability to evaluate multiple results : %v", err) @@ -212,13 +224,6 @@ func TestIdentifyResults(t *testing.T) { t.Fatalf("error merging assessment results: %v", err) } - // t.Log(assessment.Results) - - for _, item := range assessment.Results { - t.Logf("Start: %s", item.Start) - t.Logf("Props: %s", item.Props) - t.Log() - } var assessmentMap = map[string]*oscalTypes_1_1_2.AssessmentResults{ "valid.yaml": assessment, } @@ -232,6 +237,10 @@ func TestIdentifyResults(t *testing.T) { t.Fatalf("Expected results to be identified") } + if resultMap["threshold"].Start.After(resultMap["latest"].Start) { + t.Fatalf("Expected threshold result to be before latest result") + } + status, _, err := oscal.EvaluateResults(resultMap["threshold"], resultMap["latest"]) if err != nil { t.Fatalf("Expected error for inability to evaluate multiple results : %v", err)