feat(misconf): Support custom data for rego policies for cloud (#4745)

* feat(misconf): Support custom data for cloud policies Signed-off-by: Simar <[email protected]> * use policyfs Signed-off-by: Simar <[email protected]> * refactor to reduce cyclomatic complexity Signed-off-by: Simar <[email protected]> * bump defsec * update docs Signed-off-by: Simar <[email protected]> * update test assertion Signed-off-by: Simar <[email protected]> * update test Need this as OPA is currently broken on Windows open-policy-agent/opa#4521 Signed-off-by: Simar <[email protected]> * fix data path * fix(mapfs): convert volume names into dirs * revert creating temp dirs --------- Signed-off-by: Simar <[email protected]> Co-authored-by: Teppei Fukuda <[email protected]>
aquasecurity · Jul 17, 2023 · 4bc8d29 · 4bc8d29
1 parent 88243a0
commit 4bc8d29
Show file tree

Hide file tree

Showing 8 changed files with 152 additions and 74 deletions.
diff --git a/docs/docs/scanner/misconfiguration/custom/data.md b/docs/docs/scanner/misconfiguration/custom/data.md
@@ -3,7 +3,7 @@
 Custom policies may require additional data in order to determine an answer.
 
 For example, an allowed list of resources that can be created. 
-Instead of hardcoding this information inside of your policy, Trivy allows passing paths to data files with the `--data` flag.
+Instead of hardcoding this information inside your policy, Trivy allows passing paths to data files with the `--data` flag.
 
 Given the following yaml file:
 

diff --git a/docs/docs/target/aws.md b/docs/docs/target/aws.md
@@ -103,5 +103,7 @@ Regardless of whether the cache is used or not, rules will be evaluated again wi
 
 You can write custom policies for Trivy to evaluate against your AWS account.
 These policies are written in [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/), the same language used by [Open Policy Agent](https://www.openpolicyagent.org/).
-See the [Custom Policies](../scanner/misconfiguration/custom/index.md) page for more information.
+See the [Custom Policies](../scanner/misconfiguration/custom/index.md) page for more information on how to write custom policies.
 
+Custom policies in cloud scanning also support passing in custom data. This can be useful when you want to selectively enable/disable certain aspects of your cloud policies.
+See the [Custom Data](../scanner/misconfiguration/custom/data.md) page for more information on how to provide custom data to custom policies.
diff --git a/pkg/cloud/aws/commands/run_test.go b/pkg/cloud/aws/commands/run_test.go
@@ -9,9 +9,9 @@ import (
 	"time"
 
 	defsecTypes "github.com/aquasecurity/defsec/pkg/types"
+	"github.com/aquasecurity/trivy/pkg/compliance/spec"
 
 	dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
-	"github.com/aquasecurity/trivy/pkg/compliance/spec"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -287,6 +287,7 @@ const expectedS3ScanResult = `{
   ]
 }
 `
+
 const expectedCustomScanResult = `{
   "ArtifactName": "12345678",
   "ArtifactType": "aws_account",
@@ -303,13 +304,46 @@ const expectedCustomScanResult = `{
     }
   },
   "Results": [
+    {
+      "Target": "",
+      "Class": "config",
+      "Type": "cloud",
+      "MisconfSummary": {
+        "Successes": 0,
+        "Failures": 1,
+        "Exceptions": 0
+      },
+      "Misconfigurations": [
+        {
+          "Type": "AWS",
+          "Title": "Bad input data",
+          "Description": "Just failing rule with input data",
+          "Message": "Rego policy resulted in DENY",
+          "Namespace": "user.whatever",
+          "Query": "deny",
+          "Severity": "LOW",
+          "References": [
+            ""
+          ],
+          "Status": "FAIL",
+          "Layer": {},
+          "CauseMetadata": {
+            "Provider": "cloud",
+            "Service": "s3",
+            "Code": {
+              "Lines": null
+            }
+          }
+        }
+      ]
+    },
     {
       "Target": "arn:aws:s3:::examplebucket",
       "Class": "config",
       "Type": "cloud",
       "MisconfSummary": {
         "Successes": 1,
-        "Failures": 10,
+        "Failures": 9,
         "Exceptions": 0
       },
       "Misconfigurations": [
@@ -551,34 +585,13 @@ const expectedCustomScanResult = `{
               "Lines": null
             }
           }
-        },
-        {
-          "Type": "AWS",
-          "Title": "No example buckets",
-          "Description": "Buckets should not be named with \"example\" in the name",
-          "Message": "example bucket detected",
-          "Namespace": "user.whatever",
-          "Query": "deny",
-          "Severity": "LOW",
-          "References": [
-            ""
-          ],
-          "Status": "FAIL",
-          "Layer": {},
-          "CauseMetadata": {
-            "Resource": "arn:aws:s3:::examplebucket",
-            "Provider": "cloud",
-            "Service": "s3",
-            "Code": {
-              "Lines": null
-            }
-          }
         }
       ]
     }
   ]
 }
 `
+
 const expectedS3AndCloudTrailResult = `{
   "ArtifactName": "123456789",
   "ArtifactType": "aws_account",
@@ -958,7 +971,6 @@ const expectedS3AndCloudTrailResult = `{
 `
 
 func Test_Run(t *testing.T) {
-
 	regoDir := t.TempDir()
 
 	tests := []struct {
@@ -969,6 +981,7 @@ func Test_Run(t *testing.T) {
 		cacheContent string
 		regoPolicy   string
 		allServices  []string
+		inputData    string
 	}{
 		{
 			name: "fail without region",
@@ -1042,13 +1055,16 @@ func Test_Run(t *testing.T) {
 					PolicyNamespaces: []string{
 						"user",
 					},
+					DataPaths: []string{
+						filepath.Join(regoDir, "data"),
+					},
 					SkipPolicyUpdate: true,
 				},
 				MisconfOptions: flag.MisconfOptions{IncludeNonFailures: true},
 			},
 			regoPolicy: `# METADATA
-# title: No example buckets
-# description: Buckets should not be named with "example" in the name
+# title: Bad input data
+# description: Just failing rule with input data
 # scope: package
 # schemas:
 # - input: schema["input"]
@@ -1059,14 +1075,20 @@ func Test_Run(t *testing.T) {
 #     selector:
 #     - type: cloud
 package user.whatever
+import data.settings.DS123.foo
 
-deny[res] {
-	bucket := input.aws.s3.buckets[_]
-	contains(bucket.name.value, "example")
-	res := result.new("example bucket detected", bucket.name)
+deny {
+	foo == true
 }
 `,
-			cacheContent: "testdata/s3onlycache.json",
+			inputData: `{
+	"settings": {
+		"DS123": {
+			"foo": true
+		}
+	}
+}`,
+			cacheContent: filepath.Join("testdata", "s3onlycache.json"),
 			allServices:  []string{"s3"},
 			want:         expectedCustomScanResult,
 		},
@@ -1241,6 +1263,11 @@ Summary Report for compliance: my-custom-spec
 				require.NoError(t, os.WriteFile(filepath.Join(regoDir, "policies", "user.rego"), []byte(test.regoPolicy), 0644))
 			}
 
+			if test.inputData != "" {
+				require.NoError(t, os.MkdirAll(filepath.Join(regoDir, "data"), 0755))
+				require.NoError(t, os.WriteFile(filepath.Join(regoDir, "data", "data.json"), []byte(test.inputData), 0644))
+			}
+
 			if test.cacheContent != "" {
 				cacheRoot := t.TempDir()
 				test.options.CacheDir = cacheRoot
@@ -1250,7 +1277,7 @@ Summary Report for compliance: my-custom-spec
 				cacheData, err := os.ReadFile(test.cacheContent)
 				require.NoError(t, err, test.name)
 
-				require.NoError(t, os.WriteFile(cacheFile, []byte(cacheData), 0600))
+				require.NoError(t, os.WriteFile(cacheFile, cacheData, 0600))
 			}
 
 			err := Run(context.Background(), test.options)

diff --git a/pkg/cloud/aws/scanner/scanner.go b/pkg/cloud/aws/scanner/scanner.go
@@ -3,8 +3,11 @@ package scanner
 import (
 	"context"
 	"fmt"
+	"io/fs"
 	"strings"
 
+	"golang.org/x/xerrors"
+
 	"github.com/aquasecurity/defsec/pkg/framework"
 	"github.com/aquasecurity/defsec/pkg/scan"
 	"github.com/aquasecurity/defsec/pkg/scanners/cloud/aws"
@@ -14,6 +17,7 @@ import (
 	"github.com/aquasecurity/trivy/pkg/commands/operation"
 	"github.com/aquasecurity/trivy/pkg/flag"
 	"github.com/aquasecurity/trivy/pkg/log"
+	"github.com/aquasecurity/trivy/pkg/misconf"
 )
 
 type AWSScanner struct {
@@ -75,15 +79,24 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 		scannerOpts = append(scannerOpts,
 			options.ScannerWithEmbeddedPolicies(false))
 	}
-	policyPaths = append(policyPaths, option.RegoOptions.PolicyPaths...)
+
+	var policyFS fs.FS
+	policyFS, policyPaths, err = misconf.CreatePolicyFS(append(policyPaths, option.RegoOptions.PolicyPaths...))
+	if err != nil {
+		return nil, false, xerrors.Errorf("unable to create policyfs: %w", err)
+	}
+
+	scannerOpts = append(scannerOpts, options.ScannerWithPolicyFilesystem(policyFS))
 	scannerOpts = append(scannerOpts, options.ScannerWithPolicyDirs(policyPaths...))
 
-	if len(option.RegoOptions.PolicyNamespaces) > 0 {
-		scannerOpts = append(
-			scannerOpts,
-			options.ScannerWithPolicyNamespaces(option.RegoOptions.PolicyNamespaces...),
-		)
+	dataFS, dataPaths, err := misconf.CreateDataFS(option.RegoOptions.DataPaths)
+	if err != nil {
+		log.Logger.Errorf("Could not load config data: %s", err)
 	}
+	scannerOpts = append(scannerOpts, options.ScannerWithDataDirs(dataPaths...))
+	scannerOpts = append(scannerOpts, options.ScannerWithDataFilesystem(dataFS))
+
+	scannerOpts = addPolicyNamespaces(option.RegoOptions.PolicyNamespaces, scannerOpts)
 
 	if option.Compliance.Spec.ID != "" {
 		scannerOpts = append(scannerOpts, options.ScannerWithSpec(option.Compliance.Spec.ID))
@@ -104,18 +117,9 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 		}
 	}
 
-	var fullState *state.State
-	if previousState, err := awsCache.LoadState(); err == nil {
-		if freshState != nil {
-			fullState, err = previousState.Merge(freshState)
-			if err != nil {
-				return nil, false, err
-			}
-		} else {
-			fullState = previousState
-		}
-	} else {
-		fullState = freshState
+	fullState, err := createState(freshState, awsCache)
+	if err != nil {
+		return nil, false, err
 	}
 
 	if fullState == nil {
@@ -134,10 +138,36 @@ func (s *AWSScanner) Scan(ctx context.Context, option flag.Options) (scan.Result
 	return defsecResults, len(included) > 0, nil
 }
 
+func createState(freshState *state.State, awsCache *cache.Cache) (*state.State, error) {
+	var fullState *state.State
+	if previousState, err := awsCache.LoadState(); err == nil {
+		if freshState != nil {
+			fullState, err = previousState.Merge(freshState)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			fullState = previousState
+		}
+	} else {
+		fullState = freshState
+	}
+	return fullState, nil
+}
+
 type defsecLogger struct {
 }
 
 func (d *defsecLogger) Write(p []byte) (n int, err error) {
 	log.Logger.Debug("[defsec] " + strings.TrimSpace(string(p)))
 	return len(p), nil
 }
+func addPolicyNamespaces(namespaces []string, scannerOpts []options.ScannerOption) []options.ScannerOption {
+	if len(namespaces) > 0 {
+		scannerOpts = append(
+			scannerOpts,
+			options.ScannerWithPolicyNamespaces(namespaces...),
+		)
+	}
+	return scannerOpts
+}
diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go
@@ -579,7 +579,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi
 			Trace:                   opts.Trace,
 			Namespaces:              append(opts.PolicyNamespaces, defaultPolicyNamespaces...),
 			PolicyPaths:             append(opts.PolicyPaths, downloadedPolicyPaths...),
-			DataPaths:               opts.DataPaths,
+			DataPaths:               append(opts.DataPaths),
 			HelmValues:              opts.HelmValues,
 			HelmValueFiles:          opts.HelmValueFiles,
 			HelmFileValues:          opts.HelmFileValues,

diff --git a/pkg/mapfs/fs.go b/pkg/mapfs/fs.go
@@ -1,6 +1,7 @@
 package mapfs
 
 import (
+	"fmt"
 	"io"
 	"io/fs"
 	"os"
@@ -233,11 +234,11 @@ func (m *FS) RemoveAll(path string) error {
 }
 
 func cleanPath(path string) string {
-	// Return if the file path is a volume name only.
-	// Otherwise, `filepath.Clean` changes "C:" to "C:." and
-	// it will no longer match the pathname held by mapfs.
-	if path == filepath.VolumeName(path) {
-		return path
+	// Convert the volume name like 'C:' into dir like 'C\'
+	if vol := filepath.VolumeName(path); len(vol) > 0 {
+		newVol := strings.TrimSuffix(vol, ":")
+		newVol = fmt.Sprintf("%s%c", newVol, filepath.Separator)
+		path = strings.Replace(path, vol, newVol, 1)
 	}
 	path = filepath.Clean(path)
 	path = filepath.ToSlash(path)