From 79e3706e17cdf9470fbe5beaab3a7443df2cd8c9 Mon Sep 17 00:00:00 2001 From: Sandor Trombitas Date: Fri, 25 Oct 2024 15:20:16 +0300 Subject: [PATCH] chore(localfindings): add new filter workflow for local findings --- cliv2/cmd/cliv2/main.go | 6 + cliv2/cmd/cliv2/main_test.go | 98 ++++++++- cliv2/cmd/cliv2/testdata/sarif.json | 314 ++++++++++++++++++++++++++++ cliv2/go.mod | 5 +- cliv2/go.sum | 4 +- 5 files changed, 422 insertions(+), 5 deletions(-) diff --git a/cliv2/cmd/cliv2/main.go b/cliv2/cmd/cliv2/main.go index 5139cce70b..b54a68a1e1 100644 --- a/cliv2/cmd/cliv2/main.go +++ b/cliv2/cmd/cliv2/main.go @@ -171,6 +171,12 @@ func runWorkflowAndProcessData(engine workflow.Engine, logger *zerolog.Logger, n return err } + output, err = engine.InvokeWithInput(localworkflows.WORKFLOWID_FILTER_FINDINGS, output) + if err != nil { + logger.Err(err).Msg(err.Error()) + return err + } + output, err = engine.InvokeWithInput(localworkflows.WORKFLOWID_OUTPUT_WORKFLOW, output) if err == nil { err = getErrorFromWorkFlowData(engine, output) diff --git a/cliv2/cmd/cliv2/main_test.go b/cliv2/cmd/cliv2/main_test.go index 967747d2b3..ed650b413c 100644 --- a/cliv2/cmd/cliv2/main_test.go +++ b/cliv2/cmd/cliv2/main_test.go @@ -19,6 +19,7 @@ import ( localworkflows "github.com/snyk/go-application-framework/pkg/local_workflows" "github.com/snyk/go-application-framework/pkg/local_workflows/content_type" "github.com/snyk/go-application-framework/pkg/local_workflows/json_schemas" + "github.com/snyk/go-application-framework/pkg/local_workflows/local_models" "github.com/snyk/go-application-framework/pkg/mocks" "github.com/snyk/go-application-framework/pkg/workflow" "github.com/spf13/cobra" @@ -203,6 +204,9 @@ func Test_runMainWorkflow_unknownargs(t *testing.T) { assert.NoError(t, err) _ = globalEngine.Init() + // Register our data filter workflow + err = localworkflows.InitFilterFindingsWorkflow(globalEngine) + assert.NoError(t, err) config := configuration.NewWithOpts(configuration.WithAutomaticEnv()) cmd := &cobra.Command{ @@ -359,6 +363,9 @@ func Test_runWorkflowAndProcessData(t *testing.T) { _, err := globalEngine.Register(workflowId1, workflowConfig, outputFn) assert.NoError(t, err) + // Register our data filter workflow + err = localworkflows.InitFilterFindingsWorkflow(globalEngine) + assert.NoError(t, err) fn := func(invocation workflow.InvocationContext, input []workflow.Data) ([]workflow.Data, error) { typeId := workflow.NewTypeIdentifier(invocation.GetWorkflowIdentifier(), "workflowData") @@ -410,7 +417,7 @@ func Test_runWorkflowAndProcessData(t *testing.T) { assert.Equal(t, constants.SNYK_EXIT_CODE_VULNERABILITIES_FOUND, actualCode) } -func Test_runWorkflowAndProcessData_WithTransformation(t *testing.T) { +func Test_runWorkflowAndProcessData_with_Transformation(t *testing.T) { defer cleanup() globalConfiguration = configuration.New() globalConfiguration.Set(configuration.DEBUG, true) @@ -447,6 +454,10 @@ func Test_runWorkflowAndProcessData_WithTransformation(t *testing.T) { err = localworkflows.InitDataTransformationWorkflow(globalEngine) assert.NoError(t, err) + // Register our data filter workflow + err = localworkflows.InitFilterFindingsWorkflow(globalEngine) + assert.NoError(t, err) + // Invoke a custom command that returns input fn := func(invocation workflow.InvocationContext, input []workflow.Data) ([]workflow.Data, error) { typeId := workflow.NewTypeIdentifier(invocation.GetWorkflowIdentifier(), "workflowData") @@ -487,6 +498,91 @@ func Test_runWorkflowAndProcessData_WithTransformation(t *testing.T) { err = runWorkflowAndProcessData(globalEngine, &logger, testCmnd) } +func Test_runWorkflowAndProcessData_with_Filtering(t *testing.T) { + defer cleanup() + globalConfiguration = configuration.New() + globalConfiguration.Set(configuration.DEBUG, true) + globalConfiguration.Set(configuration.IN_MEMORY_THRESHOLD_BYTES, -1) + globalConfiguration.Set(configuration.FLAG_SEVERITY_THRESHOLD, "high") + globalConfiguration.Set(configuration.FF_TRANSFORMATION_WORKFLOW, true) + + globalEngine = workflow.NewWorkFlowEngine(globalConfiguration) + + testCmnd := "subcmd1" + workflowId1 := workflow.NewWorkflowIdentifier("output") + + outputFn := func(invocation workflow.InvocationContext, input []workflow.Data) ([]workflow.Data, error) { + var findings local_models.LocalFinding + for i := range input { + mimeType := input[i].GetContentType() + + if strings.HasPrefix(mimeType, content_type.LOCAL_FINDING_MODEL) { + findingsBytes := input[i].GetPayload().([]byte) + err := json.Unmarshal(findingsBytes, &findings) + assert.NoError(t, err) + } + } + + // expect all findings below high to be filtered out + assert.Equal(t, 1, len(findings.Findings)) + + return input, nil + } + + workflowConfig := workflow.ConfigurationOptionsFromFlagset(pflag.NewFlagSet("pla", pflag.ContinueOnError)) + + _, err := globalEngine.Register(workflowId1, workflowConfig, outputFn) + assert.NoError(t, err) + + // Register our data transformation workflow + err = localworkflows.InitDataTransformationWorkflow(globalEngine) + assert.NoError(t, err) + + // Register our data filter workflow + err = localworkflows.InitFilterFindingsWorkflow(globalEngine) + assert.NoError(t, err) + + // Invoke a custom command that returns input + fn := func(invocation workflow.InvocationContext, input []workflow.Data) ([]workflow.Data, error) { + typeId := workflow.NewTypeIdentifier(invocation.GetWorkflowIdentifier(), "workflowData") + testSummary := json_schemas.TestSummary{ + Results: []json_schemas.TestSummaryResult{ + { + Severity: "critical", + Total: 10, + Open: 10, + Ignored: 0, + }, + }, + Type: "sast", + SeverityOrderAsc: []string{"low", "medium", "high", "critical"}, + } + + var d []byte + d, err = json.Marshal(testSummary) + assert.NoError(t, err) + + testSummaryData := workflow.NewData(typeId, content_type.TEST_SUMMARY, d) + sarifData := workflow.NewData(typeId, content_type.SARIF_JSON, + loadJsonFile(t, "sarif.json")) + + return []workflow.Data{ + testSummaryData, + sarifData, + }, nil + } + wrkflowId := workflow.NewWorkflowIdentifier(testCmnd) + entry, err := globalEngine.Register(wrkflowId, workflowConfig, fn) + assert.NoError(t, err) + assert.NotNil(t, entry) + + err = globalEngine.Init() + assert.NoError(t, err) + + logger := zerolog.New(os.Stderr) + err = runWorkflowAndProcessData(globalEngine, &logger, testCmnd) +} + func Test_setTimeout(t *testing.T) { exitedCh := make(chan struct{}) fakeExit := func() { diff --git a/cliv2/cmd/cliv2/testdata/sarif.json b/cliv2/cmd/cliv2/testdata/sarif.json index b7ea75c6c6..6424b26a7d 100644 --- a/cliv2/cmd/cliv2/testdata/sarif.json +++ b/cliv2/cmd/cliv2/testdata/sarif.json @@ -124,6 +124,122 @@ "repoDatasetSize": 30, "cwe": ["CWE-943"] } + }, + { + "id": "javascript/NoSqliLowSeverity", + "name": "NoSqli", + "shortDescription": { + "text": "NoSQL Injection" + }, + "defaultConfiguration": { + "level": "warning" + }, + "help": { + "markdown": "## Details\n\nIn an NoSQL injection attack, the user can submit an NoSQL query directly to the database, gaining access without providing appropriate credentials. Attackers can then view, export, modify, and delete confidential information; change passwords and other authentication information; and possibly gain access to other systems within the network. This is one of the most commonly exploited categories of vulnerability, but can largely be avoided through good coding practices.", + "text": "" + }, + "properties": { + "tags": [ + "javascript", + "NoSqli", + "Security", + "SourceServer", + "SourceHttpBody", + "Taint" + ], + "categories": ["Security"], + "exampleCommitFixes": [ + { + "commitURL": "https://gilhub.com/afuh/pinstagram/commit/776a6b63f84b3bc9d38963933ff511b319b73ac5?diff=split#diff-fb901db253d2190ed5dec3508eb32e99524e0b4dcacdaea322a50f2619ae2d99L-1", + "lines": [ + { + "line": "const user = await User.findOne({ slug: req.params.user }).populate('likes')\n", + "lineNumber": 47, + "lineChange": "removed" + }, + { + "line": "const user = await User.findOne({ _id: req.user._id }).populate('likes')\n", + "lineNumber": 47, + "lineChange": "added" + } + ] + }, + { + "commitURL": "https://github.com/mercmobily/hotplate/commit/c9dfbe8bf6bfd03838946d0898978543589a5ea2?diff=split#diff-bdb0afd700d4dfe1801bcfe39008d413182be643063835d326641fcce15b969aL-1", + "lines": [ + { + "line": " Workspace.findOne({ _id: req.params.workspaceId }, function( err, doc ){\n", + "lineNumber": 270, + "lineChange": "removed" + }, + { + "line": " workspaces.findOne({ _id: ObjectId(req.params.workspaceId) }, function( err, doc ){\n", + "lineNumber": 264, + "lineChange": "added" + }, + { + "line": " resUtils.checkFindOneResponse( err, doc, next, function(){\n", + "lineNumber": 271, + "lineChange": "none" + }, + { + "line": "\n", + "lineNumber": 272, + "lineChange": "none" + }, + { + "line": " perms.checkPermissions( req, next, 'workspaceConfig/get', req.body, doc, function(){\n", + "lineNumber": 273, + "lineChange": "none" + }, + { + "line": " sendResponse( res, doc.settings );\n", + "lineNumber": 274, + "lineChange": "none" + }, + { + "line": " });\n", + "lineNumber": 275, + "lineChange": "none" + }, + { + "line": " });\n", + "lineNumber": 276, + "lineChange": "none" + }, + { + "line": "});\n", + "lineNumber": 277, + "lineChange": "none" + } + ] + }, + { + "commitURL": "https://github.com/JasonEtco/flintcms/commit/4ae34238ce39fde00dfa15082397541758c07af1?diff=split#diff-9abe922e7535c6f75fba7150a7a803a93be7ae235564b86f799db9f37e4c1674L-1", + "lines": [ + { + "line": "const token = req.query.t\n", + "lineNumber": 103, + "lineChange": "removed" + }, + { + "line": "const token = req.query.t.toString()\n", + "lineNumber": 103, + "lineChange": "added" + }, + { + "line": "const user = await User.findOne({ token })\n", + "lineNumber": 104, + "lineChange": "none" + } + ] + } + ], + "exampleCommitDescriptions": [], + "precision": "very-high", + "repoDatasetSize": 30, + "cwe": ["CWE-943"] + } } ] } @@ -326,6 +442,204 @@ ], "isAutofixable": false } + }, + { + "ruleId": "javascript/NoSqliLowSeverity", + "ruleIndex": 0, + "level": "warning", + "message": { + "text": "Unsanitized input from the HTTP request body flows into findOne, where it is used in an NoSQL query. This may result in an NoSQL Injection vulnerability.", + "markdown": "Unsanitized input from {0} {1} into {2}, where it is used in an NoSQL query. This may result in an NoSQL Injection vulnerability.", + "arguments": [ + "[the HTTP request body](0)", + "[flows](1),(2),(3),(4),(5),(6)", + "[findOne](7)" + ] + }, + "locations": [ + { + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 18, + "endLine": 18, + "startColumn": 26, + "endColumn": 33 + } + } + } + ], + "fingerprints": { + "0": "d3e6d95802bfa65cdee1cc840eda6a7b8422f24962e436dd01730e6116e317ec", + "1": "93652555.4773f344.07efaa4d.9163ada2.ef9f7d82.34a4d81a.df7e59ba.d66579bf.f759b1f9.706318d0.07efaa4d.08906714.79a7d027.847dd466.0334236c.041df0b3" + }, + "codeFlows": [ + { + "threadFlows": [ + { + "locations": [ + { + "location": { + "id": 0, + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 16, + "endLine": 16, + "startColumn": 20, + "endColumn": 24 + } + } + } + }, + { + "location": { + "id": 1, + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 16, + "endLine": 16, + "startColumn": 20, + "endColumn": 24 + } + } + } + }, + { + "location": { + "id": 2, + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 16, + "endLine": 16, + "startColumn": 16, + "endColumn": 24 + } + } + } + }, + { + "location": { + "id": 3, + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 16, + "endLine": 16, + "startColumn": 11, + "endColumn": 13 + } + } + } + }, + { + "location": { + "id": 4, + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 18, + "endLine": 18, + "startColumn": 41, + "endColumn": 43 + } + } + } + }, + { + "location": { + "id": 5, + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 18, + "endLine": 18, + "startColumn": 36, + "endColumn": 39 + } + } + } + }, + { + "location": { + "id": 6, + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 18, + "endLine": 18, + "startColumn": 34, + "endColumn": 45 + } + } + } + }, + { + "location": { + "id": 7, + "physicalLocation": { + "artifactLocation": { + "uri": "routes/likeProductReviews.ts", + "uriBaseId": "%SRCROOT%" + }, + "region": { + "startLine": 18, + "endLine": 18, + "startColumn": 26, + "endColumn": 33 + } + } + } + } + ] + } + ] + } + ], + "properties": { + "priorityScore": 802, + "priorityScoreFactors": [ + { + "label": true, + "type": "multipleOccurrence" + }, + { + "label": true, + "type": "hotFileSource" + }, + { + "label": true, + "type": "fixExamples" + } + ], + "isAutofixable": false + } } ], "properties": { diff --git a/cliv2/go.mod b/cliv2/go.mod index 03307de67b..f2045035d6 100644 --- a/cliv2/go.mod +++ b/cliv2/go.mod @@ -17,7 +17,7 @@ require ( github.com/snyk/cli-extension-sbom v0.0.0-20241016065306-0df2be5b3b8f github.com/snyk/container-cli v0.0.0-20240821111304-7ca1c415a5d7 github.com/snyk/error-catalog-golang-public v0.0.0-20240809094525-c48d19c27edb - github.com/snyk/go-application-framework v0.0.0-20241029131533-b495e35f30ac + github.com/snyk/go-application-framework v0.0.0-20241105130756-9d4de29b4af4 github.com/snyk/go-httpauth v0.0.0-20240307114523-1f5ea3f55c65 github.com/snyk/snyk-iac-capture v0.6.5 github.com/snyk/snyk-ls v0.0.0-20241105141623-a829a9cd7dda @@ -221,6 +221,7 @@ require ( // version 2491eb6c1c75 contains a valid license replace github.com/mattn/go-localereader v0.0.1 => github.com/mattn/go-localereader v0.0.2-0.20220822084749-2491eb6c1c75 -//replace github.com/snyk/go-application-framework => ../../go-application-framework +// replace github.com/snyk/go-application-framework => ../../go-application-framework + //replace github.com/snyk/snyk-ls => ../../snyk-ls //replace github.com/snyk/code-client-go => ../../code-client-go diff --git a/cliv2/go.sum b/cliv2/go.sum index 17014f31da..b255d0ba31 100644 --- a/cliv2/go.sum +++ b/cliv2/go.sum @@ -762,8 +762,8 @@ github.com/snyk/container-cli v0.0.0-20240821111304-7ca1c415a5d7 h1:Zn5BcV76oFAb github.com/snyk/container-cli v0.0.0-20240821111304-7ca1c415a5d7/go.mod h1:38w+dcAQp9eG3P5t2eNS9eG0reut10AeJjLv5lJ5lpM= github.com/snyk/error-catalog-golang-public v0.0.0-20240809094525-c48d19c27edb h1:w9tJhpTFxWqAhLeraGsMExDjGK9x5Dwj1NRFwb+t+QE= github.com/snyk/error-catalog-golang-public v0.0.0-20240809094525-c48d19c27edb/go.mod h1:Ytttq7Pw4vOCu9NtRQaOeDU2dhBYUyNBe6kX4+nIIQ4= -github.com/snyk/go-application-framework v0.0.0-20241029131533-b495e35f30ac h1:kqy5svAquK3HRnyIt7TkxbgUw7yP1vXtiWQjEuQ5DyU= -github.com/snyk/go-application-framework v0.0.0-20241029131533-b495e35f30ac/go.mod h1:5aMJH42nmeZgPRXvN5fdAZVoutATPnY9yTb68Iz7zYk= +github.com/snyk/go-application-framework v0.0.0-20241105130756-9d4de29b4af4 h1:lwfk5tos7680HuWbdoENzqYZIrs5Y5x3CC1Ow80riKQ= +github.com/snyk/go-application-framework v0.0.0-20241105130756-9d4de29b4af4/go.mod h1:5aMJH42nmeZgPRXvN5fdAZVoutATPnY9yTb68Iz7zYk= github.com/snyk/go-httpauth v0.0.0-20240307114523-1f5ea3f55c65 h1:CEQuYv0Go6MEyRCD3YjLYM2u3Oxkx8GpCpFBd4rUTUk= github.com/snyk/go-httpauth v0.0.0-20240307114523-1f5ea3f55c65/go.mod h1:88KbbvGYlmLgee4OcQ19yr0bNpXpOr2kciOthaSzCAg= github.com/snyk/policy-engine v0.31.3 h1:FepCg6QN/X8uvxYjF+WwB2aiBPJB+NENDgKQeI/FwLg=