fix: line content and private key rule (#267)

Co-authored-by: arturfalcao <[email protected]>

.2ms.yml

@@ -1,21 +1,26 @@
 log-level: info
 ignore-result:
-  - 4f1d13e1bbebef31175ffe9a8d752609b9edc174
-  - 65706aeda7939dca8035f4b0a3446babffc7fcef
-  - d696fe501f3860f76cf768c7ebbccc416db6e4d2
-  - d766d69fed184582fc0cba1515f9beef7901e7a1
-  - deddd58b4aa4999419d6b9046dffa9fffdfd4860
-  - e7bf294c124122a6cf919edbffa40bf6572927b6
-  - 0b217706e100e9a05bbaa8427070d181bb2e2465
-  - 1e68cf841873862527a00ee5ef8d7957e319b6bc
-  - 4d358e6dd9e2f21c647dad571f13dde3fd77d107
-  - 7c73d41f23ba8e59a1c8d744594dbb54f87197b4
-  - 8f3e5dec63edc317daa4bff154939555cd35b2c3
-  - 9c6853ebe9b5e20774224ba6e5ea739191330e53
-  - 59f8916ff79257c8f86207d6e89767cc8e156814
-  - a3a83b7224e7e98e3cca6bd2cd138dbca831e06d
-  - ba1f0517b77a5b451d1d55078218cd23d96b686e
-  - c5748512948b492f5c07849ae2e69e7e831d36d3
-  - 5e73b4b73bf4a59b11f37066829af01478879067 # False positive, see https://github.com/gitleaks/gitleaks/pull/1358
-  - 255853e2044119bf502261713e2f892265d4b5c1 # False positive, see https://github.com/gitleaks/gitleaks/pull/1358
-  - a324bc00bebfbd268b1b9e4cddcd095da1193cd2
+  - 4f1d13e1bbebef31175ffe9a8d752609b9edc174 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/8156dfa1eb31c74665a15532af06ae8f33173c92#diff-99df3a2722b4e0b13959d54e29d73ef7b84aae7fbba329b1bfd4f88178bbf010R21
+  - 65706aeda7939dca8035f4b0a3446babffc7fcef # value used for testing, found at https://github.com/Checkmarx/2ms/commit/8156dfa1eb31c74665a15532af06ae8f33173c92#diff-99df3a2722b4e0b13959d54e29d73ef7b84aae7fbba329b1bfd4f88178bbf010R20
+  - d696fe501f3860f76cf768c7ebbccc416db6e4d2 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/8156dfa1eb31c74665a15532af06ae8f33173c92#diff-d135844ba25069f53a866a454c13f82f25b4f22fe3a67310125e4529a72fd50bR373
+  - d766d69fed184582fc0cba1515f9beef7901e7a1 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/8156dfa1eb31c74665a15532af06ae8f33173c92#diff-99df3a2722b4e0b13959d54e29d73ef7b84aae7fbba329b1bfd4f88178bbf010R22
+  - deddd58b4aa4999419d6b9046dffa9fffdfd4860 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/8156dfa1eb31c74665a15532af06ae8f33173c92#diff-d135844ba25069f53a866a454c13f82f25b4f22fe3a67310125e4529a72fd50bR378
+  - e7bf294c124122a6cf919edbffa40bf6572927b6 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/8156dfa1eb31c74665a15532af06ae8f33173c92#diff-99df3a2722b4e0b13959d54e29d73ef7b84aae7fbba329b1bfd4f88178bbf010R22
+  - 0b217706e100e9a05bbaa8427070d181bb2e2465 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/8156dfa1eb31c74665a15532af06ae8f33173c92#diff-d135844ba25069f53a866a454c13f82f25b4f22fe3a67310125e4529a72fd50bR388
+  - 1e68cf841873862527a00ee5ef8d7957e319b6bc # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10
+  - 4d358e6dd9e2f21c647dad571f13dde3fd77d107 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/2aa7733a457a145725aa11f99f05f60ddc6b09a1#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040
+  - 7c73d41f23ba8e59a1c8d744594dbb54f87197b4 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/69e93832d8c0e27554c0185c5a11cbb894c9dc9d#diff-d9ca2cf3e88d3b3d6d58f12bc171e8e64feca51a3c851df62e51fb0bf269fa33R38
+  - 9c6853ebe9b5e20774224ba6e5ea739191330e53 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/e6e8b8ea07078ae7b66db70c91693d293f813ad6#diff-d135844ba25069f53a866a454c13f82f25b4f22fe3a67310125e4529a72fd50bR380
+  - 59f8916ff79257c8f86207d6e89767cc8e156814 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/69e93832d8c0e27554c0185c5a11cbb894c9dc9d#diff-d9ca2cf3e88d3b3d6d58f12bc171e8e64feca51a3c851df62e51fb0bf269fa33R40
+  - a3a83b7224e7e98e3cca6bd2cd138dbca831e06d # value used for testing, found at https://github.com/Checkmarx/2ms/commit/69e93832d8c0e27554c0185c5a11cbb894c9dc9d#diff-d9ca2cf3e88d3b3d6d58f12bc171e8e64feca51a3c851df62e51fb0bf269fa33R45
+  - ba1f0517b77a5b451d1d55078218cd23d96b686e # value used for testing, found at https://github.com/Checkmarx/2ms/commit/69e93832d8c0e27554c0185c5a11cbb894c9dc9d#diff-d9ca2cf3e88d3b3d6d58f12bc171e8e64feca51a3c851df62e51fb0bf269fa33R39
+  - c5748512948b492f5c07849ae2e69e7e831d36d3 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/e6e8b8ea07078ae7b66db70c91693d293f813ad6#diff-d135844ba25069f53a866a454c13f82f25b4f22fe3a67310125e4529a72fd50bR375
+  - 5e73b4b73bf4a59b11f37066829af01478879067 # False positive, see https://github.com/gitleaks/gitleaks/pull/1358, found at https://github.com/Checkmarx/2ms/commit/c6868197ae0910b1ae34a19822b32ae9d8eb7fdd#diff-d712d2256df359061d691b711ca7ed30ba408199b1e3801cef289779778d8badR36
+  - 255853e2044119bf502261713e2f892265d4b5c1 # False positive, see https://github.com/gitleaks/gitleaks/pull/1358, found at https://github.com/Checkmarx/2ms/commit/45a5c9d35ff910dfec5e5a76cdedb8977da5dd34#diff-d761ee3cdf26960bab9cc0e88d92015b3c38c0bd480c60d7d58755dea5421e36R24
+  - a324bc00bebfbd268b1b9e4cddcd095da1193cd2 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/757c4af1608ec63a3f758f4b2b2b3be915cfbedc#diff-cbf9ce27965b15f1e13be995f164fd6b33a19381364c4be3e8c047d528994495R66
+  - cf413577a1df23446f1916be0b6c31679f2042a8 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/4bf7a43e33f3f27bedc97a35e9d74dc95026cf28#diff-878a64a63b9c3845d915db9e5b182aba76adfffde863994b3840f76c4046f8f2R18
+  - 2cbfe7687bd4b859c51869dc2c1af25e70a8be4b # value used for testing, found at https://github.com/Checkmarx/2ms/commit/4bf7a43e33f3f27bedc97a35e9d74dc95026cf28/engine/rules/privateKey.go#diff-878a64a63b9c3845d915db9e5b182aba76adfffde863994b3840f76c4046f8f2R19
+  - 9c1749703c1017ebf05455df0e8f5b5752ec08a8 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/4bf7a43e33f3f27bedc97a35e9d74dc95026cf28/engine/rules/privateKey.go#diff-878a64a63b9c3845d915db9e5b182aba76adfffde863994b3840f76c4046f8f2R22
+  - 92c0192a71f1c299a8b8f8ebf63009582146a573 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/4bf7a43e33f3f27bedc97a35e9d74dc95026cf28/engine/rules/privateKey.go#diff-878a64a63b9c3845d915db9e5b182aba76adfffde863994b3840f76c4046f8f2R26
+  - e53a3a4e8c0665454eb9a4c36eaf040e9317e450 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/2aa7733a457a145725aa11f99f05f60ddc6b09a1/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R9
+  - ffc22deda44ebb0d4633bed184c5e26e99657084 # value used for testing, found at https://github.com/Checkmarx/2ms/commit/07aab5bb214c03fd9e75e46cebe2b407c88d4f73/reporting/report_test.go#diff-31d71ec2c2ba169dce79b1c2de097e30b43f1695ce364054ee7d6b33896c7040R10

.github/workflows/security.yml

@@ -73,4 +73,4 @@ jobs:
           fetch-depth: 0
 
       - name: Run 2ms Scan
-        run: docker run -v $(pwd):/repo checkmarx/2ms:latest git /repo --config /repo/.2ms.yml
+        run: docker run -v $(pwd):/repo checkmarx/2ms:latest git /repo --config /repo/.2ms.yml

Dockerfile

@@ -20,7 +20,7 @@ COPY . .
 RUN GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -a -o /app/2ms .
 
 # Runtime image
-FROM cgr.dev/chainguard/git@sha256:08704f0b6ba76925b76a01b798215a2cecfbbd0655c423085509cb5163e8ff20
+FROM cgr.dev/chainguard/git@sha256:0389019d7ee820683793e0ad9d1863120d586962803d84e8d57aa003922060d2
 
 WORKDIR /app
 

cmd/workers.go

@@ -15,7 +15,7 @@ func processItems(engine *engine.Engine, pluginName string) {
 	for item := range channels.Items {
 		report.TotalItemsScanned++
 		wgItems.Add(1)
-		go engine.Detect(item, secretsChan, wgItems, pluginName)
+		go engine.Detect(item, secretsChan, wgItems, pluginName, channels.Errors)
 	}
 	wgItems.Wait()
 	close(secretsChan)

engine/engine.go

@@ -78,7 +78,7 @@ func Init(engineConfig EngineConfig) (*Engine, error) {
 	}, nil
 }
 
-func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string) {
+func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string, errors chan error) {
 	defer wg.Done()
 
 	fragment := detect.Fragment{
@@ -94,6 +94,11 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S
 		} else {
 			startLine = value.StartLine
 			endLine = value.EndLine
+
+		}
+		lineContent, err := linecontent.GetLineContent(value.Line, value.Secret)
+		if err != nil {
+			errors <- err
 		}
 		secret := &secrets.Secret{
 			ID:              itemId,
@@ -104,7 +109,7 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S
 			EndLine:         endLine,
 			EndColumn:       value.EndColumn,
 			Value:           value.Secret,
-			LineContent:     linecontent.GetLineContent(value.Line, value.StartColumn, value.EndColumn),
+			LineContent:     lineContent,
 			RuleDescription: value.Description,
 		}
 		if !isSecretIgnored(secret, &e.ignoredIds, &e.allowedValues) {

engine/engine_test.go

@@ -78,9 +78,10 @@ func TestDetector(t *testing.T) {
 		}
 
 		secretsChan := make(chan *secrets.Secret, 1)
+		errorsChan := make(chan error, 1)
 		wg := &sync.WaitGroup{}
 		wg.Add(1)
-		detector.Detect(i, secretsChan, wg, fsPlugin.GetName())
+		detector.Detect(i, secretsChan, wg, fsPlugin.GetName(), errorsChan)
 		close(secretsChan)
 
 		s := <-secretsChan
@@ -153,10 +154,12 @@ func TestSecrets(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			fmt.Printf("Start test %s", name)
 			secretsChan := make(chan *secrets.Secret, 1)
+			errorsChan := make(chan error, 1)
 			wg := &sync.WaitGroup{}
 			wg.Add(1)
-			detector.Detect(item{content: &secret.Content}, secretsChan, wg, fsPlugin.GetName())
+			detector.Detect(item{content: &secret.Content}, secretsChan, wg, fsPlugin.GetName(), errorsChan)
 			close(secretsChan)
+			close(errorsChan)
 
 			s := <-secretsChan
 

engine/linecontent/linecontent.go

@@ -1,22 +1,47 @@
 package linecontent
 
+import (
+	"fmt"
+	"strings"
+)
+
 const (
+	lineMaxParseSize      = 10000
 	contextLeftSizeLimit  = 250
 	contextRightSizeLimit = 250
 )
 
-func GetLineContent(lineContent string, startColumn, endColumn int) string {
-	lineContentSize := len(lineContent)
+func GetLineContent(line, secret string) (string, error) {
+	lineSize := len(line)
+	if lineSize == 0 {
+		return "", fmt.Errorf("failed to get line content: line empty")
+	}
+
+	if len(secret) == 0 {
+		return "", fmt.Errorf("failed to get line content: secret empty")
+	}
 
-	startIndex := startColumn - contextLeftSizeLimit
-	if startIndex < 0 {
-		startIndex = 0
+	// Truncate lineContent to max size
+	if lineSize > lineMaxParseSize {
+		line = line[:lineMaxParseSize]
+		lineSize = lineMaxParseSize
 	}
 
-	endIndex := endColumn + contextRightSizeLimit
-	if endIndex > lineContentSize {
-		endIndex = lineContentSize
+	// Find the secret's position in the line
+	secretStartIndex := strings.Index(line, secret)
+	if secretStartIndex == -1 {
+		// Secret not found, return truncated content based on context limits
+		maxSize := contextLeftSizeLimit + contextRightSizeLimit
+		if lineSize < maxSize {
+			return line, nil
+		}
+		return line[:maxSize], nil
 	}
 
-	return lineContent[startIndex:endIndex]
+	// Calculate bounds for the result
+	secretEndIndex := secretStartIndex + len(secret)
+	start := max(secretStartIndex-contextLeftSizeLimit, 0)
+	end := min(secretEndIndex+contextRightSizeLimit, lineSize)
+
+	return line[start:end], nil
 }

engine/linecontent/linecontent_test.go

@@ -0,0 +1,116 @@
+package linecontent
+
+import (
+	"strings"
+	"testing"
+)
+
+const (
+	dummySecret = "DummySecret"
+)
+
+func TestGetLineContent(t *testing.T) {
+	tests := []struct {
+		name         string
+		line         string
+		secret       string
+		expected     string
+		error        bool
+		errorMessage string
+	}{
+		{
+			name:         "Empty line",
+			line:         "",
+			secret:       dummySecret,
+			expected:     "",
+			error:        true,
+			errorMessage: "failed to get line content: line empty",
+		},
+		{
+			name:         "Empty secret",
+			line:         "line",
+			secret:       "",
+			expected:     "",
+			error:        true,
+			errorMessage: "failed to get line content: secret empty",
+		},
+		{
+			name:     "Secret not found with line size smaller than the parse limit",
+			line:     "Dummy content line",
+			secret:   dummySecret,
+			expected: "Dummy content line",
+			error:    false,
+		},
+		{
+			name:     "Secret not found with secret present and line size larger than the parse limit",
+			line:     "This is the start of a big line content" + strings.Repeat("A", lineMaxParseSize) + dummySecret,
+			secret:   dummySecret,
+			expected: "This is the start of a big line content" + strings.Repeat("A", contextLeftSizeLimit+contextRightSizeLimit-len("This is the start of a big line content")),
+			error:    false,
+		},
+		{
+			name:     "Secret larger than the line",
+			line:     strings.Repeat("B", contextLeftSizeLimit) + strings.Repeat("A", contextRightSizeLimit),
+			secret:   "large secret" + strings.Repeat("B", contextRightSizeLimit+contextLeftSizeLimit+100),
+			expected: strings.Repeat("B", contextLeftSizeLimit) + strings.Repeat("A", contextRightSizeLimit),
+			error:    false,
+		},
+		{
+			name:     "Secret at the beginning with line size smaller than the parse limit",
+			line:     "start:" + dummySecret + strings.Repeat("A", lineMaxParseSize/2),
+			secret:   dummySecret,
+			expected: "start:" + dummySecret + strings.Repeat("A", contextRightSizeLimit),
+			error:    false,
+		},
+		{
+			name:     "Secret found in middle with line size smaller than the parse limit",
+			line:     "start" + strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit) + "end",
+			secret:   dummySecret,
+			expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit),
+			error:    false,
+		},
+		{
+			name:     "Secret at the end with line size smaller than the parse limit",
+			line:     strings.Repeat("A", lineMaxParseSize/2) + dummySecret + ":end",
+			secret:   dummySecret,
+			expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + ":end",
+			error:    false,
+		},
+		{
+			name:     "Secret at the beginning with line size larger than the parse limit",
+			line:     "start:" + dummySecret + strings.Repeat("A", lineMaxParseSize),
+			secret:   dummySecret,
+			expected: "start:" + dummySecret + strings.Repeat("A", contextRightSizeLimit),
+			error:    false,
+		},
+		{
+			name:     "Secret found in middle with line size larger than the parse limit",
+			line:     "start" + strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", lineMaxParseSize) + "end",
+			secret:   dummySecret,
+			expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit),
+			error:    false,
+		},
+		{
+			name:     "Secret at the end with line size larger than the parse limit",
+			line:     strings.Repeat("A", lineMaxParseSize-100) + dummySecret + strings.Repeat("A", lineMaxParseSize),
+			secret:   dummySecret,
+			expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", 100-len(dummySecret)),
+			error:    false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetLineContent(tt.line, tt.secret)
+			if (err != nil) != tt.error {
+				t.Fatalf("GetLineContent() error = %v, wantErr %v", err, tt.error)
+			}
+			if err != nil && err.Error() != tt.errorMessage {
+				t.Errorf("GetLineContent() error message = %v, want %v", err.Error(), tt.errorMessage)
+			}
+			if got != tt.expected {
+				t.Errorf("GetLineContent() = %v, want %v", got, tt.expected)
+			}
+		})
+	}
+}

engine/rules/privateKey.go

@@ -0,0 +1,30 @@
+package rules
+
+import (
+	"github.com/zricethezav/gitleaks/v8/config"
+	"regexp"
+)
+
+func PrivateKey() *config.Rule {
+	// define rule
+	r := config.Rule{
+		Description: "Identified a Private Key, which may compromise cryptographic security and sensitive data encryption.",
+		RuleID:      "private-key",
+		Regex:       regexp.MustCompile(`(?i)-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----[\s\S-]*?KEY(?: BLOCK)?-----`),
+		Keywords:    []string{"-----BEGIN"},
+	}
+
+	// validate
+	tps := []string{`-----BEGIN PRIVATE KEY-----
+anything
+-----END PRIVATE KEY-----`,
+		`-----BEGIN RSA PRIVATE KEY-----
+abcdefghijklmnopqrstuvwxyz
+-----END RSA PRIVATE KEY-----
+`,
+		`-----BEGIN PRIVATE KEY BLOCK-----
+anything
+-----END PRIVATE KEY BLOCK-----`,
+	}
+	return validate(r, tps, nil)
+}

engine/rules/rules.go

@@ -187,7 +187,7 @@ func getDefaultRules() *[]Rule {
 		{Rule: *rules.PlanetScaleOAuthToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryDatabaseAsAService, RuleType: 4}},
 		{Rule: *rules.PostManAPI(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}},
 		{Rule: *rules.Prefect(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}},
-		{Rule: *rules.PrivateKey(), Tags: []string{TagPrivateKey}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
+		{Rule: *PrivateKey(), Tags: []string{TagPrivateKey}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
 		{Rule: *rules.PulumiAPIToken(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryCloudPlatform, RuleType: 4}},
 		{Rule: *rules.PyPiUploadToken(), Tags: []string{TagUploadToken}, ScoreParameters: ScoreParameters{Category: CategoryPackageManagement, RuleType: 4}},
 		{Rule: *rules.RapidAPIAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryAPIAccess, RuleType: 4}},

engine/score/score_test.go

@@ -135,7 +135,7 @@ func TestScore(t *testing.T) {
 		ruleConfig.PlanetScaleOAuthToken().RuleID:           {10, 5.2, 8.2},
 		ruleConfig.PostManAPI().RuleID:                      {10, 5.2, 8.2},
 		ruleConfig.Prefect().RuleID:                         {10, 5.2, 8.2},
-		ruleConfig.PrivateKey().RuleID:                      {10, 5.2, 8.2},
+		rules.PrivateKey().RuleID:                           {10, 5.2, 8.2},
 		ruleConfig.PulumiAPIToken().RuleID:                  {10, 5.2, 8.2},
 		ruleConfig.PyPiUploadToken().RuleID:                 {10, 5.2, 8.2},
 		ruleConfig.RapidAPIAccessToken().RuleID:             {10, 5.2, 8.2},