diff --git a/cmd/workers.go b/cmd/workers.go index ced1e4c..497e8c5 100644 --- a/cmd/workers.go +++ b/cmd/workers.go @@ -15,7 +15,7 @@ func processItems(engine *engine.Engine, pluginName string) { for item := range channels.Items { report.TotalItemsScanned++ wgItems.Add(1) - go engine.Detect(item, secretsChan, wgItems, pluginName) + go engine.Detect(item, secretsChan, wgItems, pluginName, channels.Errors) } wgItems.Wait() close(secretsChan) diff --git a/engine/engine.go b/engine/engine.go index 785f9f4..03a6935 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -78,7 +78,7 @@ func Init(engineConfig EngineConfig) (*Engine, error) { }, nil } -func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string) { +func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string, errors chan error) { defer wg.Done() fragment := detect.Fragment{ @@ -96,6 +96,10 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S endLine = value.EndLine } + lineContent, err := linecontent.GetLineContent(value.Line, value.Secret) + if err != nil { + errors <- err + } secret := &secrets.Secret{ ID: itemId, Source: item.GetSource(), @@ -105,7 +109,7 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S EndLine: endLine, EndColumn: value.EndColumn, Value: value.Secret, - LineContent: linecontent.GetLineContent(value.Line, value.Secret), + LineContent: lineContent, RuleDescription: value.Description, } if !isSecretIgnored(secret, &e.ignoredIds, &e.allowedValues) { diff --git a/engine/engine_test.go b/engine/engine_test.go index 20b65be..d31fa39 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -78,9 +78,10 @@ func TestDetector(t *testing.T) { } secretsChan := make(chan *secrets.Secret, 1) + errorsChan := make(chan error, 1) wg := &sync.WaitGroup{} wg.Add(1) - detector.Detect(i, secretsChan, wg, fsPlugin.GetName()) + detector.Detect(i, secretsChan, wg, fsPlugin.GetName(), errorsChan) close(secretsChan) s := <-secretsChan @@ -153,10 +154,12 @@ func TestSecrets(t *testing.T) { t.Run(name, func(t *testing.T) { fmt.Printf("Start test %s", name) secretsChan := make(chan *secrets.Secret, 1) + errorsChan := make(chan error, 1) wg := &sync.WaitGroup{} wg.Add(1) - detector.Detect(item{content: &secret.Content}, secretsChan, wg, fsPlugin.GetName()) + detector.Detect(item{content: &secret.Content}, secretsChan, wg, fsPlugin.GetName(), errorsChan) close(secretsChan) + close(errorsChan) s := <-secretsChan diff --git a/engine/linecontent/linecontent.go b/engine/linecontent/linecontent.go index 440167f..6562021 100644 --- a/engine/linecontent/linecontent.go +++ b/engine/linecontent/linecontent.go @@ -1,79 +1,47 @@ package linecontent +import ( + "fmt" + "strings" +) + const ( - lineContentMaxParseSize = 10000 - contextLeftSizeLimit = 250 - contextRightSizeLimit = 250 + lineMaxParseSize = 10000 + contextLeftSizeLimit = 250 + contextRightSizeLimit = 250 ) -func GetLineContent(line, secret string) string { +func GetLineContent(line, secret string) (string, error) { lineSize := len(line) - if lineSize == 0 || len(secret) == 0 { - return "" + if lineSize == 0 { + return "", fmt.Errorf("failed to get line content: line empty") } - // Truncate line to max parse size before converting to runes - shouldRemoveLastChars := false - if lineSize > lineContentMaxParseSize { - line = line[:lineContentMaxParseSize] - shouldRemoveLastChars = true // to prevent issues when truncating a multibyte character in the middle + if len(secret) == 0 { + return "", fmt.Errorf("failed to get line content: secret empty") } - // Convert line and secret to runes - lineRunes, lineRunesSize := getLineRunes(line, shouldRemoveLastChars) - secretRunes := []rune(secret) - secretRunesSize := len(secretRunes) + // Truncate lineContent to max size + if lineSize > lineMaxParseSize { + line = line[:lineMaxParseSize] + lineSize = lineMaxParseSize + } - // Find the secret's position in the line (working with runes) - secretStartIndex := indexOf(lineRunes, secretRunes, lineRunesSize, secretRunesSize) + // Find the secret's position in the line + secretStartIndex := strings.Index(line, secret) if secretStartIndex == -1 { // Secret not found, return truncated content based on context limits maxSize := contextLeftSizeLimit + contextRightSizeLimit - if lineRunesSize < maxSize { - return string(lineRunes) + if lineSize < maxSize { + return line, nil } - return string(lineRunes[:maxSize]) + return line[:maxSize], nil } // Calculate bounds for the result - secretEndIndex := secretStartIndex + secretRunesSize + secretEndIndex := secretStartIndex + len(secret) start := max(secretStartIndex-contextLeftSizeLimit, 0) - end := min(secretEndIndex+contextRightSizeLimit, lineRunesSize) - - return string(lineRunes[start:end]) -} - -func getLineRunes(line string, shouldRemoveLastChars bool) ([]rune, int) { - lineRunes := []rune(line) - lineRunesSize := len(lineRunes) - if shouldRemoveLastChars { - // A single rune can be up to 4 bytes in UTF-8 encoding. - // If truncation occurs in the middle of a multibyte character, - // it will leave a partial byte sequence, potentially consisting of - // up to 3 bytes. Each of these remaining bytes will be treated - // as an invalid character, displayed as a replacement character (�). - // To prevent this, we adjust the rune count by removing the last - // 3 runes, ensuring no partial characters are included. - lineRunesSize -= 3 - } - return lineRunes[:lineRunesSize], lineRunesSize -} + end := min(secretEndIndex+contextRightSizeLimit, lineSize) -func indexOf(line, secret []rune, lineSize, secretSize int) int { - for i := 0; i <= lineSize-secretSize; i++ { - if compareRunes(line[i:i+secretSize], secret) { - return i - } - } - return -1 -} - -func compareRunes(a, b []rune) bool { - // a and b must have the same size. - for i := range a { - if a[i] != b[i] { - return false - } - } - return true + return line[start:end], nil } diff --git a/engine/linecontent/linecontent_test.go b/engine/linecontent/linecontent_test.go index 76082d4..e9db779 100644 --- a/engine/linecontent/linecontent_test.go +++ b/engine/linecontent/linecontent_test.go @@ -11,198 +11,106 @@ const ( func TestGetLineContent(t *testing.T) { tests := []struct { - name string - line string - secret string - expected string + name string + line string + secret string + expected string + error bool + errorMessage string }{ { - name: "Empty line", - line: "", - secret: dummySecret, - expected: "", + name: "Empty line", + line: "", + secret: dummySecret, + expected: "", + error: true, + errorMessage: "failed to get line content: line empty", }, { - name: "Empty secret", - line: "line", - secret: "", - expected: "", + name: "Empty secret", + line: "line", + secret: "", + expected: "", + error: true, + errorMessage: "failed to get line content: secret empty", }, { name: "Secret not found with line size smaller than the parse limit", line: "Dummy content line", secret: dummySecret, expected: "Dummy content line", + error: false, }, { name: "Secret not found with secret present and line size larger than the parse limit", - line: "This is the start of a big line content" + strings.Repeat("A", lineContentMaxParseSize) + dummySecret, + line: "This is the start of a big line content" + strings.Repeat("A", lineMaxParseSize) + dummySecret, secret: dummySecret, expected: "This is the start of a big line content" + strings.Repeat("A", contextLeftSizeLimit+contextRightSizeLimit-len("This is the start of a big line content")), + error: false, }, { name: "Secret larger than the line", line: strings.Repeat("B", contextLeftSizeLimit) + strings.Repeat("A", contextRightSizeLimit), secret: "large secret" + strings.Repeat("B", contextRightSizeLimit+contextLeftSizeLimit+100), expected: strings.Repeat("B", contextLeftSizeLimit) + strings.Repeat("A", contextRightSizeLimit), + error: false, }, { name: "Secret at the beginning with line size smaller than the parse limit", - line: "start:" + dummySecret + strings.Repeat("A", lineContentMaxParseSize/2), + line: "start:" + dummySecret + strings.Repeat("A", lineMaxParseSize/2), secret: dummySecret, expected: "start:" + dummySecret + strings.Repeat("A", contextRightSizeLimit), + error: false, }, { name: "Secret found in middle with line size smaller than the parse limit", line: "start" + strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit) + "end", secret: dummySecret, expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit), + error: false, }, { name: "Secret at the end with line size smaller than the parse limit", - line: strings.Repeat("A", lineContentMaxParseSize/2) + dummySecret + ":end", + line: strings.Repeat("A", lineMaxParseSize/2) + dummySecret + ":end", secret: dummySecret, expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + ":end", + error: false, }, { name: "Secret at the beginning with line size larger than the parse limit", - line: "start:" + dummySecret + strings.Repeat("A", lineContentMaxParseSize), + line: "start:" + dummySecret + strings.Repeat("A", lineMaxParseSize), secret: dummySecret, expected: "start:" + dummySecret + strings.Repeat("A", contextRightSizeLimit), + error: false, }, { name: "Secret found in middle with line size larger than the parse limit", - line: "start" + strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", lineContentMaxParseSize) + "end", + line: "start" + strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", lineMaxParseSize) + "end", secret: dummySecret, expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", contextRightSizeLimit), + error: false, }, { name: "Secret at the end with line size larger than the parse limit", - line: strings.Repeat("A", lineContentMaxParseSize-100) + dummySecret + strings.Repeat("A", lineContentMaxParseSize), - secret: dummySecret, - expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(100, 1, len(dummySecret))), - }, - { - name: "Secret at the beginning with line containing 2 byte chars and size smaller than the parse limit", - line: "start:" + dummySecret + strings.Repeat("é", lineContentMaxParseSize/4), - secret: dummySecret, - expected: "start:" + dummySecret + strings.Repeat("é", contextRightSizeLimit), - }, - { - name: "Secret found in middle with line containing 2 byte chars and size smaller than the parse limit", - line: "start" + strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", contextRightSizeLimit) + "end", - secret: dummySecret, - expected: strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", contextRightSizeLimit), - }, - { - name: "Secret at the end with line containing 2 byte chars and size smaller than the parse limit", - line: strings.Repeat("é", lineContentMaxParseSize/4) + dummySecret + ":end", - secret: dummySecret, - expected: strings.Repeat("é", contextLeftSizeLimit) + dummySecret + ":end", - }, - { - name: "Secret at the beginning with line containing 2 byte chars and size larger than the parse limit", - line: "start:" + dummySecret + strings.Repeat("é", lineContentMaxParseSize/2), - secret: dummySecret, - expected: "start:" + dummySecret + strings.Repeat("é", contextRightSizeLimit), - }, - { - name: "Secret found in middle with line containing 2 byte chars and size larger than the parse limit", - line: "start" + strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", lineContentMaxParseSize/2) + "end", - secret: dummySecret, - expected: strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", contextRightSizeLimit), - }, - { - name: "Secret at the end with line containing 2 byte chars and size larger than the parse limit", - line: strings.Repeat("é", lineContentMaxParseSize/2-100) + dummySecret + strings.Repeat("é", lineContentMaxParseSize/2), + line: strings.Repeat("A", lineMaxParseSize-100) + dummySecret + strings.Repeat("A", lineMaxParseSize), secret: dummySecret, - expected: strings.Repeat("é", contextLeftSizeLimit) + dummySecret + strings.Repeat("é", calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(100, 2, len(dummySecret))), - }, - { - name: "Secret at the beginning with line containing 3 byte chars and size smaller than the parse limit", - line: "start:" + dummySecret + strings.Repeat("ࠚ", lineContentMaxParseSize/6), - secret: dummySecret, - expected: "start:" + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit), - }, - { - name: "Secret found in middle with line containing 3 byte chars and size smaller than the parse limit", - line: "start" + strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit) + "end", - secret: dummySecret, - expected: strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit), - }, - { - name: "Secret at the end with line containing 3 byte chars and size smaller than the parse limit", - line: strings.Repeat("ࠚ", lineContentMaxParseSize/6) + dummySecret + ":end", - secret: dummySecret, - expected: strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + ":end", - }, - { - name: "Secret at the beginning with line containing 3 byte chars and size larger than the parse limit", - line: "start:" + dummySecret + strings.Repeat("ࠚ", lineContentMaxParseSize/3), - secret: dummySecret, - expected: "start:" + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit), - }, - { - name: "Secret found in middle with line containing 3 byte chars and size larger than the parse limit", - line: "start" + strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", lineContentMaxParseSize/3) + "end", - secret: dummySecret, - expected: strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", contextRightSizeLimit), - }, - { - name: "Secret at the end with line containing 3 byte chars and size larger than the parse limit", - line: strings.Repeat("ࠚ", lineContentMaxParseSize/3-100) + dummySecret + strings.Repeat("ࠚ", lineContentMaxParseSize/3), - secret: dummySecret, - expected: strings.Repeat("ࠚ", contextLeftSizeLimit) + dummySecret + strings.Repeat("ࠚ", calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(100, 3, len(dummySecret))), - }, - { - name: "Secret at the beginning with line containing 4 byte chars and size smaller than the parse limit", - line: "start:" + dummySecret + strings.Repeat("𝄞", lineContentMaxParseSize/8), - secret: dummySecret, - expected: "start:" + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit), - }, - { - name: "Secret found in middle with line containing 4 byte chars and size smaller than the parse limit", - line: "start" + strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit) + "end", - secret: dummySecret, - expected: strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit), - }, - { - name: "Secret at the end with line containing 4 byte chars and size smaller than the parse limit", - line: strings.Repeat("𝄞", lineContentMaxParseSize/8) + dummySecret + ":end", - secret: dummySecret, - expected: strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + ":end", - }, - { - name: "Secret at the beginning with line containing 4 byte chars and size larger than the parse limit", - line: "start:" + dummySecret + strings.Repeat("𝄞", lineContentMaxParseSize/4), - secret: dummySecret, - expected: "start:" + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit), - }, - { - name: "Secret found in middle with line containing 4 byte chars and size larger than the parse limit", - line: "start" + strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", lineContentMaxParseSize/4) + "end", - secret: dummySecret, - expected: strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", contextRightSizeLimit), - }, - { - name: "Secret at the end with line containing 4 byte chars and size larger than the parse limit", - line: strings.Repeat("𝄞", lineContentMaxParseSize/4-100) + dummySecret + strings.Repeat("𝄞", lineContentMaxParseSize/4), - secret: dummySecret, - expected: strings.Repeat("𝄞", contextLeftSizeLimit) + dummySecret + strings.Repeat("𝄞", calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(100, 4, len(dummySecret))), + expected: strings.Repeat("A", contextLeftSizeLimit) + dummySecret + strings.Repeat("A", 100-len(dummySecret)), + error: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := GetLineContent(tt.line, tt.secret) + got, err := GetLineContent(tt.line, tt.secret) + if (err != nil) != tt.error { + t.Fatalf("GetLineContent() error = %v, wantErr %v", err, tt.error) + } + if err != nil && err.Error() != tt.errorMessage { + t.Errorf("GetLineContent() error message = %v, want %v", err.Error(), tt.errorMessage) + } if got != tt.expected { t.Errorf("GetLineContent() = %v, want %v", got, tt.expected) } }) } } - -func calculateRepeatForSecretAtTheEndWithLargerThanParseLimit(offset, bytes, secretLength int) int { - remainingSize := lineContentMaxParseSize - ((lineContentMaxParseSize/bytes - offset) * bytes) - secretLength - return (remainingSize - ((3 - ((remainingSize) % bytes)) * bytes)) / bytes -}