Skip to content

Commit

Permalink
Simplify line content logic
Browse files Browse the repository at this point in the history
  • Loading branch information
LeonardoLordelloFontes committed Jan 20, 2025
1 parent c41a1b7 commit ba174ce
Show file tree
Hide file tree
Showing 5 changed files with 79 additions and 196 deletions.
2 changes: 1 addition & 1 deletion cmd/workers.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ func processItems(engine *engine.Engine, pluginName string) {
for item := range channels.Items {
report.TotalItemsScanned++
wgItems.Add(1)
go engine.Detect(item, secretsChan, wgItems, pluginName)
go engine.Detect(item, secretsChan, wgItems, pluginName, channels.Errors)
}
wgItems.Wait()
close(secretsChan)
Expand Down
8 changes: 6 additions & 2 deletions engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func Init(engineConfig EngineConfig) (*Engine, error) {
}, nil
}

func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string) {
func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.Secret, wg *sync.WaitGroup, pluginName string, errors chan error) {
defer wg.Done()

fragment := detect.Fragment{
Expand All @@ -96,6 +96,10 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S
endLine = value.EndLine

}
lineContent, err := linecontent.GetLineContent(value.Line, value.Secret)
if err != nil {
errors <- err
}
secret := &secrets.Secret{
ID: itemId,
Source: item.GetSource(),
Expand All @@ -105,7 +109,7 @@ func (e *Engine) Detect(item plugins.ISourceItem, secretsChannel chan *secrets.S
EndLine: endLine,
EndColumn: value.EndColumn,
Value: value.Secret,
LineContent: linecontent.GetLineContent(value.Line, value.Secret),
LineContent: lineContent,
RuleDescription: value.Description,
}
if !isSecretIgnored(secret, &e.ignoredIds, &e.allowedValues) {
Expand Down
7 changes: 5 additions & 2 deletions engine/engine_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,10 @@ func TestDetector(t *testing.T) {
}

secretsChan := make(chan *secrets.Secret, 1)
errorsChan := make(chan error, 1)
wg := &sync.WaitGroup{}
wg.Add(1)
detector.Detect(i, secretsChan, wg, fsPlugin.GetName())
detector.Detect(i, secretsChan, wg, fsPlugin.GetName(), errorsChan)
close(secretsChan)

s := <-secretsChan
Expand Down Expand Up @@ -153,10 +154,12 @@ func TestSecrets(t *testing.T) {
t.Run(name, func(t *testing.T) {
fmt.Printf("Start test %s", name)
secretsChan := make(chan *secrets.Secret, 1)
errorsChan := make(chan error, 1)
wg := &sync.WaitGroup{}
wg.Add(1)
detector.Detect(item{content: &secret.Content}, secretsChan, wg, fsPlugin.GetName())
detector.Detect(item{content: &secret.Content}, secretsChan, wg, fsPlugin.GetName(), errorsChan)
close(secretsChan)
close(errorsChan)

s := <-secretsChan

Expand Down
84 changes: 26 additions & 58 deletions engine/linecontent/linecontent.go
Original file line number Diff line number Diff line change
@@ -1,79 +1,47 @@
package linecontent

import (
"fmt"
"strings"
)

const (
lineContentMaxParseSize = 10000
contextLeftSizeLimit = 250
contextRightSizeLimit = 250
lineMaxParseSize = 10000
contextLeftSizeLimit = 250
contextRightSizeLimit = 250
)

func GetLineContent(line, secret string) string {
func GetLineContent(line, secret string) (string, error) {
lineSize := len(line)
if lineSize == 0 || len(secret) == 0 {
return ""
if lineSize == 0 {
return "", fmt.Errorf("failed to get line content: line empty")
}

// Truncate line to max parse size before converting to runes
shouldRemoveLastChars := false
if lineSize > lineContentMaxParseSize {
line = line[:lineContentMaxParseSize]
shouldRemoveLastChars = true // to prevent issues when truncating a multibyte character in the middle
if len(secret) == 0 {
return "", fmt.Errorf("failed to get line content: secret empty")
}

// Convert line and secret to runes
lineRunes, lineRunesSize := getLineRunes(line, shouldRemoveLastChars)
secretRunes := []rune(secret)
secretRunesSize := len(secretRunes)
// Truncate lineContent to max size
if lineSize > lineMaxParseSize {
line = line[:lineMaxParseSize]
lineSize = lineMaxParseSize
}

// Find the secret's position in the line (working with runes)
secretStartIndex := indexOf(lineRunes, secretRunes, lineRunesSize, secretRunesSize)
// Find the secret's position in the line
secretStartIndex := strings.Index(line, secret)
if secretStartIndex == -1 {
// Secret not found, return truncated content based on context limits
maxSize := contextLeftSizeLimit + contextRightSizeLimit
if lineRunesSize < maxSize {
return string(lineRunes)
if lineSize < maxSize {
return line, nil
}
return string(lineRunes[:maxSize])
return line[:maxSize], nil
}

// Calculate bounds for the result
secretEndIndex := secretStartIndex + secretRunesSize
secretEndIndex := secretStartIndex + len(secret)
start := max(secretStartIndex-contextLeftSizeLimit, 0)
end := min(secretEndIndex+contextRightSizeLimit, lineRunesSize)

return string(lineRunes[start:end])
}

func getLineRunes(line string, shouldRemoveLastChars bool) ([]rune, int) {
lineRunes := []rune(line)
lineRunesSize := len(lineRunes)
if shouldRemoveLastChars {
// A single rune can be up to 4 bytes in UTF-8 encoding.
// If truncation occurs in the middle of a multibyte character,
// it will leave a partial byte sequence, potentially consisting of
// up to 3 bytes. Each of these remaining bytes will be treated
// as an invalid character, displayed as a replacement character (�).
// To prevent this, we adjust the rune count by removing the last
// 3 runes, ensuring no partial characters are included.
lineRunesSize -= 3
}
return lineRunes[:lineRunesSize], lineRunesSize
}
end := min(secretEndIndex+contextRightSizeLimit, lineSize)

func indexOf(line, secret []rune, lineSize, secretSize int) int {
for i := 0; i <= lineSize-secretSize; i++ {
if compareRunes(line[i:i+secretSize], secret) {
return i
}
}
return -1
}

func compareRunes(a, b []rune) bool {
// a and b must have the same size.
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
return line[start:end], nil
}
Loading

0 comments on commit ba174ce

Please sign in to comment.