Skip to content

Commit

Permalink
Added line numbers
Browse files Browse the repository at this point in the history 
Added Coloring output
Added check for max file size
Added exts to ignore
Added ignore directory function
  • Loading branch information
k4ch0w committed Jul 10, 2017
1 parent 2700a40 commit 6341292
Show file tree
Hide file tree
Showing 4 changed files with 181 additions and 136 deletions.
153 changes: 85 additions & 68 deletions filesystem.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,33 @@ import (
"strings"
"io/ioutil"
"bytes"
"github.com/fatih/color"
"strconv"
"github.com/pkg/errors"
)


type FileStruct struct {
Filename string
Path string
Path string
}

var ignoreExts = []string{".dll", ".nupkg"}
const MAX_FILE_SIZE_TO_SCAN = 200000
var ignoreExts = []string{".dll", ".nupkg", ".mp4", ".mkv", ".avi", ".flv", ".wmv", ".mov", ".pdf" }

func appendFilesystemMatch(pattern Pattern, filestruct FileStruct, lineMatched string, table map[string]*Match) {
func appendFilesystemMatch(pattern Pattern, filestruct FileStruct, lineMatched string,
lineNum int, table map[string]*Match) {
name := filestruct.Filename
_, exists := table[name]
if !exists {
table[name] = &Match{filestruct.Filename, filestruct.Path,
nil,pattern.Description, pattern.Value, lineMatched}
fmt.Println(fmt.Sprintf("Found match %s: %s %s", pattern.Description, filestruct.Path, pattern.Value))
nil, pattern.Description,
pattern.Value, lineMatched, lineNum}
fmt.Println(color.BlueString("Found match %s %s: ", pattern.Description, pattern.Value))
if lineNum == NO_LINE_NUMBER_APPLICABLE{
color.Green("%s", filestruct.Path)
} else{
color.Green("%s:%d ", filestruct.Path, lineNum)
}
}
}
func getAllFilesInDirectory(dir string) ([]FileStruct, error) {
Expand All @@ -32,99 +42,106 @@ func getAllFilesInDirectory(dir string) ([]FileStruct, error) {
fileList = append(fileList, FileStruct{f.Name(), path})
return nil
})
if err != nil{
if err != nil {
return nil, err
}
return fileList, nil
}

func searchThroughFile(path string) (bool, Pattern, string, error){
fi, err := os.Stat(path)
if err != nil {
return false, Pattern{}, "", err
}
ext := filepath.Ext(path)
for _, ignore := range ignoreExts{
if ext == ignore{
return false, Pattern{}, "", err
}
}
if fi.IsDir() || strings.Contains(path, ".git"){ //Tons of false positives in .git folder
return false, Pattern{}, "", err
}
f, err := ioutil.ReadFile(path)
if err != nil{
return false, Pattern{}, "", err
}

lines := bytes.Split(f, []byte("\n"))
for _, pattern := range fileContentRegexes{
for _, line := range lines{
if pattern.Regex.Match(line){
return true, pattern, string(line), nil
}
}
}
return false, Pattern{}, "", nil
func searchThroughFile(path string) (bool, Pattern, string, int, error) {
fi, err := os.Stat(path)
if err != nil {
return false, Pattern{}, "", -1, err
}
ext := filepath.Ext(path)
for _, ignore := range ignoreExts {
if ext == ignore {
return false, Pattern{}, IS_FILENAME_MATCH,
NO_LINE_NUMBER_APPLICABLE, errors.New("Ignoring file extension " + path)
}
}
if fi.IsDir() || checkIfInsideIgnoredDirectory(path) {
return false, Pattern{}, IS_FILENAME_MATCH,
NO_LINE_NUMBER_APPLICABLE, err
}
if fi.Size() / 1024 >= MAX_FILE_SIZE_TO_SCAN{
return false, Pattern{}, IS_FILENAME_MATCH,
NO_LINE_NUMBER_APPLICABLE, errors.New(path + " is over max file size to scan")
}
f, err := ioutil.ReadFile(path)
if err != nil {
return false, Pattern{}, IS_FILENAME_MATCH,
NO_LINE_NUMBER_APPLICABLE, err
}
lines := bytes.Split(f, []byte("\n"))
for _, pattern := range fileContentRegexes {
lineNum := 0
for _, line := range lines {
lineNum++
if pattern.Regex.Match(line) {
return true, pattern, string(line), lineNum, nil
}
}
}
return false, Pattern{}, IS_FILENAME_MATCH, NO_LINE_NUMBER_APPLICABLE, nil
}
func searchFileContents(files []FileStruct, table map[string]*Match){
for _, f := range files{
match, p, line, err :=searchThroughFile(f.Path)
if err == nil && match{
appendFilesystemMatch(p,f, line,table)
}
}
func searchFileContents(files []FileStruct, table map[string]*Match) {
for _, f := range files {
match, p, line, lineNum, err := searchThroughFile(f.Path)
if err == nil && match {
appendFilesystemMatch(p, f, line, lineNum, table)
}
}
}

func outputCSVFilesystem(matches []*Match){
records := [][]string{{"Filename", "Description", "Filepath", "LineMatched"}}
func outputCSVFilesystem(matches []*Match) {
records := [][]string{{"Filename", "Description", "Filepath", "LineMatched", "LineNumber"}}
for _, match := range matches {
records = append(records, []string{match.Filename, match.Description,
match.Filepath, truncateString(match.LineMatched, 160)})
match.Filepath, truncateString(match.LineMatched, 160),
strconv.Itoa(match.LineNumber)})
}
outputCSV(*outputCSVFlag, records)
}

func filesystemSecretFilenameRegexSearch(files []FileStruct, table map[string]*Match){
func filesystemSecretFilenameRegexSearch(files []FileStruct, table map[string]*Match) {
for _, filestruct := range files {
for _, pattern := range secretFileNameRegexes {
if pattern.Regex.MatchString(filestruct.Filename) {
appendFilesystemMatch(pattern, filestruct, "", table)
appendFilesystemMatch(pattern, filestruct, IS_FILENAME_MATCH, NO_LINE_NUMBER_APPLICABLE, table)
break
}
}
}
}


func filesystemSecretFilenameLiteralSearch(files []FileStruct, table map[string]*Match) {
for _, pattern := range secretFileNameLiterals {
for _, filename := range files {
if strings.Contains(filename.Filename, pattern.Value) {
appendFilesystemMatch(pattern, filename, "", table)
appendFilesystemMatch(pattern, filename, IS_FILENAME_MATCH, NO_LINE_NUMBER_APPLICABLE, table)
}
}
}

}

func startFileSystemScan(){
func startFileSystemScan() {
files, err := getAllFilesInDirectory(*dirToScanFlag)
if err != nil {
fmt.Println(err)
os.Exit(-1)
}
var uniqFileSet = make(map[string]*Match)
filesystemSecretFilenameLiteralSearch(files, uniqFileSet)
filesystemSecretFilenameRegexSearch(files, uniqFileSet)
if !*fileNamesOnlyFlag{
searchFileContents(files, uniqFileSet)
}
if len(*outputCSVFlag) != 0 {
var results []*Match
for _, values := range uniqFileSet{
results = append(results, values)
}
outputCSVFilesystem(results)
if err != nil {
color.Red(err.Error())
os.Exit(-1)
}
var uniqFileSet = make(map[string]*Match)
filesystemSecretFilenameLiteralSearch(files, uniqFileSet)
filesystemSecretFilenameRegexSearch(files, uniqFileSet)
if !*fileNamesOnlyFlag {
searchFileContents(files, uniqFileSet)
}
if len(*outputCSVFlag) != 0 {
var results []*Match
for _, values := range uniqFileSet {
results = append(results, values)
}
}
outputCSVFilesystem(results)
}
}
64 changes: 39 additions & 25 deletions git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ import (
"net/http"
"fmt"
"strings"
"bytes"
"github.com/fatih/color"
"strconv"
)

type GitFile struct {
Expand All @@ -18,18 +21,19 @@ type GitFile struct {
RepoUrl string
}

const gitHubApiVersion = "application/vnd.github.v3+json"
const GITHUB_API_VERSION = "application/vnd.github.v3+json"
const INVALID_PLAINTEXT_CREDS = -10
const INVALID_SSH_CREDS= -11


func gitHubPerformRequest(urlString string) ([]string, error) {
var urls = []string{}
client := &http.Client{}
req, err := http.NewRequest("GET", urlString, nil)
if err != nil {
return urls, err
}
req.Header.Set("Accept", gitHubApiVersion)
req.Header.Set("Accept", GITHUB_API_VERSION)
//TODO: Add auth here
//req.SetBasicAuth(Github.Username, Github.AccessToken)
resp, err := client.Do(req)
Expand Down Expand Up @@ -110,7 +114,18 @@ func certificateCheckCallback(cert *git.Certificate, valid bool, hostname string
return 0
}

func searchBlobContents(contents []byte) (bool, Pattern){
func searchBlobContents(contents []byte) (bool, Pattern, string, int){
lines := bytes.Split(contents, []byte("\n"))
for _, pattern := range fileContentRegexes {
lineNumb := 0
for _, line := range lines {
lineNumb++
if pattern.Regex.Match(line) {
return true, pattern, string(line), lineNumb
}
}
}
return false, Pattern{}, "", -1
/*
This is matching on things it shouldn't
for _, pattern := range fileContentLiterals{
Expand All @@ -121,12 +136,6 @@ func searchBlobContents(contents []byte) (bool, Pattern){
}
}
*/
for _, pattern := range fileContentRegexes{
if pattern.Regex.Match(contents){
return true, pattern
}
}
return false, Pattern{}
}

func gitRepoSearch(repoUrl string) ([]*Match, error){
Expand All @@ -144,7 +153,7 @@ func gitRepoSearch(repoUrl string) ([]*Match, error){
func getOrgRepo(url string, matchChan chan[]*Match){
matches, err := gitRepoSearch(url)
if err != nil{
fmt.Println(err)
color.Red(err.Error())
}
matchChan <- matches
}
Expand Down Expand Up @@ -219,12 +228,12 @@ func getRepoFilenames(repoUrl string) ([]GitFile, []*Match, error){
if te.Type == git.ObjectBlob && te.Filemode == git.FilemodeBlob{
gitFile := GitFile{te.Name,
td, commit.Id().String(), repoUrl}
if !*fileNamesOnlyFlag {
if !*fileNamesOnlyFlag && checkIfInsideIgnoredDirectory(td){
blob, err := repo.LookupBlob(te.Id)
if err == nil {
match, pattern := searchBlobContents(blob.Contents())
match, pattern, line, lineNum := searchBlobContents(blob.Contents())
if match {
appendGitMatch(pattern, gitFile, table)
appendGitMatch(pattern, gitFile, line, lineNum, table)
}
}
}
Expand All @@ -241,8 +250,8 @@ func getRepoFilenames(repoUrl string) ([]GitFile, []*Match, error){
}



func appendGitMatch(pattern Pattern, filename GitFile, table map[string]*Match) {
func appendGitMatch(pattern Pattern, filename GitFile,
lineMatched string, lineNumb int, table map[string]*Match) {
//You can't assume a file hasn't moved directories in history
// which is why you use the filepath + filename for uniqueness
path := filename.Filepath
Expand All @@ -255,12 +264,16 @@ func appendGitMatch(pattern Pattern, filename GitFile, table map[string]*Match)
if exists {
table[path].CommitIds = append(table[path].CommitIds, filename.CommitId)
} else {
//TODO: Line matched
table[path] = &Match{filename.Name, path,
[]string{filename.CommitId}, pattern.Description,
pattern.Value, ""}
fmt.Println(fmt.Sprintf("Found match %s %s %s %s", pattern.Description,
path, filename.RepoUrl, pattern.Value))
pattern.Value, lineMatched, lineNumb}

fmt.Print(color.BlueString("Found match %s %s ", pattern.Description, pattern.Value))
if lineNumb == NO_LINE_NUMBER_APPLICABLE{
color.Green("%s %s", path, filename.RepoUrl)
} else{
color.Green("%s:%d %s", path, lineNumb, filename.RepoUrl)
}
}
}

Expand All @@ -269,7 +282,7 @@ func gitSecretFilenameLiteralSearch(files []GitFile) []*Match {
for _, pattern := range secretFileNameLiterals {
for _, filename := range files {
if strings.Contains(filename.Name, pattern.Value) {
appendGitMatch(pattern, filename, table)
appendGitMatch(pattern, filename, IS_FILENAME_MATCH, NO_LINE_NUMBER_APPLICABLE, table)
}
}
}
Expand All @@ -285,7 +298,7 @@ func gitSecretFilenameRegexSearch(files []GitFile) []*Match {
for _, filename := range files {
for _, pattern := range secretFileNameRegexes {
if pattern.Regex.MatchString(filename.Name) {
appendGitMatch(pattern, filename, table)
appendGitMatch(pattern, filename, IS_FILENAME_MATCH, NO_LINE_NUMBER_APPLICABLE, table)
break
}
}
Expand All @@ -298,18 +311,19 @@ func gitSecretFilenameRegexSearch(files []GitFile) []*Match {
}

func outputCSVGitRepo(matches []*Match){
records := [][]string{{"Filename", "Description", "Filepath", "CommitID", "Value"}}
records := [][]string{{"Filename", "Description", "Filepath", "CommitID", "Value", "Line Number"}}
for _, match := range matches {
records = append(records, []string{match.Filename, match.Description, match.Filepath,
strings.Join(match.CommitIds, "|"), match.Value})
strings.Join(match.CommitIds, "|"),
match.Value, strconv.Itoa(match.LineNumber)})
}
outputCSV(*outputCSVFlag, records)
}

func startGitRepoScan(){
results, err := gitRepoSearch(*repoToScanFlag)
if err != nil{
fmt.Println(err)
color.Red(err.Error())
}
if len(*outputCSVFlag) != 0 {
outputCSVGitRepo(results)
Expand All @@ -320,7 +334,7 @@ func startGitOrganizationScan(){
var results = []*Match{}
urls, err := getAllOrganizationsRepoUrls(*organizationFlag)
if err!= nil {
fmt.Println(err)
color.Red(err.Error())
os.Exit(-1)
}
matchChan := make(chan []*Match, len(urls))
Expand Down
Loading

0 comments on commit 6341292

Please sign in to comment.