Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve MongoDB connection string matching #1550

Merged
merged 2 commits into from
Sep 23, 2024
Merged

Improve MongoDB connection string matching #1550

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8f0059c
feat(mongodb): improve conn string matching
rgmz Jul 26, 2023
446c8f0
fix(mongodb): err -> verificationErr
rgmz Sep 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 30 additions & 25 deletions pkg/detectors/mongodb/mongodb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package mongodb

import (
"context"
"errors"
"net/url"
"strings"
"time"
Expand All @@ -12,7 +13,6 @@ import (
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
"go.mongodb.org/mongo-driver/x/mongo/driver/topology"

"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
Expand All @@ -29,8 +29,9 @@ var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
var (
defaultTimeout = 2 * time.Second
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
connStrPat = regexp.MustCompile(`\b(mongodb(?:\+srv)?://(?P<username>\S{3,50}):(?P<password>\S{3,88})@(?P<host>[-.%\w]+(?::\d{1,5})?(?:,[-.%\w]+(?::\d{1,5})?)*)(?:/(?P<authdb>[\w-]+)?(?P<options>\?\w+=[\w@/.$-]+(?:&(?:amp;)?\w+=[\w@/.$-]+)*)?)?)(?:\b|$)`)
// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
placeholderPasswordPat = regexp.MustCompile(`^[xX]+|\*+$`)
)

// Keywords are used for efficiently pre-filtering chunks.
Expand All @@ -43,11 +44,17 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)

matches := keyPat.FindAllStringSubmatch(dataStr, -1)
matches := connStrPat.FindAllStringSubmatch(dataStr, -1)

for _, match := range matches {
resMatch := strings.TrimSpace(match[1])
// Filter out common placeholder passwords.
password := match[3]
if password == "" || placeholderPasswordPat.MatchString(password) {
continue
}

// If the query string contains `&amp;` the options will not be parsed.
resMatch := strings.Replace(strings.TrimSpace(match[1]), "&amp;", "&", -1)
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_MongoDB,
Raw: []byte(resMatch),
Expand All @@ -61,10 +68,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if timeout == 0 {
timeout = defaultTimeout
}
err := verifyUri(resMatch, timeout)
s1.Verified = err == nil
if !isErrDeterminate(err) {
s1.SetVerificationError(err, resMatch)
isVerified, verificationErr := verifyUri(ctx, resMatch, timeout)
s1.Verified = isVerified
if !isErrDeterminate(verificationErr) {
s1.SetVerificationError(verificationErr, resMatch)
}
}
results = append(results, s1)
Expand All @@ -78,23 +85,14 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {
}

func isErrDeterminate(err error) bool {
switch e := err.(type) {
case topology.ConnectionError:
switch e.Unwrap().(type) {
case *auth.Error:
return true
default:
return false
}
default:
return false
}
var authErr *auth.Error
return errors.As(err, &authErr)
}

func verifyUri(uri string, timeout time.Duration) error {
func verifyUri(ctx context.Context, uri string, timeout time.Duration) (bool, error) {
parsed, err := url.Parse(uri)
if err != nil {
return err
return false, err
}

params := url.Values{}
Expand All @@ -114,16 +112,23 @@ func verifyUri(uri string, timeout time.Duration) error {
parsed.Path = "/"
uri = parsed.String()

ctx, cancel := context.WithTimeout(context.Background(), timeout)
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
client, err := mongo.Connect(ctx, options.Client().SetTimeout(timeout).ApplyURI(uri))

clientOptions := options.Client().SetTimeout(timeout).ApplyURI(uri)
if err = clientOptions.Validate(); err != nil {
return false, err
}

client, err := mongo.Connect(ctx, clientOptions)
if err != nil {
return err
return false, err
}
defer func() {
_ = client.Disconnect(ctx)
}()
return client.Ping(ctx, readpref.Primary())
err = client.Ping(ctx, readpref.Primary())
return err == nil, err
}

func (s Scanner) Type() detectorspb.DetectorType {
Expand Down
195 changes: 195 additions & 0 deletions pkg/detectors/mongodb/mongodb_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,201 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)

func TestMongoDB_Pattern(t *testing.T) {
tests := []struct {
name string
data string
shouldMatch bool
match string
}{
// True positives
{
name: "long_password",
data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`,
shouldMatch: true,
},
{
name: "long_password2",
data: `mongodb://csb0230eada-2354-4c73-b3e4-8a1aaa996894:AiNtEyASbdXR5neJmTStMzKGItX2xvKuyEkcy65rviKD0ggZR19E1iVFIJ5ZAIY1xvvAiS5tOXsmACDbKDJIhQ==@csb0230eada-2354-4c73-b3e4-8a1aaa996894.mongo.cosmos.cloud-hostname.com:10255/csb-db0230eada-2354-4c73-b3e4-8a1aaa996894?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@csb0230eada-2354-4c73-b3e4-8a1aaa996894@`,
shouldMatch: true,
},
{
name: "long_password3",
data: `mongodb://amsdfasfsadfdfdfpshot:6xNRRsdfsdfafd9NodO8vAFFBEHidfdfdfa87QDKXdCMubACDbhfQH1g==@amssdfafdafdadbsnapshot.mongo.cosmos.azure.com:10255/?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@amssadfasdfdbsnsdfadfapshot@`,
shouldMatch: true,
},
{
name: "single_host",
data: `mongodb://myDBReader:D1fficultP%[email protected]`,
shouldMatch: true,
},
{
name: "single_host+port",
data: `mongodb://myDBReader:D1fficultP%[email protected]:27017`,
shouldMatch: true,
},
{
name: "single_host+port+authdb",
data: `mongodb://myDBReader:D1fficultP%[email protected]:27017/?authSource=admin`,
shouldMatch: true,
},
{
name: "single_host_ip",
data: `mongodb://myDBReader:D1fficultP%[email protected]`,
shouldMatch: true,
},
{
name: "single_host_ip+port",
data: `mongodb://myDBReader:D1fficultP%[email protected]:27017`,
shouldMatch: true,
},
{
name: "multiple_hosts_ip",
data: `mongodb://root:[email protected]:27018,192.168.74.143:27019`,
shouldMatch: true,
},
{
name: "multiple_hosts_ip+slash",
data: `mongodb://root:[email protected]:27018,192.168.74.143:27019/`,
shouldMatch: true,
},
{
name: "multiple_hosts+port+authdb",
data: `mongodb://myDBReader:D1fficultP%[email protected]:27017,mongodb0.example.com:27017,mongodb0.example.com:27017/?authSource=admin`,
shouldMatch: true,
},
{
name: "multiple_hosts+options",
data: `mongodb://username:[email protected]:27317,mongodb2.example.com,mongodb2.example.com:270/?connectTimeoutMS=300000&replicaSet=mySet&authSource=aDifferentAuthDB`,
shouldMatch: true,
},
{
name: "multiple_hosts2",
data: `mongodb://prisma:[email protected]:27017,srv2.bu2lt.mongodb.net:27017,srv3.bu2lt.mongodb.net:27017/test?retryWrites=true&w=majority`,
shouldMatch: true,
},
// TODO: These fail because the Go driver doesn't explicitly support `authMechanism=DEFAULT`[1].
// However, this seems like a valid option[2] and I'm going to try to get that behaviour changed.
//
// [1] https://github.com/mongodb/mongo-go-driver/blob/master/x/mongo/driver/connstring/connstring.go#L450-L451
// [2] https://www.mongodb.com/docs/drivers/node/current/fundamentals/authentication/mechanisms/
{
name: "encoded_options1",
data: `mongodb://dave:password@localhost:27017/?authMechanism=DEFAULT&amp;authSource=db&amp;ssl=true&quot;`,
shouldMatch: true,
match: "mongodb://dave:password@localhost:27017/?authMechanism=DEFAULT&authSource=db&ssl=true",
},
{
name: "encoded_options2",
data: `mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&amp;sslverifycertificate=false`,
shouldMatch: true,
match: "mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&sslverifycertificate=false",
},
{
name: "unix_socket",
data: `mongodb://u%24ername:pa%24%24w%7B%7Drd@%2Ftmp%2Fmongodb-27017.sock/test`,
shouldMatch: true,
},
{
name: "dashes",
data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`,
shouldMatch: true,
},
{
name: "protocol+srv",
// TODO: Figure out how to handle `mongodb+srv`. It performs a DNS lookup, which fails if the host doesn't exist.
//data: `mongodb+srv://root:[email protected]/mydb?retryWrites=true&w=majority`,
data: `mongodb://root:[email protected]/mydb?retryWrites=true&w=majority`,
shouldMatch: true,
},
{
name: "0.0.0.0_host",
data: `mongodb://username:[email protected]:27017/?authSource=admin`,
shouldMatch: true,
},
{
name: "localhost_host",
data: `mongodb://username:password@localhost:27017/?authSource=admin`,
shouldMatch: true,
},
{
name: "127.0.0.1_host",
data: `mongodb://username:[email protected]:27017/?authSource=admin`,
shouldMatch: true,
},
{
name: "docker_internal_host",
data: `mongodb://username:[email protected]:27018/?authMechanism=PLAIN&tls=true&tlsCertificateKeyFile=/etc/certs/client.pem&tlsCaFile=/etc/certs/rootCA-cert.pem`,
shouldMatch: true,
},
{
name: "options_authsource_external",
data: `mongodb://AKIAAAAAAAAAAAA:t9t2mawssecretkey@localhost:27017/?authMechanism=MONGODB-AWS&authsource=$external`,
shouldMatch: true,
},
{
name: "generic1",
data: `mongodb://root:[email protected]:27017/`,
shouldMatch: true,
},

// False positives
{
name: "no_password",
data: `mongodb://mongodb0.example.com:27017/?replicaSet=myRepl`,
shouldMatch: false,
},
{
name: "empty",
data: `mongodb://username:@mongodb0.example.com:27017/?replicaSet=myRepl`,
shouldMatch: false,
},
{
name: "placeholders_x+single_host",
data: `mongodb://xxxx:xxxxx@xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`,
shouldMatch: false,
},
{
name: "placeholders_x+multiple_hosts",
data: `mongodb://xxxx:xxxxx@xxxxxxx:3717,xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`,
shouldMatch: false,
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
s := Scanner{}

results, err := s.FromData(context.Background(), false, []byte(test.data))
if err != nil {
t.Errorf("MongoDB.FromData() error = %v", err)
return
}

if test.shouldMatch {
if len(results) == 0 {
t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
return
}
expected := test.data
if test.match != "" {
expected = test.match
}
result := string(results[0].Raw)
if result != expected {
t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, result)
return
}
} else {
if len(results) > 0 {
t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
return
}
}
})
}
}

func TestMongoDB_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
Expand Down
Loading