From 50a7b49d4beb1d721c5f419dafd2230b9d1e3c9c Mon Sep 17 00:00:00 2001 From: Gregory Boddin Date: Sat, 6 May 2023 13:55:18 +0200 Subject: [PATCH 01/15] Code cleaning, re-order sources (#828) --- v2/pkg/passive/sources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index f8a27d792..02550bb71 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -70,6 +70,7 @@ var AllSources = [...]subscraping.Source{ &hackertarget.Source{}, &hunter.Source{}, &intelx.Source{}, + &leakix.Source{}, &passivetotal.Source{}, &quake.Source{}, &rapiddns.Source{}, @@ -84,7 +85,6 @@ var AllSources = [...]subscraping.Source{ &whoisxmlapi.Source{}, &zoomeye.Source{}, &zoomeyeapi.Source{}, - &leakix.Source{}, // &threatminer.Source{}, // failing api // &reconcloud.Source{}, // failing due to cloudflare bot protection } From 48ec6c4b4408d34fd154ef8f5af20dba73a63ece Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 15:39:02 +0530 Subject: [PATCH 02/15] chore(deps): bump golang from 1.20.3-alpine to 1.20.4-alpine (#830) Bumps golang from 1.20.3-alpine to 1.20.4-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f2b63c2bf..91d844f1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.20.3-alpine AS build-env +FROM golang:1.20.4-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app From f19bb44535c179d26b6d40a133e138bd69f931db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 15:39:16 +0530 Subject: [PATCH 03/15] chore(deps): bump github.com/projectdiscovery/gologger in /v2 (#833) Bumps [github.com/projectdiscovery/gologger](https://github.com/projectdiscovery/gologger) from 1.1.8 to 1.1.10. - [Release notes](https://github.com/projectdiscovery/gologger/releases) - [Commits](https://github.com/projectdiscovery/gologger/compare/v1.1.8...v1.1.10) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/gologger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 19 +++---------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 02002f0d8..6183c1c9a 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -10,7 +10,7 @@ require ( github.com/projectdiscovery/chaos-client v0.5.1 github.com/projectdiscovery/dnsx v1.1.4 github.com/projectdiscovery/fdmax v0.0.4 - github.com/projectdiscovery/gologger v1.1.8 + github.com/projectdiscovery/gologger v1.1.10 github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/utils v0.0.26 github.com/rs/xid v1.5.0 diff --git a/v2/go.sum b/v2/go.sum index 3ca138e37..10251b337 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -23,7 +23,6 @@ github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdk github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -36,17 +35,14 @@ github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdf github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= @@ -64,12 +60,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora v2.0.3+incompatible h1:tOpm7WcpBTn4fjmVfgpQq0EfczGlG91VSDkswnjF5A8= @@ -104,12 +96,10 @@ github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKt github.com/muesli/termenv v0.13.0/go.mod h1:sP1+uffeLaEYpyOTb8pLCUctGcGLnoFjSn4YJK5e2bc= github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -126,8 +116,8 @@ github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUd github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.8 h1:Urhm2Isq2BdRt8h4h062lHKYXO65RHRjGTDSkUwex/g= github.com/projectdiscovery/goflags v0.1.8/go.mod h1:Yxi9tclgwGczzDU65ntrwaIql5cXeTvW5j2WxFuF+Jk= -github.com/projectdiscovery/gologger v1.1.8 h1:CFlCzGlqAhPqWIrAXBt1OVh5jkMs1qgoR/z4xhdzLNE= -github.com/projectdiscovery/gologger v1.1.8/go.mod h1:bNyVaC1U/NpJtFkJltcesn01NR3K8Hg6RsLVce6yvrw= +github.com/projectdiscovery/gologger v1.1.10 h1:XNRdtzLTdxiFGuK9gutoL752mykzXDoii4P2yDovqck= +github.com/projectdiscovery/gologger v1.1.10/go.mod h1:VqANHK7qcEq3i6/vV5HNWwdyv2aFPSrlaVDU4Ogrc6U= github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= github.com/projectdiscovery/retryabledns v1.0.23 h1:8X6HECevl6n7K7kxqfHLTcsCVXDdz9HR+1hS6fto7sw= @@ -158,7 +148,6 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= @@ -246,13 +235,11 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= gopkg.in/djherbis/times.v1 v1.3.0/go.mod h1:AQlg6unIsrsCEdQYhTzERy542dz6SFdQFZFv6mUY0P8= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From bc0737fec10f644f7321ece3f6740df1b5932fff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 15 May 2023 14:57:08 +0000 Subject: [PATCH 04/15] chore(deps): bump alpine from 3.17.3 to 3.18.0 Bumps alpine from 3.17.3 to 3.18.0. --- updated-dependencies: - dependency-name: alpine dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 91d844f1f..46610aa97 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ RUN go mod download RUN go build ./cmd/subfinder # Release -FROM alpine:3.17.3 +FROM alpine:3.18.0 RUN apk -U upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=build-env /app/v2/subfinder /usr/local/bin/ From 64f1c9d9aca3673082cb08c5e12d2322ccd91438 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 13:57:27 +0530 Subject: [PATCH 05/15] chore(deps): bump github.com/projectdiscovery/utils in /v2 (#842) Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.26 to 0.0.32. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.26...v0.0.32) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 9 +++++---- v2/go.sum | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 6183c1c9a..892ec8177 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.10 github.com/projectdiscovery/ratelimit v0.0.6 - github.com/projectdiscovery/utils v0.0.26 + github.com/projectdiscovery/utils v0.0.32 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.8.2 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -51,6 +51,7 @@ require ( github.com/nwaples/rardecode v1.1.3 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect + github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.0.1 // indirect github.com/projectdiscovery/retryablehttp-go v1.0.15 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -64,7 +65,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.7.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.8.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -83,6 +84,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.8 github.com/projectdiscovery/retryabledns v1.0.23 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.8.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 10251b337..c8710199e 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -106,6 +106,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= +github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= github.com/projectdiscovery/cdncheck v1.0.1 h1:kv1LRsHJl8TY5zMOhhcpGaBrBAdu01nmMVYqOxGwcMU= github.com/projectdiscovery/cdncheck v1.0.1/go.mod h1:5l2DFwQNbR3uDYiyUpUQLN0I37+JnE8rSeBpd7WQR/Y= github.com/projectdiscovery/chaos-client v0.5.1 h1:NFxU7cxBY7AL1OKxQQRLRvJrkXlV+SP8zv7VEkVVJ7U= @@ -124,8 +126,8 @@ github.com/projectdiscovery/retryabledns v1.0.23 h1:8X6HECevl6n7K7kxqfHLTcsCVXDd github.com/projectdiscovery/retryabledns v1.0.23/go.mod h1:tj8BEe0jWCB4m3aAhHq4S2dqfpod0h+BiCZmXkXAGac= github.com/projectdiscovery/retryablehttp-go v1.0.15 h1:kP9x9f++QimRwb8ABqnI1dhEymvnZXS2Wp2Zs4rWk/c= github.com/projectdiscovery/retryablehttp-go v1.0.15/go.mod h1:+OzSFUv3sQcPt+MgbNx6X/Q3ESxqPUQSphqG5kxoIgI= -github.com/projectdiscovery/utils v0.0.26 h1:89t05av9r5gtHBZLCiEkid1rrUjHlwKppz5z5iYzeG0= -github.com/projectdiscovery/utils v0.0.26/go.mod h1:lCiVsbo79iqy4ihUQWUY4ywz0J13Mqudirgk+IV767o= +github.com/projectdiscovery/utils v0.0.32 h1:TjUxFmRG9hiV6jauYsnIRiy08lTgGHGi8avstedNGXw= +github.com/projectdiscovery/utils v0.0.32/go.mod h1:SaOpcZ2dJ47NE3t4R/YC2XpUyRZC6v5k8sj2TFro6+k= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= @@ -188,11 +190,11 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= @@ -207,8 +209,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 2b53bddf7b6b67eb7f543d67e294decd7fe985d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 13:57:52 +0530 Subject: [PATCH 06/15] chore(deps): bump github.com/stretchr/testify from 1.8.2 to 1.8.3 in /v2 (#844) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.2 to 1.8.3. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.2...v1.8.3) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 892ec8177..069e99503 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,7 +14,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.6 github.com/projectdiscovery/utils v0.0.32 github.com/rs/xid v1.5.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 golang.org/x/exp v0.0.0-20230420155640-133eef4313cb gopkg.in/yaml.v3 v3.0.1 diff --git a/v2/go.sum b/v2/go.sum index c8710199e..937ffa31d 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -138,15 +138,11 @@ github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= From 90741cd356a024632fe2a00d24d77d6577c5a43f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 May 2023 14:06:50 +0530 Subject: [PATCH 07/15] chore(deps): bump github.com/projectdiscovery/ratelimit in /v2 (#843) Bumps [github.com/projectdiscovery/ratelimit](https://github.com/projectdiscovery/ratelimit) from 0.0.6 to 0.0.8. - [Release notes](https://github.com/projectdiscovery/ratelimit/releases) - [Commits](https://github.com/projectdiscovery/ratelimit/compare/v0.0.6...v0.0.8) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/ratelimit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 069e99503..38df63047 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -11,7 +11,7 @@ require ( github.com/projectdiscovery/dnsx v1.1.4 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.10 - github.com/projectdiscovery/ratelimit v0.0.6 + github.com/projectdiscovery/ratelimit v0.0.8 github.com/projectdiscovery/utils v0.0.32 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.8.3 diff --git a/v2/go.sum b/v2/go.sum index 937ffa31d..0d08d51b7 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -120,8 +120,8 @@ github.com/projectdiscovery/goflags v0.1.8 h1:Urhm2Isq2BdRt8h4h062lHKYXO65RHRjGT github.com/projectdiscovery/goflags v0.1.8/go.mod h1:Yxi9tclgwGczzDU65ntrwaIql5cXeTvW5j2WxFuF+Jk= github.com/projectdiscovery/gologger v1.1.10 h1:XNRdtzLTdxiFGuK9gutoL752mykzXDoii4P2yDovqck= github.com/projectdiscovery/gologger v1.1.10/go.mod h1:VqANHK7qcEq3i6/vV5HNWwdyv2aFPSrlaVDU4Ogrc6U= -github.com/projectdiscovery/ratelimit v0.0.6 h1:SAD2ArdT9F8NmbkAIZpl7DjNnbiXdUQLnMZt5dbVmZ0= -github.com/projectdiscovery/ratelimit v0.0.6/go.mod h1:WFL6gIggPLTwYwDbxqQODuWrz/lcMP2E5ofKSAz3YwI= +github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= +github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= github.com/projectdiscovery/retryabledns v1.0.23 h1:8X6HECevl6n7K7kxqfHLTcsCVXDdz9HR+1hS6fto7sw= github.com/projectdiscovery/retryabledns v1.0.23/go.mod h1:tj8BEe0jWCB4m3aAhHq4S2dqfpod0h+BiCZmXkXAGac= github.com/projectdiscovery/retryablehttp-go v1.0.15 h1:kP9x9f++QimRwb8ABqnI1dhEymvnZXS2Wp2Zs4rWk/c= From acde91c42aec9310e918bb5bb752fe672dc4f014 Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Thu, 25 May 2023 12:22:23 +0300 Subject: [PATCH 08/15] Extract subdomains in case of 404 (#845) * output conditional result based on result count * Revert "output conditional result based on result count" This reverts commit 3741069f0d4f1365fb14a86baa86c6369f8f6d93. * try extracting subdomains from body in case of 404 * adding doc context to 404 page --------- Co-authored-by: Sandeep Singh Co-authored-by: Mzack9999 --- v2/pkg/subscraping/sources/digitorus/digitorus.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/digitorus/digitorus.go b/v2/pkg/subscraping/sources/digitorus/digitorus.go index 86d6798ab..7d578072d 100644 --- a/v2/pkg/subscraping/sources/digitorus/digitorus.go +++ b/v2/pkg/subscraping/sources/digitorus/digitorus.go @@ -5,6 +5,7 @@ import ( "bufio" "context" "fmt" + "net/http" "strings" "time" @@ -31,7 +32,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se }(time.Now()) resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://certificatedetails.com/%s", domain)) - if err != nil { + // the 404 page still contains around 100 subdomains - https://github.com/projectdiscovery/subfinder/issues/774 + if err != nil && resp.StatusCode != http.StatusNotFound { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) From 0a25894cba1ea1036ba356f4647a8a4b1a55bf95 Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Sat, 27 May 2023 22:34:15 +0530 Subject: [PATCH 09/15] Implement Close method for session (#846) --- v2/pkg/passive/passive.go | 1 + v2/pkg/subscraping/agent.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index fe41971fb..826228146 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -28,6 +28,7 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p } return } + defer session.Close() ctx, cancel := context.WithTimeout(ctx, maxEnumTime) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 5b985c87a..dd4e3f0c8 100644 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -123,6 +123,12 @@ func (s *Session) DiscardHTTPResponse(response *http.Response) { } } +// Close the session +func (s *Session) Close() { + s.RateLimiter.Stop() + s.Client.CloseIdleConnections() +} + func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { response, err := client.Do(request) if err != nil { From 8abb0c380d2beed97b6f5b11b7e71ec783281f0a Mon Sep 17 00:00:00 2001 From: Shubham Rasal Date: Sat, 27 May 2023 22:37:19 +0530 Subject: [PATCH 10/15] Update the regex to match domain names case in-sensitive (#837) --- v2/pkg/subscraping/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/utils.go b/v2/pkg/subscraping/utils.go index 0c4468b73..7b2b6e4c6 100644 --- a/v2/pkg/subscraping/utils.go +++ b/v2/pkg/subscraping/utils.go @@ -23,7 +23,7 @@ func init() { func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) { subdomainExtractorMutex.Lock() defer subdomainExtractorMutex.Unlock() - extractor, err := regexp.Compile(`[a-zA-Z0-9\*_.-]+\.` + domain) + extractor, err := regexp.Compile(`(?i)[a-zA-Z0-9\*_.-]+\.` + domain) if err != nil { return nil, err } From 963c6515aca73b69703601475336f89e1f838b18 Mon Sep 17 00:00:00 2001 From: n0kovo Date: Tue, 30 May 2023 21:08:44 +0200 Subject: [PATCH 11/15] Fix #747 (#849) Reference: https://docs.securitytrails.com/reference/domain-subdomains Co-authored-by: Sandeep Singh --- .../subscraping/sources/securitytrails/securitytrails.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go index c8c6ca88c..42669b381 100644 --- a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -4,7 +4,6 @@ package securitytrails import ( "context" "fmt" - "strings" "time" jsoniter "github.com/json-iterator/go" @@ -63,12 +62,6 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se resp.Body.Close() for _, subdomain := range securityTrailsResponse.Subdomains { - if strings.HasSuffix(subdomain, ".") { - subdomain += domain - } else { - subdomain = subdomain + "." + domain - } - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } From cea3df93b9f44df0094c778352dbd6a90953dbe5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 00:41:11 +0530 Subject: [PATCH 12/15] chore(deps): bump github.com/projectdiscovery/utils in /v2 (#853) Bumps [github.com/projectdiscovery/utils](https://github.com/projectdiscovery/utils) from 0.0.32 to 0.0.34. - [Release notes](https://github.com/projectdiscovery/utils/releases) - [Commits](https://github.com/projectdiscovery/utils/compare/v0.0.32...v0.0.34) --- updated-dependencies: - dependency-name: github.com/projectdiscovery/utils dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 4 ++-- v2/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 38df63047..929f8dc69 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.10 github.com/projectdiscovery/ratelimit v0.0.8 - github.com/projectdiscovery/utils v0.0.32 + github.com/projectdiscovery/utils v0.0.34 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.8.3 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -44,7 +44,7 @@ require ( github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mholt/archiver v3.1.1+incompatible // indirect - github.com/microcosm-cc/bluemonday v1.0.23 // indirect + github.com/microcosm-cc/bluemonday v1.0.24 // indirect github.com/minio/selfupdate v0.6.0 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.1 // indirect diff --git a/v2/go.sum b/v2/go.sum index 0d08d51b7..388531ca0 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -80,8 +80,8 @@ github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mholt/archiver v3.1.1+incompatible h1:1dCVxuqs0dJseYEhi5pl7MYPH9zDa1wBi7mF09cbNkU= github.com/mholt/archiver v3.1.1+incompatible/go.mod h1:Dh2dOXnSdiLxRiPoVfIr/fI1TwETms9B8CTWfeh7ROU= github.com/microcosm-cc/bluemonday v1.0.21/go.mod h1:ytNkv4RrDrLJ2pqlsSI46O6IVXmZOBBD4SaJyDwwTkM= -github.com/microcosm-cc/bluemonday v1.0.23 h1:SMZe2IGa0NuHvnVNAZ+6B38gsTbi5e4sViiWJyDDqFY= -github.com/microcosm-cc/bluemonday v1.0.23/go.mod h1:mN70sk7UkkF8TUr2IGBpNN0jAgStuPzlK76QuruE/z4= +github.com/microcosm-cc/bluemonday v1.0.24 h1:NGQoPtwGVcbGkKfvyYk1yRqknzBuoMiUrO6R7uFTPlw= +github.com/microcosm-cc/bluemonday v1.0.24/go.mod h1:ArQySAMps0790cHSkdPEJ7bGkF2VePWH773hsJNSHf8= github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= @@ -126,8 +126,8 @@ github.com/projectdiscovery/retryabledns v1.0.23 h1:8X6HECevl6n7K7kxqfHLTcsCVXDd github.com/projectdiscovery/retryabledns v1.0.23/go.mod h1:tj8BEe0jWCB4m3aAhHq4S2dqfpod0h+BiCZmXkXAGac= github.com/projectdiscovery/retryablehttp-go v1.0.15 h1:kP9x9f++QimRwb8ABqnI1dhEymvnZXS2Wp2Zs4rWk/c= github.com/projectdiscovery/retryablehttp-go v1.0.15/go.mod h1:+OzSFUv3sQcPt+MgbNx6X/Q3ESxqPUQSphqG5kxoIgI= -github.com/projectdiscovery/utils v0.0.32 h1:TjUxFmRG9hiV6jauYsnIRiy08lTgGHGi8avstedNGXw= -github.com/projectdiscovery/utils v0.0.32/go.mod h1:SaOpcZ2dJ47NE3t4R/YC2XpUyRZC6v5k8sj2TFro6+k= +github.com/projectdiscovery/utils v0.0.34 h1:SnNJT2WeSoWhpnonVPb8YdbJiWtnll9U3SM3sfdMcnU= +github.com/projectdiscovery/utils v0.0.34/go.mod h1:cUXbXdUIaRu+bpot7IlCCqPEp0tyB9M1VHMtGA0eJzE= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= From 3385c7551a0cb898edc59e0adcb05bb756c5c965 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 31 May 2023 00:41:20 +0530 Subject: [PATCH 13/15] chore(deps): bump github.com/stretchr/testify from 1.8.3 to 1.8.4 in /v2 (#859) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.3 to 1.8.4. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.3...v1.8.4) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- v2/go.mod | 2 +- v2/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 929f8dc69..5bae618c8 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -14,7 +14,7 @@ require ( github.com/projectdiscovery/ratelimit v0.0.8 github.com/projectdiscovery/utils v0.0.34 github.com/rs/xid v1.5.0 - github.com/stretchr/testify v1.8.3 + github.com/stretchr/testify v1.8.4 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 golang.org/x/exp v0.0.0-20230420155640-133eef4313cb gopkg.in/yaml.v3 v3.0.1 diff --git a/v2/go.sum b/v2/go.sum index 388531ca0..9cc6f2a15 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -141,8 +141,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= From 8d75460c41a0bc7bacd2582ca838ab5558509b0e Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Tue, 30 May 2023 22:32:33 +0300 Subject: [PATCH 14/15] Test sources require an API key (#852) * test providers require key * remove redundant tests * ignore failing(often) test * added crtsh to ignore list --------- Co-authored-by: sandeep <8293321+ehsandeep@users.noreply.github.com> --- .github/workflows/build-test.yml | 34 ++++++-- v2/cmd/integration-test/integration-test.go | 86 --------------------- v2/cmd/integration-test/run.sh | 19 ----- v2/cmd/integration-test/source-test.go | 33 -------- v2/pkg/passive/sources_w_auth_test.go | 61 +++++++++++++++ v2/pkg/passive/sources_wo_auth_test.go | 13 ++-- 6 files changed, 96 insertions(+), 150 deletions(-) delete mode 100644 v2/cmd/integration-test/integration-test.go delete mode 100755 v2/cmd/integration-test/run.sh delete mode 100644 v2/cmd/integration-test/source-test.go create mode 100644 v2/pkg/passive/sources_w_auth_test.go diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index cea8627ed..1edb0610a 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -28,19 +28,39 @@ jobs: working-directory: v2/ - name: Run tests + env: + BEVIGIL_API_KEY: ${{secrets.BEVIGIL_API_KEY}} + BINARYEDGE_API_KEY: ${{secrets.BINARYEDGE_API_KEY}} + BUFFEROVER_API_KEY: ${{secrets.BUFFEROVER_API_KEY}} + C99_API_KEY: ${{secrets.C99_API_KEY}} + CENSYS_API_KEY: ${{secrets.CENSYS_API_KEY}} + CERTSPOTTER_API_KEY: ${{secrets.CERTSPOTTER_API_KEY}} + CHAOS_API_KEY: ${{secrets.CHAOS_API_KEY}} + CHINAZ_API_KEY: ${{secrets.CHINAZ_API_KEY}} + DNSDB_API_KEY: ${{secrets.DNSDB_API_KEY}} + DNSREPO_API_KEY: ${{secrets.DNSREPO_API_KEY}} + FOFA_API_KEY: ${{secrets.FOFA_API_KEY}} + FULLHUNT_API_KEY: ${{secrets.FULLHUNT_API_KEY}} + GITHUB_API_KEY: ${{secrets.GITHUB_API_KEY}} + HUNTER_API_KEY: ${{secrets.HUNTER_API_KEY}} + INTELX_API_KEY: ${{secrets.INTELX_API_KEY}} + LEAKIX_API_KEY: ${{secrets.LEAKIX_API_KEY}} + PASSIVETOTAL_API_KEY: ${{secrets.PASSIVETOTAL_API_KEY}} + QUAKE_API_KEY: ${{secrets.QUAKE_API_KEY}} + ROBTEX_API_KEY: ${{secrets.ROBTEX_API_KEY}} + SECURITYTRAILS_API_KEY: ${{secrets.SECURITYTRAILS_API_KEY}} + SHODAN_API_KEY: ${{secrets.SHODAN_API_KEY}} + THREATBOOK_API_KEY: ${{secrets.THREATBOOK_API_KEY}} + VIRUSTOTAL_API_KEY: ${{secrets.VIRUSTOTAL_API_KEY}} + WHOISXMLAPI_API_KEY: ${{secrets.WHOISXMLAPI_API_KEY}} + ZOOMEYE_API_KEY: ${{secrets.ZOOMEYE_API_KEY}} + ZOOMEYEAPI_API_KEY: ${{secrets.ZOOMEYEAPI_API_KEY}} uses: nick-invision/retry@v2 with: timeout_seconds: 360 max_attempts: 3 command: cd v2; go test ./... -v - - name: Integration Tests - env: - GH_ACTION: true - DNSREPO_API_KEY: ${{secrets.DNSREPO_API}} - run: bash run.sh - working-directory: v2/cmd/integration-test/ - - name: Race Condition Tests run: go build -race ./... working-directory: v2/ \ No newline at end of file diff --git a/v2/cmd/integration-test/integration-test.go b/v2/cmd/integration-test/integration-test.go deleted file mode 100644 index 608eceaf2..000000000 --- a/v2/cmd/integration-test/integration-test.go +++ /dev/null @@ -1,86 +0,0 @@ -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/logrusorgru/aurora" - - "github.com/projectdiscovery/subfinder/v2/pkg/testutils" -) - -var ( - debug = os.Getenv("DEBUG") == "true" - githubAction = os.Getenv("GH_ACTION") == "true" - customTests = os.Getenv("TESTS") - - success = aurora.Green("[✓]").String() - failed = aurora.Red("[✘]").String() - - sourceTests = map[string]testutils.TestCase{ - "dnsrepo": dnsrepoTestcases{}, - } -) - -func main() { - failedTestCases := runTests(toMap(toSlice(customTests))) - - if len(failedTestCases) > 0 { - if githubAction { - debug = true - fmt.Println("::group::Failed integration tests in debug mode") - _ = runTests(failedTestCases) - fmt.Println("::endgroup::") - } - os.Exit(1) - } -} - -func runTests(customTestCases map[string]struct{}) map[string]struct{} { - failedTestCases := map[string]struct{}{} - - for source, testCase := range sourceTests { - if len(customTestCases) == 0 { - fmt.Printf("Running test cases for %q source\n", aurora.Blue(source)) - } - if err, failedTemplatePath := execute(source, testCase); err != nil { - failedTestCases[failedTemplatePath] = struct{}{} - } - } - return failedTestCases -} - -func execute(source string, testCase testutils.TestCase) (error, string) { - if err := testCase.Execute(); err != nil { - _, _ = fmt.Fprintf(os.Stderr, "%s Test \"%s\" failed: %s\n", failed, source, err) - return err, source - } - - fmt.Printf("%s Test \"%s\" passed!\n", success, source) - return nil, "" -} - -func expectResultsGreaterThanCount(results []string, expectedNumber int) error { - if len(results) > expectedNumber { - return nil - } - return fmt.Errorf("incorrect number of results: expected a result greater than %d,but got %d", expectedNumber, len(results)) -} -func toSlice(value string) []string { - if strings.TrimSpace(value) == "" { - return []string{} - } - - return strings.Split(value, ",") -} - -func toMap(slice []string) map[string]struct{} { - result := make(map[string]struct{}, len(slice)) - for _, value := range slice { - if _, ok := result[value]; !ok { - result[value] = struct{}{} - } - } - return result -} diff --git a/v2/cmd/integration-test/run.sh b/v2/cmd/integration-test/run.sh deleted file mode 100755 index 2d826de24..000000000 --- a/v2/cmd/integration-test/run.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -echo "::task~> Clean up & Build binaries files" -rm integration-test subfinder 2>/dev/null -cd ../subfinder -go build -mv subfinder ../integration-test/subfinder -cd ../integration-test -go build -echo "::done::" -echo "::task~> Run integration test" -./integration-test -echo "::done::" -if [ $? -eq 0 ] -then - exit 0 -else - exit 1 -fi diff --git a/v2/cmd/integration-test/source-test.go b/v2/cmd/integration-test/source-test.go deleted file mode 100644 index 3b5e4ca0a..000000000 --- a/v2/cmd/integration-test/source-test.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "os" - - "github.com/projectdiscovery/subfinder/v2/pkg/testutils" -) - -type dnsrepoTestcases struct{} - -func (h dnsrepoTestcases) Execute() error { - token := os.Getenv("DNSREPO_API_KEY") - if token == "" { - return errors.New("missing dns repo api key") - } - dnsToken := fmt.Sprintf(`dnsrepo: [%s]`, token) - file, err := os.CreateTemp("", "provider.yaml") - if err != nil { - return err - } - defer os.RemoveAll(file.Name()) - _, err = file.WriteString(dnsToken) - if err != nil { - return err - } - results, err := testutils.RunSubfinderAndGetResults(debug, "hackerone.com", "-s", "dnsrepo", "-provider-config", file.Name()) - if err != nil { - return err - } - return expectResultsGreaterThanCount(results, 0) -} diff --git a/v2/pkg/passive/sources_w_auth_test.go b/v2/pkg/passive/sources_w_auth_test.go new file mode 100644 index 000000000..452bde6f5 --- /dev/null +++ b/v2/pkg/passive/sources_w_auth_test.go @@ -0,0 +1,61 @@ +package passive + +import ( + "context" + "fmt" + "os" + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +func TestSourcesWithKeys(t *testing.T) { + domain := "hackerone.com" + timeout := 60 + + gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) + + ctx := context.Background() + session, err := subscraping.NewSession(domain, "", 0, timeout) + assert.Nil(t, err) + + var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} + + for _, source := range AllSources { + if !source.NeedsKey() { + continue + } + + apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))) + if apiKey == "" { + fmt.Printf("Skipping %s as no API key is provided\n", source.Name()) + continue + } + source.AddApiKeys([]string{apiKey}) + + t.Run(source.Name(), func(t *testing.T) { + var results []subscraping.Result + + for result := range source.Run(ctx, domain, session) { + results = append(results, result) + + assert.Equal(t, source.Name(), result.Source, "wrong source name") + + if result.Type != subscraping.Error { + assert.True(t, strings.HasSuffix(strings.ToLower(result.Value), strings.ToLower(expected.Value)), + fmt.Sprintf("result(%s) is not subdomain of %s", strings.ToLower(result.Value), expected.Value)) + } else { + assert.Equal(t, reflect.TypeOf(expected.Error), reflect.TypeOf(result.Error), fmt.Sprintf("%s: %s", result.Source, result.Error)) + } + } + + assert.GreaterOrEqual(t, len(results), 1, fmt.Sprintf("No result found for %s", source.Name())) + }) + } +} diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index 14f135b14..d603ae802 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "golang.org/x/exp/slices" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" @@ -15,6 +16,12 @@ import ( ) func TestSourcesWithoutKeys(t *testing.T) { + ignoredSources := []string{ + "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ + "riddler", // Fails with 403: There might be too much traffic or a configuration error + "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. + } + domain := "hackerone.com" timeout := 60 @@ -27,14 +34,10 @@ func TestSourcesWithoutKeys(t *testing.T) { var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} for _, source := range AllSources { - if source.NeedsKey() { + if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) { continue } - if source.Name() == "commoncrawl" { - continue // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ - } - t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result From 3ffcdc1cf850a68eed5d351c521850f1c3fc9f1f Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Wed, 31 May 2023 01:12:37 +0530 Subject: [PATCH 15/15] version update --- v2/pkg/runner/banners.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 087e83e3f..136975162 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -18,7 +18,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.5.8` +const version = `v2.5.9` // showBanner is used to show the banner to the user func showBanner() {