From 8e322dfb79c43cc078201ade94238d8c7191dfe7 Mon Sep 17 00:00:00 2001 From: Arthur Reading Date: Sat, 31 Jul 2021 22:17:36 -0700 Subject: [PATCH 01/42] Fix method to denu connections in docs. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b9e7941ca..4dba7ecdf 100644 --- a/README.md +++ b/README.md @@ -136,12 +136,12 @@ For example: ```go // This rejects the HTTPS request to *.reddit.com during HTTP CONNECT phase -proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("reddit.*:443$"))).HandleConnect(goproxy.RejectConnect) +proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("reddit.*:443$"))).HandleConnect(goproxy.AlwaysReject) // This will NOT reject the HTTPS request with URL ending with gif, due to the fact that proxy // only got the URL.Hostname and URL.Port during the HTTP CONNECT phase if the scheme is HTTPS, which is // quiet common these days. -proxy.OnRequest(goproxy.UrlMatches(regexp.MustCompile(`.*gif$`))).HandleConnect(goproxy.RejectConnect) +proxy.OnRequest(goproxy.UrlMatches(regexp.MustCompile(`.*gif$`))).HandleConnect(goproxy.AlwaysReject) // The correct way to manipulate the HTTP request using URL.Path as condition is: proxy.OnRequest(goproxy.UrlMatches(regexp.MustCompile(`.*gif$`))).Do(YourReqHandlerFunc()) From d06c3be7c11b750d8cd76d0f094936e07cac0ada Mon Sep 17 00:00:00 2001 From: suxianbaozi Date: Fri, 12 Nov 2021 15:52:30 +0800 Subject: [PATCH 02/42] sample certstorage --- examples/goproxy-certstorage/README.md | 5 ++++ examples/goproxy-certstorage/main.go | 26 +++++++++++++++++++ examples/goproxy-certstorage/storage.go | 34 +++++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 examples/goproxy-certstorage/README.md create mode 100644 examples/goproxy-certstorage/main.go create mode 100644 examples/goproxy-certstorage/storage.go diff --git a/examples/goproxy-certstorage/README.md b/examples/goproxy-certstorage/README.md new file mode 100644 index 000000000..b8213681a --- /dev/null +++ b/examples/goproxy-certstorage/README.md @@ -0,0 +1,5 @@ +# certstorage + +## use certstorage + +## this could make https proxy faster diff --git a/examples/goproxy-certstorage/main.go b/examples/goproxy-certstorage/main.go new file mode 100644 index 000000000..a010a1438 --- /dev/null +++ b/examples/goproxy-certstorage/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "flag" + "github.com/elazarl/goproxy" + "log" + "net/http" +) + +func main() { + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("addr", ":8080", "proxy listen address") + flag.Parse() + + proxy := goproxy.NewProxyHttpServer() + proxy.CertStore = NewCertStorage() //设置storage + + proxy.Verbose = *verbose + + proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) + proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + log.Println(req.URL.String()) + return req, nil + }) + log.Fatal(http.ListenAndServe(*addr, proxy)) +} diff --git a/examples/goproxy-certstorage/storage.go b/examples/goproxy-certstorage/storage.go new file mode 100644 index 000000000..5593d8546 --- /dev/null +++ b/examples/goproxy-certstorage/storage.go @@ -0,0 +1,34 @@ +package main + +import ( + "crypto/tls" + "sync" +) + +type CertStorage struct { + certs sync.Map +} + +func (tcs *CertStorage) Fetch(hostname string, gen func() (*tls.Certificate, error)) (*tls.Certificate, error) { + var cert tls.Certificate + icert, ok := tcs.certs.Load(hostname) + if ok { + cert = icert.(tls.Certificate) + } else { + certp, err := gen() + if err != nil { + return nil, err + } + // store as concrete implementation + cert = *certp + tcs.certs.Store(hostname, cert) + } + return &cert, nil +} + +func NewCertStorage() *CertStorage { + tcs := &CertStorage{} + tcs.certs = sync.Map{} + + return tcs +} From 6723c80d63af339eb908df08e9fb0b6006c8aa49 Mon Sep 17 00:00:00 2001 From: hgsgtk Date: Sat, 15 Jan 2022 11:36:37 +0900 Subject: [PATCH 03/42] docs: fix the term 'precanned' to 'pre-canned' --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4dba7ecdf..495afc2d4 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ a `ReqCondition` accepting only requests directed to "www.reddit.com". `DoFunc` will receive a function that will preprocess the request. We can change the request, or return a response. If the time is between 8:00am and 17:00pm, we will reject the request, and -return a precanned text response saying "do not waste your time". +return a pre-canned text response saying "do not waste your time". See additional examples in the examples directory. From 94fc67bc9d8f3c8aafd8c54f7fd638c7a165e341 Mon Sep 17 00:00:00 2001 From: hgsgtk Date: Sat, 15 Jan 2022 11:56:02 +0900 Subject: [PATCH 04/42] docs: fix sentences in README --- examples/goproxy-transparent/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/goproxy-transparent/README.md b/examples/goproxy-transparent/README.md index 7edb0989b..829eb2a96 100644 --- a/examples/goproxy-transparent/README.md +++ b/examples/goproxy-transparent/README.md @@ -1,10 +1,10 @@ # Transparent Proxy -This transparent example in goproxy is meant to show how to transparenty proxy and hijack all http and https connections while doing a man-in-the-middle to the TLS session. It requires that goproxy sees all the packets traversing out to the internet. Linux iptables rules deal with changing the source/destination IPs to act transparently, but you do need to setup your network configuration so that goproxy is a mandatory stop on the outgoing route. Primarily you can do this by placing the proxy inline. goproxy does not have any WCCP support itself; patches welcome. +This transparent example in goproxy is meant to show how to transparent proxy and hijack all http and https connections while doing a man-in-the-middle to the TLS session. It requires that goproxy sees all the packets traversing out to the internet. Linux iptables rules deal with changing the source/destination IPs to act transparently, but you do need to set up your network configuration so that goproxy is a mandatory stop on the outgoing route. Primarily you can do this by placing the proxy inline. goproxy does not have any WCCP support itself; patches are welcome. ## Why not explicit? -Transparent proxies are more difficult to maintain and setup from a server side, but they require no configuration on the client(s) which could be in unmanaged systems or systems that don't support a proxy configuration. See the [eavesdropper example](https://github.com/elazarl/goproxy/blob/master/examples/goproxy-eavesdropper/main.go) if you want to see an explicit proxy example. +Transparent proxies are more difficult to maintain and set up from a server side, but they require no configuration on the client(s) which could be in unmanaged systems or systems that don't support a proxy configuration. See the [eavesdropper example](https://github.com/elazarl/goproxy/blob/master/examples/goproxy-eavesdropper/main.go) if you want to see an explicit proxy example. ## Potential Issues @@ -14,4 +14,4 @@ If you're routing table allows for it, an explicit http request to goproxy will ## Routing Rules -Example routing rules are included in [proxy.sh](https://github.com/elazarl/goproxy/blob/master/examples/goproxy-transparent/proxy.sh) but are best when setup using your distribution's configuration. +Example routing rules are included in [proxy.sh](https://github.com/elazarl/goproxy/blob/master/examples/goproxy-transparent/proxy.sh) but are best when set up using your distribution's configuration. From 0e85d07094194f69d072b5dbc955ef2ac2b2521f Mon Sep 17 00:00:00 2001 From: hgsgtk Date: Sat, 15 Jan 2022 11:57:02 +0900 Subject: [PATCH 05/42] refactor: use DialContext() instead of Dial() --- examples/goproxy-transparent/transparent.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/goproxy-transparent/transparent.go b/examples/goproxy-transparent/transparent.go index e3ece1d7c..d4dc687a3 100644 --- a/examples/goproxy-transparent/transparent.go +++ b/examples/goproxy-transparent/transparent.go @@ -3,6 +3,7 @@ package main import ( "bufio" "bytes" + "context" "flag" "fmt" "log" @@ -54,7 +55,8 @@ func main() { client.Close() }() clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client)) - remote, err := connectDial(proxy, "tcp", req.URL.Host) + + remote, err := connectDial(req.Context(), proxy, "tcp", req.URL.Host) orPanic(err) remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote)) for { @@ -110,17 +112,17 @@ func main() { } // copied/converted from https.go -func dial(proxy *goproxy.ProxyHttpServer, network, addr string) (c net.Conn, err error) { - if proxy.Tr.Dial != nil { - return proxy.Tr.Dial(network, addr) +func dial(ctx context.Context, proxy *goproxy.ProxyHttpServer, network, addr string) (c net.Conn, err error) { + if proxy.Tr.DialContext != nil { + return proxy.Tr.DialContext(ctx, network, addr) } return net.Dial(network, addr) } // copied/converted from https.go -func connectDial(proxy *goproxy.ProxyHttpServer, network, addr string) (c net.Conn, err error) { +func connectDial(ctx context.Context, proxy *goproxy.ProxyHttpServer, network, addr string) (c net.Conn, err error) { if proxy.ConnectDial == nil { - return dial(proxy, network, addr) + return dial(ctx, proxy, network, addr) } return proxy.ConnectDial(network, addr) } From adb46da277acd7aea06aeb8b5a21ec6bef7fb247 Mon Sep 17 00:00:00 2001 From: hgsgtk Date: Sat, 15 Jan 2022 12:16:49 +0900 Subject: [PATCH 06/42] refactor: use DialContext() instead of Dial() --- examples/goproxy-transparent/transparent.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/goproxy-transparent/transparent.go b/examples/goproxy-transparent/transparent.go index d4dc687a3..b3d03645b 100644 --- a/examples/goproxy-transparent/transparent.go +++ b/examples/goproxy-transparent/transparent.go @@ -116,7 +116,8 @@ func dial(ctx context.Context, proxy *goproxy.ProxyHttpServer, network, addr str if proxy.Tr.DialContext != nil { return proxy.Tr.DialContext(ctx, network, addr) } - return net.Dial(network, addr) + var d net.Dialer + return d.DialContext(ctx, network, addr) } // copied/converted from https.go From f5c0d0953e107ea8fe29bc45bbf4d4aacbb45697 Mon Sep 17 00:00:00 2001 From: duke Date: Wed, 16 Mar 2022 17:02:57 +0800 Subject: [PATCH 07/42] Fix the signer signing an invalid certificate when the hostname is an ipv6 address --- https.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/https.go b/https.go index 6fcf17a91..24be8f581 100644 --- a/https.go +++ b/https.go @@ -47,9 +47,25 @@ type ConnectAction struct { } func stripPort(s string) string { - ix := strings.IndexRune(s, ':') - if ix == -1 { - return s + var ix int + if strings.Contains(s,"[") && strings.Contains(s,"]") { + //ipv6 : for example : [2606:4700:4700::1111]:443 + + //strip '[' and ']' + s = strings.ReplaceAll(s,"[","") + s = strings.ReplaceAll(s,"]","") + + ix = strings.LastIndexAny(s,":") + if ix == -1 { + return s + } + } else { + //ipv4 + ix = strings.IndexRune(s, ':') + if ix == -1 { + return s + } + } return s[:ix] } From 894aeddb713ece400b68656e1ca23f9488b9ca86 Mon Sep 17 00:00:00 2001 From: cuishuang Date: Mon, 28 Mar 2022 19:55:03 +0800 Subject: [PATCH 08/42] fix some typos Signed-off-by: cuishuang --- doc.go | 4 ++-- ext/auth/basic_test.go | 2 +- regretable/regretreader_test.go | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc.go b/doc.go index 6f92fd203..6f44317b9 100644 --- a/doc.go +++ b/doc.go @@ -3,7 +3,7 @@ Package goproxy provides a customizable HTTP proxy, supporting hijacking HTTPS connection. The intent of the proxy, is to be usable with reasonable amount of traffic -yet, customizable and programable. +yet, customizable and programmable. The proxy itself is simply an `net/http` handler. @@ -63,7 +63,7 @@ Finally, we have convenience function to throw a quick response return goproxy.NewResponse(ctx.Req, goproxy.ContentTypeText, http.StatusForbidden, "Can't see response with X-GoProxy header!") }) -we close the body of the original repsonse, and return a new 403 response with a short message. +we close the body of the original response, and return a new 403 response with a short message. Example use cases: diff --git a/ext/auth/basic_test.go b/ext/auth/basic_test.go index 792d789bc..118e70ed3 100644 --- a/ext/auth/basic_test.go +++ b/ext/auth/basic_test.go @@ -147,7 +147,7 @@ func TestWithBrowser(t *testing.T) { // to test, run with // $ go test -run TestWithBrowser -- server // configure a browser to use the printed proxy address, use the proxy - // and exit with Ctrl-C. It will throw error if your haven't acutally used the proxy + // and exit with Ctrl-C. It will throw error if your haven't actually used the proxy if os.Args[len(os.Args)-1] != "server" { return } diff --git a/regretable/regretreader_test.go b/regretable/regretreader_test.go index 559697479..bc631b455 100644 --- a/regretable/regretreader_test.go +++ b/regretable/regretreader_test.go @@ -21,7 +21,7 @@ func TestRegretableReader(t *testing.T) { s, _ := ioutil.ReadAll(mb) if string(s) != word { - t.Errorf("Uncommited read is gone, [%d,%d] actual '%v' expected '%v'\n", len(s), len(word), string(s), word) + t.Errorf("Uncommitted read is gone, [%d,%d] actual '%v' expected '%v'\n", len(s), len(word), string(s), word) } } @@ -37,7 +37,7 @@ func TestRegretableEmptyRead(t *testing.T) { s, err := ioutil.ReadAll(mb) if string(s) != word { - t.Error("Uncommited read is gone, actual:", string(s), "expected:", word, "err:", err) + t.Error("Uncommitted read is gone, actual:", string(s), "expected:", word, "err:", err) } } @@ -57,7 +57,7 @@ func TestRegretableAlsoEmptyRead(t *testing.T) { s, _ := ioutil.ReadAll(mb) if string(s) != word { - t.Error("Uncommited read is gone", string(s), "expected", word) + t.Error("Uncommitted read is gone", string(s), "expected", word) } } @@ -73,7 +73,7 @@ func TestRegretableRegretBeforeRead(t *testing.T) { s, err := ioutil.ReadAll(mb) if string(s) != "678" { - t.Error("Uncommited read is gone", string(s), len(string(s)), "expected", "678", len("678"), "err:", err) + t.Error("Uncommitted read is gone", string(s), len(string(s)), "expected", "678", len("678"), "err:", err) } } @@ -89,7 +89,7 @@ func TestRegretableFullRead(t *testing.T) { s, _ := ioutil.ReadAll(mb) if string(s) != word { - t.Error("Uncommited read is gone", string(s), len(string(s)), "expected", word, len(word)) + t.Error("Uncommitted read is gone", string(s), len(string(s)), "expected", word, len(word)) } } From a53172b9392e8b19b19cf325ffbd1da1007c2708 Mon Sep 17 00:00:00 2001 From: Elazar Leibovich Date: Sun, 3 Apr 2022 07:25:43 +0300 Subject: [PATCH 09/42] gofmt --- https.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/https.go b/https.go index 24be8f581..bee26a27a 100644 --- a/https.go +++ b/https.go @@ -48,14 +48,14 @@ type ConnectAction struct { func stripPort(s string) string { var ix int - if strings.Contains(s,"[") && strings.Contains(s,"]") { + if strings.Contains(s, "[") && strings.Contains(s, "]") { //ipv6 : for example : [2606:4700:4700::1111]:443 //strip '[' and ']' - s = strings.ReplaceAll(s,"[","") - s = strings.ReplaceAll(s,"]","") + s = strings.ReplaceAll(s, "[", "") + s = strings.ReplaceAll(s, "]", "") - ix = strings.LastIndexAny(s,":") + ix = strings.LastIndexAny(s, ":") if ix == -1 { return s } From 99af03e6e0da7117d4369f6612b901541792d2aa Mon Sep 17 00:00:00 2001 From: hgsgtk Date: Sat, 16 Apr 2022 10:44:10 +0900 Subject: [PATCH 10/42] fix: return `Connection established` instead `OK` This commit changes to return `200 Connection established` instead `200 OK`. ref: https://datatracker.ietf.org/doc/html/draft-luotonen-web-proxy-tunneling-01#section-3.2 --- https.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/https.go b/https.go index bee26a27a..5be74139e 100644 --- a/https.go +++ b/https.go @@ -128,7 +128,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request return } ctx.Logf("Accepting CONNECT to %s", host) - proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) + proxyClient.Write([]byte("HTTP/1.0 200 Connection established\r\n\r\n")) targetTCP, targetOK := targetSiteCon.(halfClosable) proxyClientTCP, clientOK := proxyClient.(halfClosable) From 416226498f94c6c6b3ee6bf14d7b89d6d91b02d1 Mon Sep 17 00:00:00 2001 From: Wilson Tian Date: Sat, 16 Apr 2022 10:20:31 +0800 Subject: [PATCH 11/42] preserve Content-Length in header of HEAD response --- https.go | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/https.go b/https.go index 5be74139e..b6f68dd7f 100644 --- a/https.go +++ b/https.go @@ -265,10 +265,15 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request ctx.Warnf("Cannot write TLS response HTTP status from mitm'd client: %v", err) return } - // Since we don't know the length of resp, return chunked encoded response - // TODO: use a more reasonable scheme - resp.Header.Del("Content-Length") - resp.Header.Set("Transfer-Encoding", "chunked") + + if resp.Request.Method == "HEAD" { + // don't change Content-Length for HEAD request + } else { + // Since we don't know the length of resp, return chunked encoded response + // TODO: use a more reasonable scheme + resp.Header.Del("Content-Length") + resp.Header.Set("Transfer-Encoding", "chunked") + } // Force connection close otherwise chrome will keep CONNECT tunnel open forever resp.Header.Set("Connection", "close") if err := resp.Header.Write(rawClientTls); err != nil { @@ -279,18 +284,23 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request ctx.Warnf("Cannot write TLS response header end from mitm'd client: %v", err) return } - chunked := newChunkedWriter(rawClientTls) - if _, err := io.Copy(chunked, resp.Body); err != nil { - ctx.Warnf("Cannot write TLS response body from mitm'd client: %v", err) - return - } - if err := chunked.Close(); err != nil { - ctx.Warnf("Cannot write TLS chunked EOF from mitm'd client: %v", err) - return - } - if _, err = io.WriteString(rawClientTls, "\r\n"); err != nil { - ctx.Warnf("Cannot write TLS response chunked trailer from mitm'd client: %v", err) - return + + if resp.Request.Method == "HEAD" { + // Don't write out a response body for HEAD request + } else { + chunked := newChunkedWriter(rawClientTls) + if _, err := io.Copy(chunked, resp.Body); err != nil { + ctx.Warnf("Cannot write TLS response body from mitm'd client: %v", err) + return + } + if err := chunked.Close(); err != nil { + ctx.Warnf("Cannot write TLS chunked EOF from mitm'd client: %v", err) + return + } + if _, err = io.WriteString(rawClientTls, "\r\n"); err != nil { + ctx.Warnf("Cannot write TLS response chunked trailer from mitm'd client: %v", err) + return + } } } ctx.Logf("Exiting on EOF") From 8ea89ba92021a445673ddc24d6da7a4af7d75249 Mon Sep 17 00:00:00 2001 From: Dan Slov Date: Fri, 27 May 2022 13:12:56 -0700 Subject: [PATCH 12/42] Flush response body for server-side events --- proxy.go | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/proxy.go b/proxy.go index cac3f8891..5aeb20b10 100644 --- a/proxy.go +++ b/proxy.go @@ -107,6 +107,20 @@ func removeProxyHeaders(ctx *ProxyCtx, r *http.Request) { r.Header.Del("Connection") } +type flushWriter struct { + w io.Writer +} + +func (fw flushWriter) Write(p []byte) (int, error) { + n, err := fw.w.Write(p) + if f, ok := fw.w.(http.Flusher); ok { + // only flush if the Writer implements the Flusher interface. + f.Flush() + } + + return n, err +} + // Standard net/http function. Shouldn't be used directly, http.Serve will use it. func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { //r.Header["X-Forwarded-For"] = w.RemoteAddr() @@ -177,7 +191,13 @@ func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) } copyHeaders(w.Header(), resp.Header, proxy.KeepDestinationHeaders) w.WriteHeader(resp.StatusCode) - nr, err := io.Copy(w, resp.Body) + var copyWriter io.Writer = w + if w.Header().Get("content-type") == "text/event-stream" { + // server-side events, flush the buffered data to the client. + copyWriter = &flushWriter{w: w} + } + + nr, err := io.Copy(copyWriter, resp.Body) if err := resp.Body.Close(); err != nil { ctx.Warnf("Can't close response body %v", err) } From fbd10ff4f5a16de73dca5030fc12245548f76141 Mon Sep 17 00:00:00 2001 From: Winston Chen Date: Sun, 28 Aug 2022 10:09:14 +0800 Subject: [PATCH 13/42] Pass ctx to connectDial User-supplied ConnectDialWithReq can dial based on proxy requests --- https.go | 13 +++++++++---- proxy.go | 7 ++++--- websocket.go | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/https.go b/https.go index b6f68dd7f..5a601a800 100644 --- a/https.go +++ b/https.go @@ -77,10 +77,15 @@ func (proxy *ProxyHttpServer) dial(network, addr string) (c net.Conn, err error) return net.Dial(network, addr) } -func (proxy *ProxyHttpServer) connectDial(network, addr string) (c net.Conn, err error) { - if proxy.ConnectDial == nil { +func (proxy *ProxyHttpServer) connectDial(ctx *ProxyCtx, network, addr string) (c net.Conn, err error) { + if proxy.ConnectDialWithReq == nil && proxy.ConnectDial == nil { return proxy.dial(network, addr) } + + if proxy.ConnectDialWithReq != nil { + return proxy.ConnectDialWithReq(ctx.Req, network, addr) + } + return proxy.ConnectDial(network, addr) } @@ -122,7 +127,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request if !hasPort.MatchString(host) { host += ":80" } - targetSiteCon, err := proxy.connectDial("tcp", host) + targetSiteCon, err := proxy.connectDial(ctx, "tcp", host) if err != nil { httpError(proxyClient, ctx, err) return @@ -153,7 +158,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request case ConnectHTTPMitm: proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) ctx.Logf("Assuming CONNECT is plain HTTP tunneling, mitm proxying it") - targetSiteCon, err := proxy.connectDial("tcp", host) + targetSiteCon, err := proxy.connectDial(ctx, "tcp", host) if err != nil { ctx.Warnf("Error dialing to %s: %s", host, err.Error()) return diff --git a/proxy.go b/proxy.go index 5aeb20b10..fa5494c6a 100644 --- a/proxy.go +++ b/proxy.go @@ -28,9 +28,10 @@ type ProxyHttpServer struct { Tr *http.Transport // ConnectDial will be used to create TCP connections for CONNECT requests // if nil Tr.Dial will be used - ConnectDial func(network string, addr string) (net.Conn, error) - CertStore CertStorage - KeepHeader bool + ConnectDial func(network string, addr string) (net.Conn, error) + ConnectDialWithReq func(req *http.Request, network string, addr string) (net.Conn, error) + CertStore CertStorage + KeepHeader bool } var hasPort = regexp.MustCompile(`:\d+$`) diff --git a/websocket.go b/websocket.go index 2a9699117..522b88e32 100644 --- a/websocket.go +++ b/websocket.go @@ -49,7 +49,7 @@ func (proxy *ProxyHttpServer) serveWebsocketTLS(ctx *ProxyCtx, w http.ResponseWr func (proxy *ProxyHttpServer) serveWebsocket(ctx *ProxyCtx, w http.ResponseWriter, req *http.Request) { targetURL := url.URL{Scheme: "ws", Host: req.URL.Host, Path: req.URL.Path} - targetConn, err := proxy.connectDial("tcp", targetURL.Host) + targetConn, err := proxy.connectDial(ctx, "tcp", targetURL.Host) if err != nil { ctx.Warnf("Error dialing target site: %v", err) return From a0805db90819b464f8fadc660e23a6a6b9d4942f Mon Sep 17 00:00:00 2001 From: xujin zheng Date: Wed, 17 Mar 2021 18:06:34 +0800 Subject: [PATCH 14/42] fix issue: #431 (ERR_CERT_VALIDITY_TOO_LONG) --- signer.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/signer.go b/signer.go index 2d02294e7..aa511ca9f 100644 --- a/signer.go +++ b/signer.go @@ -44,11 +44,9 @@ func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err er if x509ca, err = x509.ParseCertificate(ca.Certificate[0]); err != nil { return } - start := time.Unix(0, 0) - end, err := time.Parse("2006-01-02", "2049-12-31") - if err != nil { - panic(err) - } + + start := time.Unix(time.Now().Unix()-2592000, 0) // 2592000 = 30 day + end := time.Unix(time.Now().Unix()+31536000, 0) // 31536000 = 365 day serial := big.NewInt(rand.Int63()) template := x509.Certificate{ From fc00ffa95ded1e8a8c27a52357feeb1c50cb4df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Mon, 7 Nov 2022 14:17:12 +0100 Subject: [PATCH 15/42] IsLocalHost: support host:port Without this patch IsLocalHost does not work for URLs with port specified i.e. it works for `http://localhost` but does not work for `http://localhost:80` or `http://localhost:10000`. Fixes #487 --- dispatcher.go | 23 +++++++++++++++-------- dispatcher_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 dispatcher_test.go diff --git a/dispatcher.go b/dispatcher.go index 25c949c0d..9b79bcc39 100644 --- a/dispatcher.go +++ b/dispatcher.go @@ -97,15 +97,22 @@ func ReqHostIs(hosts ...string) ReqConditionFunc { } } -var localHostIpv4 = regexp.MustCompile(`127\.0\.0\.\d+`) - -// IsLocalHost checks whether the destination host is explicitly local host -// (buggy, there can be IPv6 addresses it doesn't catch) +// IsLocalHost checks whether the destination host is localhost. var IsLocalHost ReqConditionFunc = func(req *http.Request, ctx *ProxyCtx) bool { - return req.URL.Host == "::1" || - req.URL.Host == "0:0:0:0:0:0:0:1" || - localHostIpv4.MatchString(req.URL.Host) || - req.URL.Host == "localhost" + h := req.URL.Hostname() + if h == "localhost" { + return true + } + if ip := net.ParseIP(h); ip != nil { + return ip.IsLoopback() + } + + // In case of IPv6 without a port number Hostname() sometimes returns the invalid value. + if ip := net.ParseIP(req.URL.Host); ip != nil { + return ip.IsLoopback() + } + + return false } // UrlMatches returns a ReqCondition testing whether the destination URL diff --git a/dispatcher_test.go b/dispatcher_test.go new file mode 100644 index 000000000..c7e50245e --- /dev/null +++ b/dispatcher_test.go @@ -0,0 +1,47 @@ +package goproxy + +import ( + "net" + "net/http" + "strings" + "testing" +) + +func TestIsLocalHost(t *testing.T) { + hosts := []string{ + "localhost", + "127.0.0.1", + "127.0.0.7", + "::ffff:127.0.0.1", + "::ffff:127.0.0.7", + "::1", + "0:0:0:0:0:0:0:1", + } + ports := []string{ + "", + "80", + "443", + } + + for _, host := range hosts { + for _, port := range ports { + if port == "" && strings.HasPrefix(host, "::ffff:") { + continue + } + + addr := host + if port != "" { + addr = net.JoinHostPort(host, port) + } + t.Run(addr, func(t *testing.T) { + req, err := http.NewRequest("GET", "http://"+addr, http.NoBody) + if err != nil { + t.Fatal(err) + } + if !IsLocalHost(req, nil) { + t.Fatal("expected true") + } + }) + } + } +} From 7b86e075023da3f7225a00946a5c8e60b527fb6b Mon Sep 17 00:00:00 2001 From: rapt0r Date: Thu, 13 Jul 2023 10:08:19 +0900 Subject: [PATCH 16/42] Added control for the nil request --- https.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/https.go b/https.go index 5a601a800..d8624c3fa 100644 --- a/https.go +++ b/https.go @@ -246,7 +246,11 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request return } if err != nil { - ctx.Warnf("Illegal URL %s", "https://"+r.Host+req.URL.Path) + if req != nil { + ctx.Warnf("Illegal URL %s", "https://"+r.Host+req.URL.Path) + }else { + ctx.Warnf("Illegal URL %s", "https://"+r.Host) + } return } removeProxyHeaders(ctx, req) From e0779594e990a1089e3c088ba8ce715b218f6210 Mon Sep 17 00:00:00 2001 From: rapt0r Date: Fri, 21 Jul 2023 11:24:37 +0900 Subject: [PATCH 17/42] Add missing space --- https.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/https.go b/https.go index d8624c3fa..dc09d4b79 100644 --- a/https.go +++ b/https.go @@ -248,7 +248,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request if err != nil { if req != nil { ctx.Warnf("Illegal URL %s", "https://"+r.Host+req.URL.Path) - }else { + } else { ctx.Warnf("Illegal URL %s", "https://"+r.Host) } return From f99041a5c0273fcf6a144498b458a630585872fa Mon Sep 17 00:00:00 2001 From: rapt0r Date: Sat, 29 Jul 2023 18:27:42 +0900 Subject: [PATCH 18/42] fixed nil check --- https.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/https.go b/https.go index dc09d4b79..5238d6933 100644 --- a/https.go +++ b/https.go @@ -246,7 +246,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request return } if err != nil { - if req != nil { + if req.URL != nil { ctx.Warnf("Illegal URL %s", "https://"+r.Host+req.URL.Path) } else { ctx.Warnf("Illegal URL %s", "https://"+r.Host) From 2592e75ae04ae4282011ea5c2c25f236451039b7 Mon Sep 17 00:00:00 2001 From: WGH Date: Wed, 26 Jul 2023 22:21:00 +0300 Subject: [PATCH 19/42] Fix data race in MITM'ed HTTPS request handling This data race requires quite of few conditions to appear. 1) The transport has to have HTTP/2 enabled (in goproxy, it's not by default). 2) The target server must be an HTTP/2 one. 3) The request must have a body. 4) The server must return certain erroneus HTTP status code. 5) The client must ignore Connection: close header from the proxy (4ec79339bd51dec356e283d8c8de29748b42b1fe). Somehow, I managed to hit all these conditions, and got this error: panic: runtime error: slice bounds out of range [96:48] goroutine 341699 [running]: bufio.(*Reader).ReadSlice(0xc000b0c1e0, 0x80?) /usr/local/go/src/bufio/bufio.go:347 +0x225 bufio.(*Reader).ReadLine(0xc000b0c1e0) /usr/local/go/src/bufio/bufio.go:401 +0x27 net/textproto.(*Reader).readLineSlice(0xc0013400f0) /usr/local/go/src/net/textproto/reader.go:56 +0x99 net/textproto.(*Reader).ReadLine(...) /usr/local/go/src/net/textproto/reader.go:39 net/http.readRequest(0xc000847d40?) /usr/local/go/src/net/http/request.go:1042 +0xba net/http.ReadRequest(0xc000b0c1e0?) /usr/local/go/src/net/http/request.go:1025 +0x19 github.com/elazarl/goproxy.(*ProxyHttpServer).handleHttps.func2() /root/go/pkg/mod/github.com/elazarl/goproxy@v0.0.0-20220901064549-fbd10ff4f5a1/https.go:221 +0x3db created by github.com/elazarl/goproxy.(*ProxyHttpServer).handleHttps /root/go/pkg/mod/github.com/elazarl/goproxy@v0.0.0-20220901064549-fbd10ff4f5a1/https.go:211 +0x611 Despite 5), goproxy should not break from non-compliant HTTP clients, and "Connection: close" is more like of an accidental fix. See also https://github.com/golang/go/issues/61596#issuecomment-1652345131 --- https.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/https.go b/https.go index 5238d6933..e105caf87 100644 --- a/https.go +++ b/https.go @@ -254,7 +254,12 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request return } removeProxyHeaders(ctx, req) - resp, err = ctx.RoundTrip(req) + resp, err = func() (*http.Response, error) { + // explicitly discard request body to avoid data races in certain RoundTripper implementations + // see https://github.com/golang/go/issues/61596#issuecomment-1652345131 + defer req.Body.Close() + return ctx.RoundTrip(req) + }() if err != nil { ctx.Warnf("Cannot read TLS response from mitm'd server %v", err) return From 1fe6677f404d56c7e96376b75332c5fd25faf385 Mon Sep 17 00:00:00 2001 From: guoguangwu Date: Tue, 17 Oct 2023 22:18:32 +0800 Subject: [PATCH 20/42] chore: use strings.ContainsRune instead --- https.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/https.go b/https.go index e105caf87..6f7a2c2d2 100644 --- a/https.go +++ b/https.go @@ -378,7 +378,7 @@ func (proxy *ProxyHttpServer) NewConnectDialToProxyWithHandler(https_proxy strin return nil } if u.Scheme == "" || u.Scheme == "http" { - if strings.IndexRune(u.Host, ':') == -1 { + if !strings.ContainsRune(u.Host, ':') { u.Host += ":80" } return func(network, addr string) (net.Conn, error) { @@ -418,7 +418,7 @@ func (proxy *ProxyHttpServer) NewConnectDialToProxyWithHandler(https_proxy strin } } if u.Scheme == "https" || u.Scheme == "wss" { - if strings.IndexRune(u.Host, ':') == -1 { + if !strings.ContainsRune(u.Host, ':') { u.Host += ":443" } return func(network, addr string) (net.Conn, error) { From 3ec07828be7ac4d84b1561216ae707dc12b2739a Mon Sep 17 00:00:00 2001 From: Omri Bahumi Date: Sun, 22 Aug 2021 17:04:35 +0300 Subject: [PATCH 21/42] httpError: write error string as body in 502 response --- https.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/https.go b/https.go index 6f7a2c2d2..608863fad 100644 --- a/https.go +++ b/https.go @@ -4,6 +4,7 @@ import ( "bufio" "crypto/tls" "errors" + "fmt" "io" "io/ioutil" "net" @@ -129,6 +130,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request } targetSiteCon, err := proxy.connectDial(ctx, "tcp", host) if err != nil { + ctx.Warnf("Error dialing to %s: %s", host, err.Error()) httpError(proxyClient, ctx, err) return } @@ -333,7 +335,8 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request } func httpError(w io.WriteCloser, ctx *ProxyCtx, err error) { - if _, err := io.WriteString(w, "HTTP/1.1 502 Bad Gateway\r\n\r\n"); err != nil { + errStr := fmt.Sprintf("HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s", len(err.Error()), err.Error()) + if _, err := io.WriteString(w, errStr); err != nil { ctx.Warnf("Error responding to client: %s", err) } if err := w.Close(); err != nil { From 7cc037d33fb57d20c2fa7075adaf0e2d2862da78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=97=BA=E6=B1=AA?= <2246521484@qq.com> Date: Fri, 17 Nov 2023 10:29:38 +0800 Subject: [PATCH 22/42] fix: Package renaming --- examples/goproxy-stats/main.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/goproxy-stats/main.go b/examples/goproxy-stats/main.go index e4cde8d93..6865c19a6 100644 --- a/examples/goproxy-stats/main.go +++ b/examples/goproxy-stats/main.go @@ -2,12 +2,13 @@ package main import ( "fmt" - "github.com/elazarl/goproxy" - "github.com/elazarl/goproxy/ext/html" "io" "log" . "net/http" "time" + + "github.com/elazarl/goproxy" + goproxy_html "github.com/elazarl/goproxy/ext/html" ) type Count struct { From 03be62527ccbb67ca603c9ca5cd741fbf5f58b9e Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Thu, 13 Jun 2024 12:12:13 -0400 Subject: [PATCH 23/42] Clean up SimpleHttpRequest test. Per RFC 6761, example.com is a standards-based choice for a functioning http site while the .invalid TLD will not exist. --- proxy_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/proxy_test.go b/proxy_test.go index dc756e534..429c666bb 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -921,10 +921,6 @@ func TestHttpsMitmURLRewrite(t *testing.T) { } } -func returnNil(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { - return nil -} - func TestSimpleHttpRequest(t *testing.T) { proxy := goproxy.NewProxyHttpServer() @@ -950,21 +946,25 @@ func TestSimpleHttpRequest(t *testing.T) { } client := http.Client{Transport: tr} - resp, err := client.Get("http://google.de") - if err != nil || resp.StatusCode != 200 { - t.Error("Error while requesting google with http", err) + resp, err := client.Get("http://example.com") + if err != nil { + t.Error("Error requesting http site", err) + } else if resp.StatusCode != 200 { + t.Error("Non-OK status requesting http site", err) } - resp, err = client.Get("http://google20012312031.de") - fmt.Println(resp) + resp, _ = client.Get("http://example.invalid") if resp == nil { - t.Error("Error while requesting random string with http", resp) + t.Error("No response requesting invalid http site") + } + + returnNil := func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { + return nil } proxy.OnResponse(goproxy.UrlMatches(regexp.MustCompile(".*"))).DoFunc(returnNil) - resp, err = client.Get("http://google20012312031.de") - fmt.Println(resp) + resp, _ = client.Get("http://example.invalid") if resp == nil { - t.Error("Error while requesting random string with http", resp) + t.Error("No response requesting invalid http site") } server.Shutdown(context.TODO()) From 6ca80f56455468b7ee41e5d78d848b6b1951c2de Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Fri, 5 Jul 2024 11:09:42 +0200 Subject: [PATCH 24/42] Ensure tls client is closed if handshake fails The tls client wasn't close when a handshake error occurs, this caused the proxy to keep a connection open without ever responding. --- https.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/https.go b/https.go index 608863fad..defadbc3f 100644 --- a/https.go +++ b/https.go @@ -213,11 +213,11 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request go func() { //TODO: cache connections to the remote website rawClientTls := tls.Server(proxyClient, tlsConfig) + defer rawClientTls.Close() if err := rawClientTls.Handshake(); err != nil { ctx.Warnf("Cannot handshake client %v %v", r.Host, err) return } - defer rawClientTls.Close() clientTlsReader := bufio.NewReader(rawClientTls) for !isEof(clientTlsReader) { req, err := http.ReadRequest(clientTlsReader) From 5ecc1787942952888507f62ba880c6e8d287f9e3 Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Thu, 13 Jun 2024 14:13:22 -0400 Subject: [PATCH 25/42] Add basic passthrough HTTP/2 support. --- go.mod | 9 ++- go.sum | 4 ++ h2.go | 171 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ https.go | 24 ++++++++ proxy.go | 1 + 5 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 h2.go diff --git a/go.mod b/go.mod index 30554d588..023ec0cf0 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,10 @@ module github.com/elazarl/goproxy -require github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 +go 1.23 + +require ( + github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 + golang.org/x/net v0.26.0 +) + +require golang.org/x/text v0.16.0 // indirect diff --git a/go.sum b/go.sum index 9a18f4f31..05c67afd6 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= diff --git a/h2.go b/h2.go new file mode 100644 index 000000000..7c0f35790 --- /dev/null +++ b/h2.go @@ -0,0 +1,171 @@ +package goproxy + +import ( + "bufio" + "crypto/tls" + "errors" + "io" + "net" + "net/http" + "strings" + + "golang.org/x/net/http2" +) + +// H2Transport is an implementation of RoundTripper that abstracts an entire +// HTTP/2 session, sending all client frames to the server and responses back +// to the client. +type H2Transport struct { + ClientReader io.Reader + ClientWriter io.Writer + TLSConfig *tls.Config + Host string +} + +// RoundTrip executes an HTTP/2 session (including all contained streams). +// The request and response are ignored but any error encountered during the +// proxying from the session is returned as a result of the invocation. +func (r *H2Transport) RoundTrip(prefaceReq *http.Request) (*http.Response, error) { + raddr := r.Host + if !strings.Contains(raddr, ":") { + raddr = raddr + ":443" + } + rawServerTLS, err := dial("tcp", raddr) + if err != nil { + return nil, err + } + defer rawServerTLS.Close() + // Ensure that we only advertise HTTP/2 as the accepted protocol. + r.TLSConfig.NextProtos = []string{http2.NextProtoTLS} + // Initiate TLS and check remote host name against certificate. + rawServerTLS = tls.Client(rawServerTLS, r.TLSConfig) + if err = rawServerTLS.(*tls.Conn).Handshake(); err != nil { + return nil, err + } + if r.TLSConfig == nil || !r.TLSConfig.InsecureSkipVerify { + if err = rawServerTLS.(*tls.Conn).VerifyHostname(raddr[:strings.LastIndex(raddr, ":")]); err != nil { + return nil, err + } + } + // Send new client preface to match the one parsed in req. + if _, err := io.WriteString(rawServerTLS, http2.ClientPreface); err != nil { + return nil, err + } + serverTLSReader := bufio.NewReader(rawServerTLS) + cToS := http2.NewFramer(rawServerTLS, r.ClientReader) + sToC := http2.NewFramer(r.ClientWriter, serverTLSReader) + errSToC := make(chan error) + errCToS := make(chan error) + go func() { + for { + if err := proxyFrame(sToC); err != nil { + errSToC <- err + break + } + } + }() + go func() { + for { + if err := proxyFrame(cToS); err != nil { + errCToS <- err + break + } + } + }() + for i := 0; i < 2; i++ { + select { + case err := <-errSToC: + if err != io.EOF { + return nil, err + } + case err := <-errCToS: + if err != io.EOF { + return nil, err + } + } + } + return nil, nil +} + +func dial(network, addr string) (c net.Conn, err error) { + addri, err := net.ResolveTCPAddr(network, addr) + if err != nil { + return + } + c, err = net.DialTCP(network, nil, addri) + return +} + +// proxyFrame reads a single frame from the Framer and, when successful, writes +// a ~identical one back to the Framer. +func proxyFrame(fr *http2.Framer) error { + f, err := fr.ReadFrame() + if err != nil { + return err + } + switch f.Header().Type { + case http2.FrameData: + tf := f.(*http2.DataFrame) + terr := fr.WriteData(tf.StreamID, tf.StreamEnded(), tf.Data()) + if terr == nil && tf.StreamEnded() { + terr = io.EOF + } + return terr + case http2.FrameHeaders: + tf := f.(*http2.HeadersFrame) + terr := fr.WriteHeaders(http2.HeadersFrameParam{ + StreamID: tf.StreamID, + BlockFragment: tf.HeaderBlockFragment(), + EndStream: tf.StreamEnded(), + EndHeaders: tf.HeadersEnded(), + PadLength: 0, + Priority: tf.Priority, + }) + if terr == nil && tf.StreamEnded() { + terr = io.EOF + } + return terr + case http2.FrameContinuation: + tf := f.(*http2.ContinuationFrame) + return fr.WriteContinuation(tf.StreamID, tf.HeadersEnded(), tf.HeaderBlockFragment()) + case http2.FrameGoAway: + tf := f.(*http2.GoAwayFrame) + return fr.WriteGoAway(tf.StreamID, tf.ErrCode, tf.DebugData()) + case http2.FramePing: + tf := f.(*http2.PingFrame) + return fr.WritePing(tf.IsAck(), tf.Data) + case http2.FrameRSTStream: + tf := f.(*http2.RSTStreamFrame) + return fr.WriteRSTStream(tf.StreamID, tf.ErrCode) + case http2.FrameSettings: + tf := f.(*http2.SettingsFrame) + if tf.IsAck() { + return fr.WriteSettingsAck() + } + var settings []http2.Setting + // NOTE: If we want to parse headers, need to handle + // settings where s.ID == http2.SettingHeaderTableSize and + // accordingly update the Framer options. + for i := 0; i < tf.NumSettings(); i++ { + settings = append(settings, tf.Setting(i)) + } + return fr.WriteSettings(settings...) + case http2.FrameWindowUpdate: + tf := f.(*http2.WindowUpdateFrame) + return fr.WriteWindowUpdate(tf.StreamID, tf.Increment) + case http2.FramePriority: + tf := f.(*http2.PriorityFrame) + return fr.WritePriority(tf.StreamID, tf.PriorityParam) + case http2.FramePushPromise: + tf := f.(*http2.PushPromiseFrame) + return fr.WritePushPromise(http2.PushPromiseParam{ + StreamID: tf.StreamID, + PromiseID: tf.PromiseID, + BlockFragment: tf.HeaderBlockFragment(), + EndHeaders: tf.HeadersEnded(), + PadLength: 0, + }) + default: + return errors.New("Unsupported frame: " + string(f.Header().Type)) + } +} diff --git a/https.go b/https.go index defadbc3f..271b55cab 100644 --- a/https.go +++ b/https.go @@ -242,6 +242,30 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request req, resp := proxy.filterRequest(req, ctx) if resp == nil { + if req.Method == "PRI" { + // Handle HTTP/2 connections. + + // NOTE: As of 1.22, golang's http module will not recognize or + // parse the HTTP Body for PRI requests. This leaves the body of + // the http2.ClientPreface ("SM\r\n\r\n") on the wire which we need + // to clear before setting up the connection. + _, err := clientTlsReader.Discard(6) + if err != nil { + ctx.Warnf("Failed to process HTTP2 client preface: %v", err) + return + } + if !proxy.AllowHTTP2 { + ctx.Warnf("HTTP2 connection failed: disallowed") + return + } + tr := H2Transport{clientTlsReader, rawClientTls, tlsConfig.Clone(), host} + if _, err := tr.RoundTrip(req); err != nil { + ctx.Warnf("HTTP2 connection failed: %v", err) + } else { + ctx.Logf("Exiting on EOF") + } + return + } if isWebSocketRequest(req) { ctx.Logf("Request looks like websocket upgrade.") proxy.serveWebsocketTLS(ctx, w, req, tlsConfig, rawClientTls) diff --git a/proxy.go b/proxy.go index fa5494c6a..3deecfb09 100644 --- a/proxy.go +++ b/proxy.go @@ -32,6 +32,7 @@ type ProxyHttpServer struct { ConnectDialWithReq func(req *http.Request, network string, addr string) (net.Conn, error) CertStore CertStorage KeepHeader bool + AllowHTTP2 bool } var hasPort = regexp.MustCompile(`:\d+$`) From 24b50190c2009e354c781939f5d2609a5a3fd2ce Mon Sep 17 00:00:00 2001 From: Arthur Baars Date: Mon, 22 Jul 2024 11:00:58 +0200 Subject: [PATCH 26/42] ext.auth: return `nil` instead of `OkConnect` This allows falling through to a next `ConnectHandler` is authentication succeeds making if easier to combine with other `ConnectHandler`s. --- ext/auth/basic.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/auth/basic.go b/ext/auth/basic.go index a433f2d02..42d555b0e 100644 --- a/ext/auth/basic.go +++ b/ext/auth/basic.go @@ -68,7 +68,7 @@ func BasicConnect(realm string, f func(user, passwd string) bool) goproxy.HttpsH ctx.Resp = BasicUnauthorized(ctx.Req, realm) return goproxy.RejectConnect, host } - return goproxy.OkConnect, host + return nil, host }) } From 8b0c205063807802a7ac1d75351a90172a9c83fb Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Fri, 26 Jul 2024 11:16:43 -0400 Subject: [PATCH 27/42] Fix go version constraint --- .github/workflows/go.yml | 4 ++-- go.mod | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 745dae503..5b93b5cd8 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.13 + - name: Set up Go 1.18 uses: actions/setup-go@v1 with: - go-version: 1.13 + go-version: 1.18 id: go - name: Check out code into the Go module directory diff --git a/go.mod b/go.mod index 023ec0cf0..465994912 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/elazarl/goproxy -go 1.23 +go 1.18 require ( github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 From 6741dbfc16a13e515d5f2b485eb4be24692e54de Mon Sep 17 00:00:00 2001 From: Damin Lukawski Date: Fri, 7 Jul 2023 00:47:37 +0200 Subject: [PATCH 28/42] Add https to http websocket override --- https.go | 7 ++++++- websocket.go | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/https.go b/https.go index 271b55cab..2236c3e70 100644 --- a/https.go +++ b/https.go @@ -268,7 +268,12 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request } if isWebSocketRequest(req) { ctx.Logf("Request looks like websocket upgrade.") - proxy.serveWebsocketTLS(ctx, w, req, tlsConfig, rawClientTls) + if req.URL.Scheme == "http" { + ctx.Logf("Enforced HTTP websocket forwarding over TLS") + proxy.serveWebsocketHttpOverTLS(ctx, w, req, rawClientTls) + } else { + proxy.serveWebsocketTLS(ctx, w, req, tlsConfig, rawClientTls) + } return } if err != nil { diff --git a/websocket.go b/websocket.go index 522b88e32..753a1e8d3 100644 --- a/websocket.go +++ b/websocket.go @@ -46,6 +46,27 @@ func (proxy *ProxyHttpServer) serveWebsocketTLS(ctx *ProxyCtx, w http.ResponseWr proxy.proxyWebsocket(ctx, targetConn, clientConn) } +func (proxy *ProxyHttpServer) serveWebsocketHttpOverTLS(ctx *ProxyCtx, w http.ResponseWriter, req *http.Request, clientConn *tls.Conn) { + targetURL := url.URL{Scheme: "ws", Host: req.URL.Host, Path: req.URL.Path} + + // Connect to upstream + targetConn, err := proxy.connectDial(ctx, "tcp", targetURL.Host) + if err != nil { + ctx.Warnf("Error dialing target site: %v", err) + return + } + defer targetConn.Close() + + // Perform handshake + if err := proxy.websocketHandshake(ctx, req, targetConn, clientConn); err != nil { + ctx.Warnf("Websocket handshake error: %v", err) + return + } + + // Proxy wss connection + proxy.proxyWebsocket(ctx, targetConn, clientConn) +} + func (proxy *ProxyHttpServer) serveWebsocket(ctx *ProxyCtx, w http.ResponseWriter, req *http.Request) { targetURL := url.URL{Scheme: "ws", Host: req.URL.Host, Path: req.URL.Path} From 47dbfa5ad070bdf741883556c4a0bf756fb0a541 Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Thu, 28 Nov 2024 23:25:41 +0100 Subject: [PATCH 29/42] Add support for ED25519 CA cert key --- counterecryptor.go | 7 ++++++- signer.go | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/counterecryptor.go b/counterecryptor.go index d1c39d23b..2ce9da97b 100644 --- a/counterecryptor.go +++ b/counterecryptor.go @@ -4,6 +4,7 @@ import ( "crypto/aes" "crypto/cipher" "crypto/ecdsa" + "crypto/ed25519" "crypto/rsa" "crypto/sha256" "crypto/x509" @@ -26,8 +27,12 @@ func NewCounterEncryptorRandFromKey(key interface{}, seed []byte) (r CounterEncr if keyBytes, err = x509.MarshalECPrivateKey(key); err != nil { return } + case ed25519.PrivateKey: + if keyBytes, err = x509.MarshalPKCS8PrivateKey(key); err != nil { + return + } default: - err = errors.New("only RSA and ECDSA keys supported") + err = errors.New("only RSA, ED25519 and ECDSA keys supported") return } h := sha256.New() diff --git a/signer.go b/signer.go index aa511ca9f..4ed6e2c83 100644 --- a/signer.go +++ b/signer.go @@ -3,6 +3,7 @@ package goproxy import ( "crypto" "crypto/ecdsa" + "crypto/ed25519" "crypto/elliptic" "crypto/rsa" "crypto/sha1" @@ -88,6 +89,10 @@ func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err er if certpriv, err = ecdsa.GenerateKey(elliptic.P256(), &csprng); err != nil { return } + case ed25519.PrivateKey: + if _, certpriv, err = ed25519.GenerateKey(&csprng); err != nil { + return + } default: err = fmt.Errorf("unsupported key type %T", ca.PrivateKey) } From 60626aee8fb5cbd8b1323305809f0d519f68d46b Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Thu, 28 Nov 2024 23:46:00 +0100 Subject: [PATCH 30/42] Use same Subject of CA as the MITM certificate Subject --- signer.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/signer.go b/signer.go index 4ed6e2c83..506674577 100644 --- a/signer.go +++ b/signer.go @@ -9,7 +9,6 @@ import ( "crypto/sha1" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "fmt" "math/big" "math/rand" @@ -54,11 +53,9 @@ func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err er // TODO(elazar): instead of this ugly hack, just encode the certificate and hash the binary form. SerialNumber: serial, Issuer: x509ca.Subject, - Subject: pkix.Name{ - Organization: []string{"GoProxy untrusted MITM proxy Inc"}, - }, - NotBefore: start, - NotAfter: end, + Subject: x509ca.Subject, + NotBefore: start, + NotAfter: end, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, From 3df585cdee79bd231191575e57af1f32673f2ebb Mon Sep 17 00:00:00 2001 From: youngifif Date: Wed, 11 Dec 2024 11:44:38 +0800 Subject: [PATCH 31/42] ProxyHttpServer.Tr.Dial() Deprecated, use DialContext() instead. --- examples/goproxy-sokeepalive/sokeepalive.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/examples/goproxy-sokeepalive/sokeepalive.go b/examples/goproxy-sokeepalive/sokeepalive.go index 9135e575a..a50398e51 100644 --- a/examples/goproxy-sokeepalive/sokeepalive.go +++ b/examples/goproxy-sokeepalive/sokeepalive.go @@ -1,6 +1,7 @@ package main import ( + "context" "flag" "github.com/elazarl/goproxy" "log" @@ -13,10 +14,19 @@ func main() { addr := flag.String("addr", ":8080", "proxy listen address") flag.Parse() proxy := goproxy.NewProxyHttpServer() - proxy.Tr.Dial = func(network, addr string) (c net.Conn, err error) { - c, err = net.Dial(network, addr) + proxy.Tr.DialContext = func(ctx context.Context, network, addr string) (c net.Conn, err error) { + var d net.Dialer + c, err = d.DialContext(ctx, network, addr) if c, ok := c.(*net.TCPConn); err == nil && ok { c.SetKeepAlive(true) + go func() { + select { + case <-ctx.Done(): + { + c.Close() + } + } + }() } return } From 8f71f010f696f6d094b124dcb325a410a7d4e5f4 Mon Sep 17 00:00:00 2001 From: Alessio Dalla Piazza Date: Tue, 19 Nov 2024 15:54:31 +0100 Subject: [PATCH 32/42] avoid canonicalization --- proxy.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/proxy.go b/proxy.go index 3deecfb09..b50cae863 100644 --- a/proxy.go +++ b/proxy.go @@ -44,9 +44,8 @@ func copyHeaders(dst, src http.Header, keepDestHeaders bool) { } } for k, vs := range src { - for _, v := range vs { - dst.Add(k, v) - } + // direct assignment to avoid canonicalization + dst[k] = append([]string(nil), vs...) } } From e85c60b374339a6f8f804bf46dfca7bde41ddba8 Mon Sep 17 00:00:00 2001 From: ryoii Date: Mon, 16 Dec 2024 18:20:27 +0800 Subject: [PATCH 33/42] Improve the https scheme matching (#508) Improve the https scheme matching --------- Co-authored-by: Erik Pellizzon --- https.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/https.go b/https.go index 2236c3e70..cde6768b1 100644 --- a/https.go +++ b/https.go @@ -11,7 +11,6 @@ import ( "net/http" "net/url" "os" - "regexp" "strconv" "strings" "sync" @@ -34,7 +33,6 @@ var ( MitmConnect = &ConnectAction{Action: ConnectMitm, TLSConfig: TLSConfigFromCA(&GoproxyCa)} HTTPMitmConnect = &ConnectAction{Action: ConnectHTTPMitm, TLSConfig: TLSConfigFromCA(&GoproxyCa)} RejectConnect = &ConnectAction{Action: ConnectReject, TLSConfig: TLSConfigFromCA(&GoproxyCa)} - httpsRegexp = regexp.MustCompile(`^https:\/\/`) ) // ConnectAction enables the caller to override the standard connect flow. @@ -232,7 +230,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request req.RemoteAddr = r.RemoteAddr // since we're converting the request, need to carry over the original connecting IP as well ctx.Logf("req %v", r.Host) - if !httpsRegexp.MatchString(req.URL.String()) { + if !strings.HasPrefix(req.URL.String(), "https://") { req.URL, err = url.Parse("https://" + r.Host + req.URL.String()) } From 5d8f603c0208b2cc2e0815c7508efdb682407b70 Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Mon, 16 Dec 2024 21:47:37 +0100 Subject: [PATCH 34/42] math/rand call refactoring (#567) * Update go version to 1.20 * Disable rand seed initialization since Go 1.20 automatically provides it * Generate random number using Uint64() --- .github/workflows/go.yml | 4 ++-- go.mod | 8 ++++---- go.sum | 12 ++++++------ signer.go | 14 ++++++-------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 5b93b5cd8..67809e6ee 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -13,10 +13,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go 1.18 + - name: Set up Go 1.20 uses: actions/setup-go@v1 with: - go-version: 1.18 + go-version: 1.20 id: go - name: Check out code into the Go module directory diff --git a/go.mod b/go.mod index 465994912..faf302de5 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,10 @@ module github.com/elazarl/goproxy -go 1.18 +go 1.20 require ( - github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 - golang.org/x/net v0.26.0 + github.com/elazarl/goproxy/ext v0.0.0-20241216102027-e85c60b37433 + golang.org/x/net v0.32.0 ) -require golang.org/x/text v0.16.0 // indirect +require golang.org/x/text v0.21.0 // indirect diff --git a/go.sum b/go.sum index 05c67afd6..77e00d232 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/elazarl/goproxy/ext v0.0.0-20241216102027-e85c60b37433 h1:zezqs+UN/8nYMOm1PobfrT/FxliWYq5Um1DLxyHA8d0= +github.com/elazarl/goproxy/ext v0.0.0-20241216102027-e85c60b37433/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= diff --git a/signer.go b/signer.go index 506674577..de9efcf5e 100644 --- a/signer.go +++ b/signer.go @@ -48,10 +48,13 @@ func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err er start := time.Unix(time.Now().Unix()-2592000, 0) // 2592000 = 30 day end := time.Unix(time.Now().Unix()+31536000, 0) // 31536000 = 365 day - serial := big.NewInt(rand.Int63()) + // Always generate a positive int value + // (Two complement is not enabled when the first bit is 0) + generated := rand.Uint64() + generated >>= 1 + template := x509.Certificate{ - // TODO(elazar): instead of this ugly hack, just encode the certificate and hash the binary form. - SerialNumber: serial, + SerialNumber: big.NewInt(int64(generated)), Issuer: x509ca.Subject, Subject: x509ca.Subject, NotBefore: start, @@ -103,8 +106,3 @@ func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err er PrivateKey: certpriv, }, nil } - -func init() { - // Avoid deterministic random numbers - rand.Seed(time.Now().UnixNano()) -} From da3cdeedf3e03c03aa68272c9445731ee5c0e4c3 Mon Sep 17 00:00:00 2001 From: taco wang <1027480600@qq.com> Date: Tue, 17 Dec 2024 18:44:36 +0800 Subject: [PATCH 35/42] fix the problem that HTTP steam request cannot be flushed (#523) * Add check for charset in content-type header in ServeHTTP function * Add support for chunked transfer encoding in ServeHTTP function * Fix content-type and transfer-encoding checks in ServeHTTP function --------- Co-authored-by: taco.wang Co-authored-by: Erik Pellizzon --- proxy.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/proxy.go b/proxy.go index b50cae863..e55f7eef5 100644 --- a/proxy.go +++ b/proxy.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "regexp" + "strings" "sync/atomic" ) @@ -193,7 +194,10 @@ func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) copyHeaders(w.Header(), resp.Header, proxy.KeepDestinationHeaders) w.WriteHeader(resp.StatusCode) var copyWriter io.Writer = w - if w.Header().Get("content-type") == "text/event-stream" { + // Content-Type header may also contain charset definition, so here we need to check the prefix. + // Transfer-Encoding can be a list of comma separated values, so we use Contains() for it. + if strings.HasPrefix(w.Header().Get("content-type"), "text/event-stream") || + strings.Contains(w.Header().Get("transfer-encoding"), "chunked") { // server-side events, flush the buffered data to the client. copyWriter = &flushWriter{w: w} } From f31a87fcc8c4c2fb0ce686e582753faf58280d13 Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Tue, 17 Dec 2024 11:48:19 +0100 Subject: [PATCH 36/42] Case insensitive host comparison (#565) * Add case-insensitive URL block in the example * Make a case-insensitive comparation inside DstHostIs --- README.md | 5 +++-- dispatcher.go | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 495afc2d4..c11167031 100644 --- a/README.md +++ b/README.md @@ -135,8 +135,9 @@ proxy.OnResponse(Some RespConditions).Do(YourRespHandlerFunc()) For example: ```go -// This rejects the HTTPS request to *.reddit.com during HTTP CONNECT phase -proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("reddit.*:443$"))).HandleConnect(goproxy.AlwaysReject) +// This rejects the HTTPS request to *.reddit.com during HTTP CONNECT phase. +// Reddit URL check is case-insensitive, so the block will work also if the user types something like rEdDit.com. +proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("(?i)reddit.*:443$"))).HandleConnect(goproxy.AlwaysReject) // This will NOT reject the HTTPS request with URL ending with gif, due to the fact that proxy // only got the URL.Hostname and URL.Port during the HTTP CONNECT phase if the scheme is HTTPS, which is diff --git a/dispatcher.go b/dispatcher.go index 9b79bcc39..b050af490 100644 --- a/dispatcher.go +++ b/dispatcher.go @@ -126,8 +126,9 @@ func UrlMatches(re *regexp.Regexp) ReqConditionFunc { // DstHostIs returns a ReqCondition testing wether the host in the request url is the given string func DstHostIs(host string) ReqConditionFunc { + host = strings.ToLower(host) return func(req *http.Request, ctx *ProxyCtx) bool { - return req.URL.Host == host + return strings.ToLower(req.URL.Host) == host } } From 7711dfa3811c09b2e61bfd7d3c79549b1c23cf64 Mon Sep 17 00:00:00 2001 From: Faisal Chaudhry Date: Tue, 17 Dec 2024 12:09:00 +0000 Subject: [PATCH 37/42] Bugfix while using multiple handlers (#415) should pass updated req for remaining handlers --- proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy.go b/proxy.go index e55f7eef5..7542656a8 100644 --- a/proxy.go +++ b/proxy.go @@ -61,7 +61,7 @@ func isEof(r *bufio.Reader) bool { func (proxy *ProxyHttpServer) filterRequest(r *http.Request, ctx *ProxyCtx) (req *http.Request, resp *http.Response) { req = r for _, h := range proxy.reqHandlers { - req, resp = h.Handle(r, ctx) + req, resp = h.Handle(req, ctx) // non-nil resp means the handler decided to skip sending the request // and return canned response instead. if resp != nil { From a057d2cb0f38229e4c2d3806e4c6bd06fa1526b0 Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Tue, 17 Dec 2024 17:39:56 +0100 Subject: [PATCH 38/42] Add concurrency limitation feature (#570) * Update dependencies * Add concurrent request handling limitation --- examples/go.mod | 14 ++++- examples/go.sum | 15 ++++- ext/go.mod | 12 +++- ext/go.sum | 8 +++ ext/limitation/concurrency.go | 32 ++++++++++ ext/limitation/concurrency_test.go | 96 ++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- 8 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 ext/limitation/concurrency.go create mode 100644 ext/limitation/concurrency_test.go diff --git a/examples/go.mod b/examples/go.mod index 1dc42f008..949933204 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -1,8 +1,18 @@ module github.com/elazarl/goproxy/examples/goproxy-transparent +go 1.20 + +require ( + github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c + github.com/elazarl/goproxy/ext v0.0.0-20241217120900-7711dfa3811c + github.com/gorilla/websocket v1.5.3 + github.com/inconshreveable/go-vhost v1.0.0 +) + require ( - github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a - github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b + github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/text v0.21.0 // indirect ) replace github.com/elazarl/goproxy => ../ diff --git a/examples/go.sum b/examples/go.sum index 610ba4fa8..52d578e71 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -1,4 +1,13 @@ -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= -github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b h1:IpLPmn6Re21F0MaV6Zsc5RdSE6KuoFpWmHiUSEs3PrE= -github.com/inconshreveable/go-vhost v0.0.0-20160627193104-06d84117953b/go.mod h1:aA6DnFhALT3zH0y+A39we+zbrdMC2N0X/q21e6FI0LU= +github.com/elazarl/goproxy/ext v0.0.0-20241217120900-7711dfa3811c h1:R+i10jtNSzKJKqEZAYJnR9M8y14k0zrNHqD1xkv/A2M= +github.com/elazarl/goproxy/ext v0.0.0-20241217120900-7711dfa3811c/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/inconshreveable/go-vhost v1.0.0 h1:IK4VZTlXL4l9vz2IZoiSFbYaaqUW7dXJAiPriUN5Ur8= +github.com/inconshreveable/go-vhost v1.0.0/go.mod h1:aA6DnFhALT3zH0y+A39we+zbrdMC2N0X/q21e6FI0LU= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31 h1:DE4LcMKyqAVa6a0CGmVxANbnVb7stzMmPkQiieyNmfQ= +github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= diff --git a/ext/go.mod b/ext/go.mod index 2c6aa3acd..9f64fcaa4 100644 --- a/ext/go.mod +++ b/ext/go.mod @@ -1,3 +1,13 @@ module github.com/elazarl/goproxy/ext -require github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 // indirect +go 1.20 + +require ( + github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c + github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31 +) + +require ( + golang.org/x/net v0.32.0 // indirect + golang.org/x/text v0.21.0 // indirect +) diff --git a/ext/go.sum b/ext/go.sum index 5cc3048e9..e224cd952 100644 --- a/ext/go.sum +++ b/ext/go.sum @@ -1,2 +1,10 @@ +github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c h1:yWAGp1CjD1mQGLUsADqPn5s1n2AkGAX33XLDUgoXzyo= +github.com/elazarl/goproxy v0.0.0-20241217120900-7711dfa3811c/go.mod h1:P73liMk9TZCyF9fXG/RyMeSizmATvpvy3ZS61/1eXn4= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 h1:BN/Nyn2nWMoqGRA7G7paDNDqTXE30mXGqzzybrfo05w= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31 h1:DE4LcMKyqAVa6a0CGmVxANbnVb7stzMmPkQiieyNmfQ= +github.com/rogpeppe/go-charset v0.0.0-20190617161244-0dc95cdf6f31/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= diff --git a/ext/limitation/concurrency.go b/ext/limitation/concurrency.go new file mode 100644 index 000000000..af357106d --- /dev/null +++ b/ext/limitation/concurrency.go @@ -0,0 +1,32 @@ +package limitation + +import ( + "net/http" + + "github.com/elazarl/goproxy" +) + +// ConcurrentRequests implements a mechanism to limit the number of +// concurrently handled HTTP requests, configurable by the user. +// The ReqHandler can simply be added to the server with OnRequest(). +func ConcurrentRequests(limit int) goproxy.ReqHandler { + // Do nothing when the specified limit is invalid + if limit <= 0 { + return goproxy.FuncReqHandler(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + return req, nil + }) + } + + limitation := make(chan struct{}, limit) + return goproxy.FuncReqHandler(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + limitation <- struct{}{} + + // Release semaphore when request finishes + go func() { + <-req.Context().Done() + <-limitation + }() + + return req, nil + }) +} diff --git a/ext/limitation/concurrency_test.go b/ext/limitation/concurrency_test.go new file mode 100644 index 000000000..4897e29f9 --- /dev/null +++ b/ext/limitation/concurrency_test.go @@ -0,0 +1,96 @@ +package limitation_test + +import ( + "context" + "net/http" + "testing" + "time" + + "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/limitation" +) + +func TestConcurrentRequests(t *testing.T) { + mockRequest := &http.Request{Host: "test.com"} + ctx := &goproxy.ProxyCtx{} + maximumDuration := 100 * time.Millisecond + + t.Run("empty limitation", func(t *testing.T) { + timer := time.NewTimer(maximumDuration) + defer timer.Stop() + done := make(chan struct{}) + + go func() { + zeroLimiter := limitation.ConcurrentRequests(0) + zeroLimiter.Handle(mockRequest, ctx) + done <- struct{}{} + }() + + select { + case <-timer.C: + t.Error("Limiter took too long") + case <-done: + } + }) + + t.Run("normal limitation", func(t *testing.T) { + timer := time.NewTimer(maximumDuration) + defer timer.Stop() + done := make(chan struct{}) + + go func() { + oneLimiter := limitation.ConcurrentRequests(1) + oneLimiter.Handle(mockRequest, ctx) + done <- struct{}{} + }() + + select { + case <-timer.C: + t.Error("Limiter took too long") + case <-done: + } + }) + + t.Run("more than the limit", func(t *testing.T) { + timer := time.NewTimer(maximumDuration) + defer timer.Stop() + done := make(chan struct{}) + + go func() { + oneLimiter := limitation.ConcurrentRequests(1) + oneLimiter.Handle(mockRequest, ctx) + oneLimiter.Handle(mockRequest, ctx) + done <- struct{}{} + }() + + select { + case <-timer.C: + // Do nothing, we expect to reach the timeout + case <-done: + t.Error("Limiter was too fast") + } + }) + + t.Run("more than the limit but one request finishes", func(t *testing.T) { + timer := time.NewTimer(maximumDuration) + defer timer.Stop() + done := make(chan struct{}) + + timeoutCtx, cancel := context.WithCancel(mockRequest.Context()) + mockRequestWithCancel := mockRequest.WithContext(timeoutCtx) + + go func() { + oneLimiter := limitation.ConcurrentRequests(1) + oneLimiter.Handle(mockRequestWithCancel, ctx) + cancel() + oneLimiter.Handle(mockRequest, ctx) + done <- struct{}{} + }() + + select { + case <-timer.C: + t.Error("Limiter took too long") + case <-done: + } + }) +} diff --git a/go.mod b/go.mod index faf302de5..b76085118 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/elazarl/goproxy go 1.20 require ( - github.com/elazarl/goproxy/ext v0.0.0-20241216102027-e85c60b37433 + github.com/elazarl/goproxy/ext v0.0.0-20241217120900-7711dfa3811c golang.org/x/net v0.32.0 ) diff --git a/go.sum b/go.sum index 77e00d232..1449a4f32 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/elazarl/goproxy/ext v0.0.0-20241216102027-e85c60b37433 h1:zezqs+UN/8nYMOm1PobfrT/FxliWYq5Um1DLxyHA8d0= -github.com/elazarl/goproxy/ext v0.0.0-20241216102027-e85c60b37433/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= +github.com/elazarl/goproxy/ext v0.0.0-20241217120900-7711dfa3811c h1:R+i10jtNSzKJKqEZAYJnR9M8y14k0zrNHqD1xkv/A2M= +github.com/elazarl/goproxy/ext v0.0.0-20241217120900-7711dfa3811c/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= From 73b2f3f620fe13e8fc2fb9bd09523c58e55c163e Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Tue, 17 Dec 2024 20:47:42 +0100 Subject: [PATCH 39/42] Replace ioutil & magic numbers (#571) --- dispatcher.go | 13 ++++- dispatcher_test.go | 2 +- examples/cascadeproxy/main.go | 6 +- .../goproxy-jquery-version/jquery_test.go | 7 ++- examples/goproxy-transparent/transparent.go | 2 +- examples/goproxy-yui-minify/yui.go | 19 +++---- ext/auth/basic.go | 6 +- ext/auth/basic_test.go | 9 ++- ext/html/html.go | 7 +-- ext/html/html_test.go | 4 +- ext/image/image.go | 6 +- https.go | 17 +++--- proxy.go | 2 +- proxy_test.go | 55 +++++++++---------- regretable/regretreader.go | 4 +- regretable/regretreader_test.go | 15 +++-- responses.go | 4 +- signer_test.go | 8 +-- transport/transport.go | 10 ++-- 19 files changed, 99 insertions(+), 97 deletions(-) diff --git a/dispatcher.go b/dispatcher.go index b050af490..9c32530ee 100644 --- a/dispatcher.go +++ b/dispatcher.go @@ -2,7 +2,7 @@ package goproxy import ( "bytes" - "io/ioutil" + "io" "net" "net/http" "regexp" @@ -189,6 +189,7 @@ func StatusCodeIs(codes ...int) RespCondition { // You will use the ReqProxyConds struct to register a ReqHandler, that would filter // the request, only if all the given ReqCondition matched. // Typical usage: +// // proxy.OnRequest(UrlIs("example.com/foo"),UrlMatches(regexp.MustParse(`.*\.exampl.\com\./.*`)).Do(...) func (proxy *ProxyHttpServer) OnRequest(conds ...ReqCondition) *ReqProxyConds { return &ReqProxyConds{proxy, conds} @@ -209,6 +210,7 @@ func (pcond *ReqProxyConds) DoFunc(f func(req *http.Request, ctx *ProxyCtx) (*ht // ReqProxyConds.Do will register the ReqHandler on the proxy, // the ReqHandler will handle the HTTP request if all the conditions // aggregated in the ReqProxyConds are met. Typical usage: +// // proxy.OnRequest().Do(handler) // will call handler.Handle(req,ctx) on every request to the proxy // proxy.OnRequest(cond1,cond2).Do(handler) // // given request to the proxy, will test if cond1.HandleReq(req,ctx) && cond2.HandleReq(req,ctx) are true @@ -235,6 +237,7 @@ func (pcond *ReqProxyConds) Do(h ReqHandler) { // connection. // The ConnectAction struct contains possible tlsConfig that will be used for eavesdropping. If nil, the proxy // will use the default tls configuration. +// // proxy.OnRequest().HandleConnect(goproxy.AlwaysReject) // rejects all CONNECT requests func (pcond *ReqProxyConds) HandleConnect(h HttpsHandler) { pcond.proxy.httpsHandlers = append(pcond.proxy.httpsHandlers, @@ -250,6 +253,7 @@ func (pcond *ReqProxyConds) HandleConnect(h HttpsHandler) { // HandleConnectFunc is equivalent to HandleConnect, // for example, accepting CONNECT request if they contain a password in header +// // io.WriteString(h,password) // passHash := h.Sum(nil) // proxy.OnRequest().HandleConnectFunc(func(host string, ctx *ProxyCtx) (*ConnectAction, string) { @@ -310,6 +314,7 @@ func (pcond *ProxyConds) Do(h RespHandler) { } // OnResponse is used when adding a response-filter to the HTTP proxy, usual pattern is +// // proxy.OnResponse(cond1,cond2).Do(handler) // handler.Handle(resp,ctx) will be used // // if cond1.HandleResp(resp) && cond2.HandleResp(resp) func (proxy *ProxyHttpServer) OnResponse(conds ...RespCondition) *ProxyConds { @@ -318,6 +323,7 @@ func (proxy *ProxyHttpServer) OnResponse(conds ...RespCondition) *ProxyConds { // AlwaysMitm is a HttpsHandler that always eavesdrop https connections, for example to // eavesdrop all https connections to www.google.com, we can use +// // proxy.OnRequest(goproxy.ReqHostIs("www.google.com")).HandleConnect(goproxy.AlwaysMitm) var AlwaysMitm FuncHttpsHandler = func(host string, ctx *ProxyCtx) (*ConnectAction, string) { return MitmConnect, host @@ -325,6 +331,7 @@ var AlwaysMitm FuncHttpsHandler = func(host string, ctx *ProxyCtx) (*ConnectActi // AlwaysReject is a HttpsHandler that drops any CONNECT request, for example, this code will disallow // connections to hosts on any other port than 443 +// // proxy.OnRequest(goproxy.Not(goproxy.ReqHostMatches(regexp.MustCompile(":443$"))). // HandleConnect(goproxy.AlwaysReject) var AlwaysReject FuncHttpsHandler = func(host string, ctx *ProxyCtx) (*ConnectAction, string) { @@ -336,14 +343,14 @@ var AlwaysReject FuncHttpsHandler = func(host string, ctx *ProxyCtx) (*ConnectAc // and will replace the body of the original response with the resulting byte array. func HandleBytes(f func(b []byte, ctx *ProxyCtx) []byte) RespHandler { return FuncRespHandler(func(resp *http.Response, ctx *ProxyCtx) *http.Response { - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) if err != nil { ctx.Warnf("Cannot read response %s", err) return resp } resp.Body.Close() - resp.Body = ioutil.NopCloser(bytes.NewBuffer(f(b, ctx))) + resp.Body = io.NopCloser(bytes.NewBuffer(f(b, ctx))) return resp }) } diff --git a/dispatcher_test.go b/dispatcher_test.go index c7e50245e..a08cb5b5e 100644 --- a/dispatcher_test.go +++ b/dispatcher_test.go @@ -34,7 +34,7 @@ func TestIsLocalHost(t *testing.T) { addr = net.JoinHostPort(host, port) } t.Run(addr, func(t *testing.T) { - req, err := http.NewRequest("GET", "http://"+addr, http.NoBody) + req, err := http.NewRequest(http.MethodGet, "http://"+addr, http.NoBody) if err != nil { t.Fatal(err) } diff --git a/examples/cascadeproxy/main.go b/examples/cascadeproxy/main.go index 194d03a9b..1e8a5705d 100644 --- a/examples/cascadeproxy/main.go +++ b/examples/cascadeproxy/main.go @@ -3,7 +3,7 @@ package main import ( "encoding/base64" "fmt" - "io/ioutil" + "io" "log" "net/http" "net/url" @@ -82,7 +82,7 @@ func main() { // fire a http request: client --> middle proxy --> end proxy --> internet proxyUrl := "http://localhost:8081" - request, err := http.NewRequest("GET", "https://ip.cn", nil) + request, err := http.NewRequest(http.MethodGet, "https://ip.cn", nil) if err != nil { log.Fatalf("new request failed:%v", err) } @@ -94,7 +94,7 @@ func main() { } defer rsp.Body.Close() - data, _ := ioutil.ReadAll(rsp.Body) + data, _ := io.ReadAll(rsp.Body) if rsp.StatusCode != http.StatusOK { log.Fatalf("status %d, data %s", rsp.StatusCode, data) diff --git a/examples/goproxy-jquery-version/jquery_test.go b/examples/goproxy-jquery-version/jquery_test.go index af300aaf3..78b026f4f 100644 --- a/examples/goproxy-jquery-version/jquery_test.go +++ b/examples/goproxy-jquery-version/jquery_test.go @@ -2,11 +2,12 @@ package main import ( "bytes" - "io/ioutil" + "io" "log" "net/http" "net/http/httptest" "net/url" + "os" "strings" "testing" ) @@ -24,7 +25,7 @@ func equal(u, v []string) bool { } func readFile(fname string, t *testing.T) string { - b, err := ioutil.ReadFile(fname) + b, err := os.ReadFile(fname) if err != nil { t.Fatal("readFile", err) } @@ -77,7 +78,7 @@ func get(t *testing.T, server *httptest.Server, client *http.Client, url string) if err != nil { t.Fatal("cannot get proxy", err) } - ioutil.ReadAll(resp.Body) + io.ReadAll(resp.Body) resp.Body.Close() } diff --git a/examples/goproxy-transparent/transparent.go b/examples/goproxy-transparent/transparent.go index b3d03645b..95050b3a9 100644 --- a/examples/goproxy-transparent/transparent.go +++ b/examples/goproxy-transparent/transparent.go @@ -96,7 +96,7 @@ func main() { return } connectReq := &http.Request{ - Method: "CONNECT", + Method: http.MethodConnect, URL: &url.URL{ Opaque: tlsConn.Host(), Host: net.JoinHostPort(tlsConn.Host(), "443"), diff --git a/examples/goproxy-yui-minify/yui.go b/examples/goproxy-yui-minify/yui.go index 0e7eadbb1..7d83e9709 100644 --- a/examples/goproxy-yui-minify/yui.go +++ b/examples/goproxy-yui-minify/yui.go @@ -2,19 +2,18 @@ // using the command line utility YUI compressor http://yui.github.io/yuicompressor/ // Example usage: // -// ./yui -java /usr/local/bin/java -yuicompressor ~/Downloads/yuicompressor-2.4.8.jar -// $ curl -vx localhost:8080 http://golang.org/lib/godoc/godocs.js -// (function(){function g(){var u=$("#search");if(u.length===0){return}function t(){if(.... -// $ curl http://golang.org/lib/godoc/godocs.js | head -n 3 -// // Copyright 2012 The Go Authors. All rights reserved. -// // Use of this source code is governed by a BSD-style -// // license that can be found in the LICENSE file. +// ./yui -java /usr/local/bin/java -yuicompressor ~/Downloads/yuicompressor-2.4.8.jar +// $ curl -vx localhost:8080 http://golang.org/lib/godoc/godocs.js +// (function(){function g(){var u=$("#search");if(u.length===0){return}function t(){if(.... +// $ curl http://golang.org/lib/godoc/godocs.js | head -n 3 +// // Copyright 2012 The Go Authors. All rights reserved. +// // Use of this source code is governed by a BSD-style +// // license that can be found in the LICENSE file. package main import ( "flag" "io" - "io/ioutil" "log" "net/http" "os" @@ -33,7 +32,7 @@ func main() { yuicompressordir := flag.String("yuicompressordir", ".", "a folder to search yuicompressor in, will be ignored if yuicompressor is set") flag.Parse() if *yuicompressor == "" { - files, err := ioutil.ReadDir(*yuicompressordir) + files, err := os.ReadDir(*yuicompressordir) if err != nil { log.Fatal("Cannot find yuicompressor jar") } @@ -76,7 +75,7 @@ func main() { go func() { defer stderr.Close() const kb = 1024 - msg, err := ioutil.ReadAll(&io.LimitedReader{stderr, 50 * kb}) + msg, err := io.ReadAll(&io.LimitedReader{stderr, 50 * kb}) if len(msg) != 0 { ctx.Logf("Error executing yuicompress: %s", string(msg)) } diff --git a/ext/auth/basic.go b/ext/auth/basic.go index 42d555b0e..916d48e54 100644 --- a/ext/auth/basic.go +++ b/ext/auth/basic.go @@ -3,7 +3,7 @@ package auth import ( "bytes" "encoding/base64" - "io/ioutil" + "io" "net/http" "strings" @@ -15,7 +15,7 @@ var unauthorizedMsg = []byte("407 Proxy Authentication Required") func BasicUnauthorized(req *http.Request, realm string) *http.Response { // TODO(elazar): verify realm is well formed return &http.Response{ - StatusCode: 407, + StatusCode: http.StatusProxyAuthRequired, ProtoMajor: 1, ProtoMinor: 1, Request: req, @@ -23,7 +23,7 @@ func BasicUnauthorized(req *http.Request, realm string) *http.Response { "Proxy-Authenticate": []string{"Basic realm=" + realm}, "Proxy-Connection": []string{"close"}, }, - Body: ioutil.NopCloser(bytes.NewBuffer(unauthorizedMsg)), + Body: io.NopCloser(bytes.NewBuffer(unauthorizedMsg)), ContentLength: int64(len(unauthorizedMsg)), } } diff --git a/ext/auth/basic_test.go b/ext/auth/basic_test.go index 118e70ed3..e64184c75 100644 --- a/ext/auth/basic_test.go +++ b/ext/auth/basic_test.go @@ -3,7 +3,6 @@ package auth_test import ( "encoding/base64" "io" - "io/ioutil" "net" "net/http" "net/http/httptest" @@ -115,12 +114,12 @@ func TestBasicAuth(t *testing.T) { if resp.Header.Get("Proxy-Authenticate") != "Basic realm=my_realm" { t.Error("Expected Proxy-Authenticate header got", resp.Header.Get("Proxy-Authenticate")) } - if resp.StatusCode != 407 { + if resp.StatusCode != http.StatusProxyAuthRequired { t.Error("Expected status 407 Proxy Authentication Required, got", resp.Status) } // with auth - req, err := http.NewRequest("GET", background.URL, nil) + req, err := http.NewRequest(http.MethodGet, background.URL, nil) if err != nil { t.Fatal(err) } @@ -130,10 +129,10 @@ func TestBasicAuth(t *testing.T) { if err != nil { t.Fatal(err) } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { t.Error("Expected status 200 OK, got", resp.Status) } - msg, err := ioutil.ReadAll(resp.Body) + msg, err := io.ReadAll(resp.Body) if err != nil { t.Fatal(err) } diff --git a/ext/html/html.go b/ext/html/html.go index 2dc294ec0..ca3128aed 100644 --- a/ext/html/html.go +++ b/ext/html/html.go @@ -5,7 +5,6 @@ import ( "bytes" "errors" "io" - "io/ioutil" "net/http" "strings" @@ -37,7 +36,7 @@ var IsWebRelatedText goproxy.RespCondition = goproxy.ContentTypeIs("text/html", // guessing Html charset encoding from the tags is not yet implemented. func HandleString(f func(s string, ctx *goproxy.ProxyCtx) string) goproxy.RespHandler { return HandleStringReader(func(r io.Reader, ctx *goproxy.ProxyCtx) io.Reader { - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { ctx.Warnf("Cannot read string from resp body: %v", err) return r @@ -74,10 +73,10 @@ func HandleStringReader(f func(r io.Reader, ctx *goproxy.ProxyCtx) io.Reader) go return resp } newr := charset.NewTranslatingReader(f(r, ctx), tr) - resp.Body = &readFirstCloseBoth{ioutil.NopCloser(newr), resp.Body} + resp.Body = &readFirstCloseBoth{io.NopCloser(newr), resp.Body} } else { //no translation is needed, already at utf-8 - resp.Body = &readFirstCloseBoth{ioutil.NopCloser(f(resp.Body, ctx)), resp.Body} + resp.Body = &readFirstCloseBoth{io.NopCloser(f(resp.Body, ctx)), resp.Body} } return resp }) diff --git a/ext/html/html_test.go b/ext/html/html_test.go index 9c876f752..0949c1ca3 100644 --- a/ext/html/html_test.go +++ b/ext/html/html_test.go @@ -3,7 +3,7 @@ package goproxy_html_test import ( "github.com/elazarl/goproxy" "github.com/elazarl/goproxy/ext/html" - "io/ioutil" + "io" "net/http" "net/http/httptest" "net/url" @@ -39,7 +39,7 @@ func TestCharset(t *testing.T) { if err != nil { t.Fatal("GET:", err) } - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) if err != nil { t.Fatal("readAll:", err) } diff --git a/ext/image/image.go b/ext/image/image.go index cd9ac8a4d..5de7e59b4 100644 --- a/ext/image/image.go +++ b/ext/image/image.go @@ -8,7 +8,7 @@ import ( _ "image/gif" "image/jpeg" "image/png" - "io/ioutil" + "io" "net/http" ) @@ -25,7 +25,7 @@ func HandleImage(f func(img image.Image, ctx *ProxyCtx) image.Image) RespHandler if !RespIsImage.HandleResp(resp, ctx) { return resp } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { // we might get 304 - not modified response without data return resp } @@ -72,7 +72,7 @@ func HandleImage(f func(img image.Image, ctx *ProxyCtx) image.Image) RespHandler default: panic("unhandlable type" + contentType) } - resp.Body = ioutil.NopCloser(buf) + resp.Body = io.NopCloser(buf) return resp }) } diff --git a/https.go b/https.go index cde6768b1..eff755328 100644 --- a/https.go +++ b/https.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net" "net/http" "net/url" @@ -309,7 +308,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request return } - if resp.Request.Method == "HEAD" { + if resp.Request.Method == http.MethodHead { // don't change Content-Length for HEAD request } else { // Since we don't know the length of resp, return chunked encoded response @@ -328,7 +327,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request return } - if resp.Request.Method == "HEAD" { + if resp.Request.Method == http.MethodHead { // Don't write out a response body for HEAD request } else { chunked := newChunkedWriter(rawClientTls) @@ -413,7 +412,7 @@ func (proxy *ProxyHttpServer) NewConnectDialToProxyWithHandler(https_proxy strin } return func(network, addr string) (net.Conn, error) { connectReq := &http.Request{ - Method: "CONNECT", + Method: http.MethodConnect, URL: &url.URL{Opaque: addr}, Host: addr, Header: make(http.Header), @@ -436,8 +435,8 @@ func (proxy *ProxyHttpServer) NewConnectDialToProxyWithHandler(https_proxy strin return nil, err } defer resp.Body.Close() - if resp.StatusCode != 200 { - resp, err := ioutil.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + resp, err := io.ReadAll(resp.Body) if err != nil { return nil, err } @@ -458,7 +457,7 @@ func (proxy *ProxyHttpServer) NewConnectDialToProxyWithHandler(https_proxy strin } c = tls.Client(c, proxy.Tr.TLSClientConfig) connectReq := &http.Request{ - Method: "CONNECT", + Method: http.MethodConnect, URL: &url.URL{Opaque: addr}, Host: addr, Header: make(http.Header), @@ -477,8 +476,8 @@ func (proxy *ProxyHttpServer) NewConnectDialToProxyWithHandler(https_proxy strin return nil, err } defer resp.Body.Close() - if resp.StatusCode != 200 { - body, err := ioutil.ReadAll(io.LimitReader(resp.Body, 500)) + if resp.StatusCode != http.StatusOK { + body, err := io.ReadAll(io.LimitReader(resp.Body, 500)) if err != nil { return nil, err } diff --git a/proxy.go b/proxy.go index 7542656a8..496b46a70 100644 --- a/proxy.go +++ b/proxy.go @@ -126,7 +126,7 @@ func (fw flushWriter) Write(p []byte) (int, error) { // Standard net/http function. Shouldn't be used directly, http.Serve will use it. func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { //r.Header["X-Forwarded-For"] = w.RemoteAddr() - if r.Method == "CONNECT" { + if r.Method == http.MethodConnect { proxy.handleHttps(w, r) } else { ctx := &ProxyCtx{Req: r, Session: atomic.AddInt64(&proxy.sess, 1), Proxy: proxy} diff --git a/proxy_test.go b/proxy_test.go index 429c666bb..426339d3a 100644 --- a/proxy_test.go +++ b/proxy_test.go @@ -10,7 +10,6 @@ import ( "fmt" "image" "io" - "io/ioutil" "net" "net/http" "net/http/httptest" @@ -59,7 +58,7 @@ func get(url string, client *http.Client) ([]byte, error) { if err != nil { return nil, err } - txt, err := ioutil.ReadAll(resp.Body) + txt, err := io.ReadAll(resp.Body) defer resp.Body.Close() if err != nil { return nil, err @@ -139,7 +138,7 @@ func TestReplaceResponse(t *testing.T) { proxy := goproxy.NewProxyHttpServer() proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { resp.StatusCode = http.StatusOK - resp.Body = ioutil.NopCloser(bytes.NewBufferString("chico")) + resp.Body = io.NopCloser(bytes.NewBufferString("chico")) return resp }) @@ -155,7 +154,7 @@ func TestReplaceReponseForUrl(t *testing.T) { proxy := goproxy.NewProxyHttpServer() proxy.OnResponse(goproxy.UrlIs("/koko")).DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { resp.StatusCode = http.StatusOK - resp.Body = ioutil.NopCloser(bytes.NewBufferString("chico")) + resp.Body = io.NopCloser(bytes.NewBufferString("chico")) return resp }) @@ -180,7 +179,7 @@ func TestOneShotFileServer(t *testing.T) { t.Fatal("Cannot find", file) } if resp, err := client.Get(fs.URL + "/" + file); err == nil { - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) if err != nil { t.Fatal("got", string(b)) } @@ -223,7 +222,7 @@ func TestContentType(t *testing.T) { } func getImage(file string, t *testing.T) image.Image { - newimage, err := ioutil.ReadFile(file) + newimage, err := os.ReadFile(file) if err != nil { t.Fatal("Cannot read file", file, err) } @@ -235,14 +234,14 @@ func getImage(file string, t *testing.T) image.Image { } func readAll(r io.Reader, t *testing.T) []byte { - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { t.Fatal("Cannot read", err) } return b } func readFile(file string, t *testing.T) []byte { - b, err := ioutil.ReadFile(file) + b, err := os.ReadFile(file) if err != nil { t.Fatal("Cannot read", err) } @@ -342,7 +341,7 @@ func TestChangeResp(t *testing.T) { proxy := goproxy.NewProxyHttpServer() proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { resp.Body.Read([]byte{0}) - resp.Body = ioutil.NopCloser(new(bytes.Buffer)) + resp.Body = io.NopCloser(new(bytes.Buffer)) return resp }) @@ -353,7 +352,7 @@ func TestChangeResp(t *testing.T) { if err != nil { t.Fatal(err) } - ioutil.ReadAll(resp.Body) + io.ReadAll(resp.Body) _, err = client.Get(localFile("/bobo")) if err != nil { t.Fatal(err) @@ -418,7 +417,7 @@ func TestSimpleMitm(t *testing.T) { creq.Write(c2) c2buf := bufio.NewReader(c2) resp, err := http.ReadResponse(c2buf, creq) - if err != nil || resp.StatusCode != 200 { + if err != nil || resp.StatusCode != http.StatusOK { t.Fatal("Cannot CONNECT through proxy", err) } c2tls := tls.Client(c2, &tls.Config{InsecureSkipVerify: true}) @@ -521,7 +520,7 @@ func TestIcyResponse(t *testing.T) { panicOnErr(err, "dial") defer c.Close() req.WriteProxy(c) - raw, err := ioutil.ReadAll(c) + raw, err := io.ReadAll(c) panicOnErr(err, "readAll") if string(raw) != "ICY 200 OK\r\n\r\nblablabla" { t.Error("Proxy did not send the malformed response received") @@ -543,7 +542,7 @@ func TestNoProxyHeaders(t *testing.T) { s := httptest.NewServer(VerifyNoProxyHeaders{t}) client, l := oneShotProxy(goproxy.NewProxyHttpServer(), t) defer l.Close() - req, err := http.NewRequest("GET", s.URL, nil) + req, err := http.NewRequest(http.MethodGet, s.URL, nil) panicOnErr(err, "bad request") req.Header.Add("Connection", "close") req.Header.Add("Proxy-Connection", "close") @@ -558,7 +557,7 @@ func TestNoProxyHeadersHttps(t *testing.T) { proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) client, l := oneShotProxy(proxy, t) defer l.Close() - req, err := http.NewRequest("GET", s.URL, nil) + req, err := http.NewRequest(http.MethodGet, s.URL, nil) panicOnErr(err, "bad request") req.Header.Add("Connection", "close") req.Header.Add("Proxy-Connection", "close") @@ -604,11 +603,11 @@ func TestChunkedResponse(t *testing.T) { c, err := net.Dial("tcp", "localhost:10234") panicOnErr(err, "dial") defer c.Close() - req, _ := http.NewRequest("GET", "/", nil) + req, _ := http.NewRequest(http.MethodGet, "/", nil) req.Write(c) resp, err := http.ReadResponse(bufio.NewReader(c), req) panicOnErr(err, "readresp") - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) panicOnErr(err, "readall") expected := "This is the data in the first chunk\r\nand this is the second one\r\nconsequence" if string(b) != expected { @@ -618,13 +617,13 @@ func TestChunkedResponse(t *testing.T) { proxy := goproxy.NewProxyHttpServer() proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { panicOnErr(ctx.Error, "error reading output") - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) resp.Body.Close() panicOnErr(err, "readall onresp") if enc := resp.Header.Get("Transfer-Encoding"); enc != "" { t.Fatal("Chunked response should be received as plaintext", enc) } - resp.Body = ioutil.NopCloser(bytes.NewBufferString(strings.Replace(string(b), "e", "E", -1))) + resp.Body = io.NopCloser(bytes.NewBufferString(strings.Replace(string(b), "e", "E", -1))) return resp }) @@ -633,7 +632,7 @@ func TestChunkedResponse(t *testing.T) { resp, err = client.Get("http://localhost:10234/") panicOnErr(err, "client.Get") - b, err = ioutil.ReadAll(resp.Body) + b, err = io.ReadAll(resp.Body) panicOnErr(err, "readall proxy") if string(b) != strings.Replace(expected, "e", "E", -1) { t.Error("expected", expected, "w/ e->E. Got", string(b)) @@ -644,9 +643,9 @@ func TestGoproxyThroughProxy(t *testing.T) { proxy := goproxy.NewProxyHttpServer() proxy2 := goproxy.NewProxyHttpServer() doubleString := func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) panicOnErr(err, "readAll resp") - resp.Body = ioutil.NopCloser(bytes.NewBufferString(string(b) + " " + string(b))) + resp.Body = io.NopCloser(bytes.NewBufferString(string(b) + " " + string(b))) return resp } proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) @@ -693,12 +692,12 @@ func TestGoproxyHijackConnect(t *testing.T) { } func readResponse(buf *bufio.Reader) string { - req, err := http.NewRequest("GET", srv.URL, nil) + req, err := http.NewRequest(http.MethodGet, srv.URL, nil) panicOnErr(err, "NewRequest") resp, err := http.ReadResponse(buf, req) panicOnErr(err, "resp.Read") defer resp.Body.Close() - txt, err := ioutil.ReadAll(resp.Body) + txt, err := io.ReadAll(resp.Body) panicOnErr(err, "resp.Read") return string(txt) } @@ -888,7 +887,7 @@ func TestHttpsMitmURLRewrite(t *testing.T) { defer s.Close() fullURL := scheme + "://" + tc.Host + tc.RawPath - req, err := http.NewRequest("GET", fullURL, nil) + req, err := http.NewRequest(http.MethodGet, fullURL, nil) if err != nil { t.Fatal(err) } @@ -904,7 +903,7 @@ func TestHttpsMitmURLRewrite(t *testing.T) { t.Fatal(err) } - b, err := ioutil.ReadAll(resp.Body) + b, err := io.ReadAll(resp.Body) defer resp.Body.Close() if err != nil { t.Fatal(err) @@ -949,7 +948,7 @@ func TestSimpleHttpRequest(t *testing.T) { resp, err := client.Get("http://example.com") if err != nil { t.Error("Error requesting http site", err) - } else if resp.StatusCode != 200 { + } else if resp.StatusCode != http.StatusOK { t.Error("Non-OK status requesting http site", err) } resp, _ = client.Get("http://example.invalid") @@ -982,7 +981,7 @@ func TestResponseContentLength(t *testing.T) { proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { buf := &bytes.Buffer{} buf.Write([]byte("change")) - resp.Body = ioutil.NopCloser(buf) + resp.Body = io.NopCloser(buf) return resp }) proxySrv := httptest.NewServer(proxy) @@ -997,7 +996,7 @@ func TestResponseContentLength(t *testing.T) { req, _ := http.NewRequest(http.MethodGet, srv.URL, nil) resp, _ := http.DefaultClient.Do(req) - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) defer resp.Body.Close() if int64(len(body)) != resp.ContentLength { diff --git a/regretable/regretreader.go b/regretable/regretreader.go index 29b60add7..3769a9c12 100644 --- a/regretable/regretreader.go +++ b/regretable/regretreader.go @@ -6,7 +6,8 @@ import ( // A RegretableReader will allow you to read from a reader, and then // to "regret" reading it, and push back everything you've read. -// For example, +// For example: +// // rb := NewRegretableReader(bytes.NewBuffer([]byte{1,2,3})) // var b = make([]byte,1) // rb.Read(b) // b[0] = 1 @@ -52,6 +53,7 @@ func (rb *RegretableReader) Regret() { } // Will "forget" everything read so far. +// // rb := NewRegretableReader(bytes.NewBuffer([]byte{1,2,3})) // var b = make([]byte,1) // rb.Read(b) // b[0] = 1 diff --git a/regretable/regretreader_test.go b/regretable/regretreader_test.go index bc631b455..e5def117a 100644 --- a/regretable/regretreader_test.go +++ b/regretable/regretreader_test.go @@ -4,7 +4,6 @@ import ( "bytes" . "github.com/elazarl/goproxy/regretable" "io" - "io/ioutil" "strings" "testing" ) @@ -19,7 +18,7 @@ func TestRegretableReader(t *testing.T) { mb.Read(fivebytes) mb.Regret() - s, _ := ioutil.ReadAll(mb) + s, _ := io.ReadAll(mb) if string(s) != word { t.Errorf("Uncommitted read is gone, [%d,%d] actual '%v' expected '%v'\n", len(s), len(word), string(s), word) } @@ -35,7 +34,7 @@ func TestRegretableEmptyRead(t *testing.T) { mb.Read(zero) mb.Regret() - s, err := ioutil.ReadAll(mb) + s, err := io.ReadAll(mb) if string(s) != word { t.Error("Uncommitted read is gone, actual:", string(s), "expected:", word, "err:", err) } @@ -55,7 +54,7 @@ func TestRegretableAlsoEmptyRead(t *testing.T) { mb.Read(five) mb.Regret() - s, _ := ioutil.ReadAll(mb) + s, _ := io.ReadAll(mb) if string(s) != word { t.Error("Uncommitted read is gone", string(s), "expected", word) } @@ -71,7 +70,7 @@ func TestRegretableRegretBeforeRead(t *testing.T) { mb.Regret() mb.Read(five) - s, err := ioutil.ReadAll(mb) + s, err := io.ReadAll(mb) if string(s) != "678" { t.Error("Uncommitted read is gone", string(s), len(string(s)), "expected", "678", len("678"), "err:", err) } @@ -87,7 +86,7 @@ func TestRegretableFullRead(t *testing.T) { mb.Read(twenty) mb.Regret() - s, _ := ioutil.ReadAll(mb) + s, _ := io.ReadAll(mb) if string(s) != word { t.Error("Uncommitted read is gone", string(s), len(string(s)), "expected", word, len(word)) } @@ -100,7 +99,7 @@ func assertEqual(t *testing.T, expected, actual string) { } func assertReadAll(t *testing.T, r io.Reader) string { - s, err := ioutil.ReadAll(r) + s, err := io.ReadAll(r) if err != nil { t.Fatal("error when reading", err) } @@ -148,7 +147,7 @@ func TestRegretableCloserSizeRegrets(t *testing.T) { }() buf := new(bytes.Buffer) buf.WriteString("123456") - mb := NewRegretableReaderCloserSize(ioutil.NopCloser(buf), 3) + mb := NewRegretableReaderCloserSize(io.NopCloser(buf), 3) mb.Read(make([]byte, 4)) mb.Regret() } diff --git a/responses.go b/responses.go index e1bf28fc2..4081c7e42 100644 --- a/responses.go +++ b/responses.go @@ -2,7 +2,7 @@ package goproxy import ( "bytes" - "io/ioutil" + "io" "net/http" ) @@ -24,7 +24,7 @@ func NewResponse(r *http.Request, contentType string, status int, body string) * resp.Status = http.StatusText(status) buf := bytes.NewBufferString(body) resp.ContentLength = int64(buf.Len()) - resp.Body = ioutil.NopCloser(buf) + resp.Body = io.NopCloser(buf) return resp } diff --git a/signer_test.go b/signer_test.go index 0149190b1..d584b52cd 100644 --- a/signer_test.go +++ b/signer_test.go @@ -3,7 +3,7 @@ package goproxy import ( "crypto/tls" "crypto/x509" - "io/ioutil" + "io" "net/http" "net/http/httptest" "os" @@ -70,12 +70,12 @@ func testSignerTls(t *testing.T, ca tls.Certificate) { TLSClientConfig: &tls.Config{RootCAs: certpool}, } asLocalhost := strings.Replace(server.URL, "127.0.0.1", "localhost", -1) - req, err := http.NewRequest("GET", asLocalhost, nil) + req, err := http.NewRequest(http.MethodGet, asLocalhost, nil) orFatal("NewRequest", err, t) resp, err := tr.RoundTrip(req) orFatal("RoundTrip", err, t) - txt, err := ioutil.ReadAll(resp.Body) - orFatal("ioutil.ReadAll", err, t) + txt, err := io.ReadAll(resp.Body) + orFatal("io.ReadAll", err, t) if string(txt) != expected { t.Errorf("Expected '%s' got '%s'", expected, string(txt)) } diff --git a/transport/transport.go b/transport/transport.go index d8071618d..0706035a1 100644 --- a/transport/transport.go +++ b/transport/transport.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "log" "net" "net/http" @@ -367,7 +366,7 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, error) { } case cm.targetScheme == "https": connectReq := &http.Request{ - Method: "CONNECT", + Method: http.MethodConnect, URL: &url.URL{Opaque: cm.targetAddr}, Host: cm.targetAddr, Header: make(http.Header), @@ -386,7 +385,7 @@ func (t *Transport) getConn(cm *connectMethod) (*persistConn, error) { conn.Close() return nil, err } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { f := strings.SplitN(resp.Status, " ", 2) conn.Close() return nil, errors.New(f[1]) @@ -471,7 +470,6 @@ func useProxy(addr string) bool { // http://proxy.com|http http to proxy, http to anywhere after that // // Note: no support to https to the proxy yet. -// type connectMethod struct { proxyURL *url.URL // nil for no proxy, else full proxy URL targetScheme string // "http" or "https" @@ -578,7 +576,7 @@ func (pc *persistConn) readLoop() { if err != nil { pc.close() } else { - hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 + hasBody := rc.req.Method != http.MethodHead && resp.ContentLength != 0 if rc.addedGzip && hasBody && resp.Header.Get("Content-Encoding") == "gzip" { resp.Header.Del("Content-Encoding") resp.Header.Del("Content-Length") @@ -782,6 +780,6 @@ type discardOnCloseReadCloser struct { } func (d *discardOnCloseReadCloser) Close() error { - io.Copy(ioutil.Discard, d.ReadCloser) // ignore errors; likely invalid or already closed + io.Copy(io.Discard, d.ReadCloser) // ignore errors; likely invalid or already closed return d.ReadCloser.Close() } From bd0b840b596f3cb67f026bdc7183447d5b0f7061 Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Wed, 18 Dec 2024 12:32:57 +0100 Subject: [PATCH 40/42] Optimize CA certificate usage (#574) * Use Leaf certificate optimization and return the whole CA chain * Remove explicit empty handler initialization * Initialize Goproxy CA directly inside init() Co-authored-by: Roman MANZ --- certs.go | 18 ++++++++++++------ proxy.go | 7 +------ signer.go | 31 ++++++++++++++++++++++--------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/certs.go b/certs.go index 4731971e7..4a8bdda3f 100644 --- a/certs.go +++ b/certs.go @@ -5,13 +5,21 @@ import ( "crypto/x509" ) +var GoproxyCa tls.Certificate + func init() { - if goproxyCaErr != nil { - panic("Error parsing builtin CA " + goproxyCaErr.Error()) - } + // When we included the embedded certificate inside this file, we made + // sure that it was valid. + // If there is an error here, this is a really exceptional case that requires + // a panic. It should NEVER happen! var err error + GoproxyCa, err = tls.X509KeyPair(CA_CERT, CA_KEY) + if err != nil { + panic("Error parsing builtin CA: " + err.Error()) + } + if GoproxyCa.Leaf, err = x509.ParseCertificate(GoproxyCa.Certificate[0]); err != nil { - panic("Error parsing builtin CA " + err.Error()) + panic("Error parsing builtin CA leaf: " + err.Error()) } } @@ -107,5 +115,3 @@ pmcjjocD/UCCSuHgbAYNNnO/JdhnSylz1tIg26I+2iLNyeTKIepSNlsBxnkLmqM1 cj/azKBaT04IOMLaN8xfSqitJYSraWMVNgGJM5vfcVaivZnNh0lZBv+qu6YkdM88 4/avCJ8IutT+FcMM+GbGazOm5ALWqUyhrnbLGc4CQMPfe7Il6NxwcrOxT8w= -----END RSA PRIVATE KEY-----`) - -var GoproxyCa, goproxyCaErr = tls.X509KeyPair(CA_CERT, CA_KEY) diff --git a/proxy.go b/proxy.go index 496b46a70..546d938db 100644 --- a/proxy.go +++ b/proxy.go @@ -213,17 +213,12 @@ func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) // NewProxyHttpServer creates and returns a proxy server, logging to stderr by default func NewProxyHttpServer() *ProxyHttpServer { proxy := ProxyHttpServer{ - Logger: log.New(os.Stderr, "", log.LstdFlags), - reqHandlers: []ReqHandler{}, - respHandlers: []RespHandler{}, - httpsHandlers: []HttpsHandler{}, + Logger: log.New(os.Stderr, "", log.LstdFlags), NonproxyHandler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { http.Error(w, "This is a proxy server. Does not respond to non-proxy requests.", 500) }), Tr: &http.Transport{TLSClientConfig: tlsClientSkipVerify, Proxy: http.ProxyFromEnvironment}, } - proxy.ConnectDial = dialerFromEnv(&proxy) - return &proxy } diff --git a/signer.go b/signer.go index de9efcf5e..e8704c8d9 100644 --- a/signer.go +++ b/signer.go @@ -38,11 +38,13 @@ func hashSortedBigInt(lst []string) *big.Int { var goproxySignerVersion = ":goroxy1" func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err error) { - var x509ca *x509.Certificate - - // Use the provided ca and not the global GoproxyCa for certificate generation. - if x509ca, err = x509.ParseCertificate(ca.Certificate[0]); err != nil { - return + // Use the provided CA for certificate generation. + // Use already parsed Leaf certificate when present. + x509ca := ca.Leaf + if x509ca == nil { + if x509ca, err = x509.ParseCertificate(ca.Certificate[0]); err != nil { + return nil, err + } } start := time.Unix(time.Now().Unix()-2592000, 0) // 2592000 = 30 day @@ -97,12 +99,23 @@ func signHost(ca tls.Certificate, hosts []string) (cert *tls.Certificate, err er err = fmt.Errorf("unsupported key type %T", ca.PrivateKey) } - var derBytes []byte - if derBytes, err = x509.CreateCertificate(&csprng, &template, x509ca, certpriv.Public(), ca.PrivateKey); err != nil { - return + derBytes, err := x509.CreateCertificate(&csprng, &template, x509ca, certpriv.Public(), ca.PrivateKey) + if err != nil { + return nil, err } + + // Save an already parsed leaf certificate to use less CPU + // when it will be used + leafCert, err := x509.ParseCertificate(derBytes) + if err != nil { + return nil, err + } + + certBytes := [][]byte{derBytes} + certBytes = append(certBytes, ca.Certificate...) return &tls.Certificate{ - Certificate: [][]byte{derBytes, ca.Certificate[0]}, + Certificate: certBytes, PrivateKey: certpriv, + Leaf: leafCert, }, nil } From 8cb188a4a09cd44288e1c4397d58335f3c6fd733 Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Wed, 18 Dec 2024 12:34:50 +0100 Subject: [PATCH 41/42] Follow RFC7230 specification for HTTP body (#572) * HTTP MITM: establish a connection with the remote server only if the proxy doesn't produce a response * Follow RFC7230 rules for HTTP request data --- https.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/https.go b/https.go index eff755328..cb367b209 100644 --- a/https.go +++ b/https.go @@ -157,14 +157,12 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request case ConnectHTTPMitm: proxyClient.Write([]byte("HTTP/1.0 200 OK\r\n\r\n")) ctx.Logf("Assuming CONNECT is plain HTTP tunneling, mitm proxying it") - targetSiteCon, err := proxy.connectDial(ctx, "tcp", host) - if err != nil { - ctx.Warnf("Error dialing to %s: %s", host, err.Error()) - return - } + + var targetSiteCon net.Conn + var remote *bufio.Reader + for { client := bufio.NewReader(proxyClient) - remote := bufio.NewReader(targetSiteCon) req, err := http.ReadRequest(client) if err != nil && err != io.EOF { ctx.Warnf("cannot read request of MITM HTTP client: %+#v", err) @@ -174,6 +172,17 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request } req, resp := proxy.filterRequest(req, ctx) if resp == nil { + // Establish a connection with the remote server only if the proxy + // doesn't produce a response + if targetSiteCon == nil { + targetSiteCon, err = proxy.connectDial(ctx, "tcp", host) + if err != nil { + ctx.Warnf("Error dialing to %s: %s", host, err.Error()) + return + } + remote = bufio.NewReader(targetSiteCon) + } + if err := req.Write(targetSiteCon); err != nil { httpError(proxyClient, ctx, err) return @@ -310,6 +319,11 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request if resp.Request.Method == http.MethodHead { // don't change Content-Length for HEAD request + } else if (resp.StatusCode >= 100 && resp.StatusCode < 200) || + resp.StatusCode == http.StatusNoContent { + // RFC7230: A server MUST NOT send a Content-Length header field in any response + // with a status code of 1xx (Informational) or 204 (No Content) + resp.Header.Del("Content-Length") } else { // Since we don't know the length of resp, return chunked encoded response // TODO: use a more reasonable scheme @@ -327,8 +341,12 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request return } - if resp.Request.Method == http.MethodHead { - // Don't write out a response body for HEAD request + if resp.Request.Method == http.MethodHead || + (resp.StatusCode >= 100 && resp.StatusCode < 200) || + resp.StatusCode == http.StatusNoContent || + resp.StatusCode == http.StatusNotModified { + // Don't write out a response body, when it's not allowed + // in RFC7230 } else { chunked := newChunkedWriter(rawClientTls) if _, err := io.Copy(chunked, resp.Body); err != nil { From 3840d13af06f183efb6f9bdc326dc4bb54ecd930 Mon Sep 17 00:00:00 2001 From: Erik Pellizzon Date: Wed, 18 Dec 2024 18:15:10 +0100 Subject: [PATCH 42/42] Examples Refactoring (#577) * Delete old yui example * Refactor base goproxy examples * Refactor certstorage example * Delete eavesdropper example * Refactor custom ca example * Fix cascadeproxy example * Add hijack example * Refactor request filtering example * Refactor image example * Refactor sslstrip example * Edit README files * Refactor jquery example * Refactor websockets example --- examples/{goproxy-basic => base}/README.md | 14 +-- examples/{goproxy-basic => base}/main.go | 5 +- examples/cascadeproxy/README.md | 17 ++++ examples/cascadeproxy/main.go | 77 ++++++---------- examples/certstorage/README.md | 11 +++ examples/certstorage/cache.go | 39 ++++++++ .../main.go | 6 +- examples/customca/README.md | 10 +++ .../{goproxy-customca => customca}/cert.go | 27 +----- examples/customca/main.go | 43 +++++++++ examples/goproxy-certstorage/README.md | 5 -- examples/goproxy-certstorage/storage.go | 34 ------- examples/goproxy-customca/main.go | 20 ----- examples/goproxy-eavesdropper/main.go | 57 ------------ examples/goproxy-jquery-version/README.md | 31 ------- examples/goproxy-stats/README.md | 43 --------- examples/goproxy-stats/main.go | 67 -------------- examples/goproxy-yui-minify/yui.go | 90 ------------------- examples/hijack/README.md | 37 ++++++++ examples/hijack/main.go | 46 ++++++++++ examples/html-parser/README.md | 34 +++++++ .../jquery1.html | 0 .../jquery2.html | 0 .../jquery_homepage.html | 0 .../jquery_test.go | 4 +- .../main.go | 4 +- .../php_man.html | 0 .../w3schools.html | 0 examples/image-manipulation/README.md | 6 ++ .../main.go | 6 +- examples/remove-https/README.md | 6 ++ .../sslstrip.go => remove-https/main.go} | 4 +- .../README.md | 12 +-- .../noreddit.go | 0 examples/socket-keepalive/README.md | 9 ++ .../keepalive.go} | 8 +- examples/websockets/README.md | 9 ++ .../localhost-key.pem | 0 .../localhost.pem | 0 .../main.go | 65 +++++++------- 40 files changed, 360 insertions(+), 486 deletions(-) rename examples/{goproxy-basic => base}/README.md (69%) rename examples/{goproxy-basic => base}/main.go (99%) create mode 100644 examples/cascadeproxy/README.md create mode 100644 examples/certstorage/README.md create mode 100644 examples/certstorage/cache.go rename examples/{goproxy-certstorage => certstorage}/main.go (87%) create mode 100644 examples/customca/README.md rename examples/{goproxy-customca => customca}/cert.go (75%) create mode 100644 examples/customca/main.go delete mode 100644 examples/goproxy-certstorage/README.md delete mode 100644 examples/goproxy-certstorage/storage.go delete mode 100644 examples/goproxy-customca/main.go delete mode 100644 examples/goproxy-eavesdropper/main.go delete mode 100644 examples/goproxy-jquery-version/README.md delete mode 100644 examples/goproxy-stats/README.md delete mode 100644 examples/goproxy-stats/main.go delete mode 100644 examples/goproxy-yui-minify/yui.go create mode 100644 examples/hijack/README.md create mode 100644 examples/hijack/main.go create mode 100644 examples/html-parser/README.md rename examples/{goproxy-jquery-version => html-parser}/jquery1.html (100%) rename examples/{goproxy-jquery-version => html-parser}/jquery2.html (100%) rename examples/{goproxy-jquery-version => html-parser}/jquery_homepage.html (100%) rename examples/{goproxy-jquery-version => html-parser}/jquery_test.go (98%) rename examples/{goproxy-jquery-version => html-parser}/main.go (94%) rename examples/{goproxy-jquery-version => html-parser}/php_man.html (100%) rename examples/{goproxy-jquery-version => html-parser}/w3schools.html (100%) create mode 100644 examples/image-manipulation/README.md rename examples/{goproxy-upside-down-ternet => image-manipulation}/main.go (83%) create mode 100644 examples/remove-https/README.md rename examples/{goproxy-sslstrip/sslstrip.go => remove-https/main.go} (82%) rename examples/{goproxy-no-reddit-at-worktime => request-filtering}/README.md (57%) rename examples/{goproxy-no-reddit-at-worktime => request-filtering}/noreddit.go (100%) create mode 100644 examples/socket-keepalive/README.md rename examples/{goproxy-sokeepalive/sokeepalive.go => socket-keepalive/keepalive.go} (90%) create mode 100644 examples/websockets/README.md rename examples/{goproxy-websockets => websockets}/localhost-key.pem (100%) rename examples/{goproxy-websockets => websockets}/localhost.pem (100%) rename examples/{goproxy-websockets => websockets}/main.go (65%) diff --git a/examples/goproxy-basic/README.md b/examples/base/README.md similarity index 69% rename from examples/goproxy-basic/README.md rename to examples/base/README.md index 8778f2a75..72e771d4a 100644 --- a/examples/goproxy-basic/README.md +++ b/examples/base/README.md @@ -1,23 +1,23 @@ # Simple HTTP Proxy -`goproxy-basic` starts an HTTP proxy on :8080. It only handles explicit CONNECT -requests. +This example contains a base HTTP proxy server that listens on port :8080. +It only handles explicit CONNECT requests. Start it in one shell: ```sh -goproxy-basic -v +go build +base -v ``` -Fetch goproxy homepage in another: - +Fetch a website using the proxy: ```sh http_proxy=http://127.0.0.1:8080 wget -O - \ http://ripper234.com/p/introducing-goproxy-light-http-proxy/ ``` -The homepage HTML content should be displayed in the console. The proxy should -have logged the request being processed: +The homepage HTML content should be displayed in the console. +The proxy should have logged the request being processed: ```sh 2015/04/09 18:19:17 [001] INFO: Got request /p/introducing-goproxy-light-http-proxy/ ripper234.com GET http://ripper234.com/p/introducing-goproxy-light-http-proxy/ diff --git a/examples/goproxy-basic/main.go b/examples/base/main.go similarity index 99% rename from examples/goproxy-basic/main.go rename to examples/base/main.go index 22dc4a907..6863644bf 100644 --- a/examples/goproxy-basic/main.go +++ b/examples/base/main.go @@ -1,10 +1,11 @@ package main import ( - "github.com/elazarl/goproxy" - "log" "flag" + "log" "net/http" + + "github.com/elazarl/goproxy" ) func main() { diff --git a/examples/cascadeproxy/README.md b/examples/cascadeproxy/README.md new file mode 100644 index 000000000..7b49326db --- /dev/null +++ b/examples/cascadeproxy/README.md @@ -0,0 +1,17 @@ +# CascadeProxy + +`CascadeProxy` is an example that shows an aggregator server that forwards +the requests to another proxy server (end proxy). + +Diagram: +``` +client --> middle proxy --> end proxy --> internet +``` + +This example starts both proxy servers using goproxy, the middle one +listens on port `8081`, and the end one on port `8082`. + +The middle proxy must be an HTTP server, since we use goproxy that +expose only it. +The end proxy can be any type of proxy supported by Go, including SOCKS5, +there is a comment in the part where you can put its address. diff --git a/examples/cascadeproxy/main.go b/examples/cascadeproxy/main.go index 1e8a5705d..2abd4a095 100644 --- a/examples/cascadeproxy/main.go +++ b/examples/cascadeproxy/main.go @@ -1,76 +1,51 @@ package main import ( + "crypto/subtle" "encoding/base64" - "fmt" "io" "log" "net/http" "net/url" - "strings" "time" - "github.com/elazarl/goproxy/ext/auth" - "github.com/elazarl/goproxy" + "github.com/elazarl/goproxy/ext/auth" ) -const ( - ProxyAuthHeader = "Proxy-Authorization" -) +const _proxyAuthHeader = "Proxy-Authorization" func SetBasicAuth(username, password string, req *http.Request) { - req.Header.Set(ProxyAuthHeader, fmt.Sprintf("Basic %s", basicAuth(username, password))) -} - -func basicAuth(username, password string) string { - return base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) -} - -func GetBasicAuth(req *http.Request) (username, password string, ok bool) { - auth := req.Header.Get(ProxyAuthHeader) - if auth == "" { - return - } - - const prefix = "Basic " - if !strings.HasPrefix(auth, prefix) { - return - } - c, err := base64.StdEncoding.DecodeString(auth[len(prefix):]) - if err != nil { - return - } - cs := string(c) - s := strings.IndexByte(cs, ':') - if s < 0 { - return - } - return cs[:s], cs[s+1:], true + req.Header.Set(_proxyAuthHeader, "Basic "+base64.StdEncoding.EncodeToString([]byte(username+":"+password))) } func main() { username, password := "foo", "bar" - // start end proxy server + // Start end proxy server endProxy := goproxy.NewProxyHttpServer() endProxy.Verbose = true auth.ProxyBasic(endProxy, "my_realm", func(user, pwd string) bool { - return user == username && password == pwd + return subtle.ConstantTimeCompare([]byte(user), []byte(username)) == 1 && + subtle.ConstantTimeCompare([]byte(pwd), []byte(password)) == 1 }) log.Println("serving end proxy server at localhost:8082") go http.ListenAndServe("localhost:8082", endProxy) - // start middle proxy server + // Start middle proxy server middleProxy := goproxy.NewProxyHttpServer() middleProxy.Verbose = true middleProxy.Tr.Proxy = func(req *http.Request) (*url.URL, error) { + // Here we specify the proxy URL of the other server. + // If it was a socks5 proxy, we would have used an url like + // socks5://localhost:8082 return url.Parse("http://localhost:8082") } connectReqHandler := func(req *http.Request) { SetBasicAuth(username, password, req) } middleProxy.ConnectDial = middleProxy.NewConnectDialToProxyWithHandler("http://localhost:8082", connectReqHandler) + middleProxy.OnRequest().Do(goproxy.FuncReqHandler(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { SetBasicAuth(username, password, req) return req, nil @@ -80,25 +55,29 @@ func main() { time.Sleep(1 * time.Second) - // fire a http request: client --> middle proxy --> end proxy --> internet - proxyUrl := "http://localhost:8081" + // Make a single HTTP request, from client to internet, through the 2 proxies + middleProxyUrl := "http://localhost:8081" request, err := http.NewRequest(http.MethodGet, "https://ip.cn", nil) if err != nil { log.Fatalf("new request failed:%v", err) } - tr := &http.Transport{Proxy: func(req *http.Request) (*url.URL, error) { return url.Parse(proxyUrl) }} - client := &http.Client{Transport: tr} - rsp, err := client.Do(request) + client := &http.Client{ + Transport: &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + return url.Parse(middleProxyUrl) + }, + }, + } + resp, err := client.Do(request) if err != nil { - log.Fatalf("get rsp failed:%v", err) - + log.Fatalf("get resp failed: %v", err) } - defer rsp.Body.Close() - data, _ := io.ReadAll(rsp.Body) + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) - if rsp.StatusCode != http.StatusOK { - log.Fatalf("status %d, data %s", rsp.StatusCode, data) + if resp.StatusCode != http.StatusOK { + log.Fatalf("status %d, data %s", resp.StatusCode, data) } - log.Printf("rsp:%s", data) + log.Printf("resp: %s", data) } diff --git a/examples/certstorage/README.md b/examples/certstorage/README.md new file mode 100644 index 000000000..dc18e5a2f --- /dev/null +++ b/examples/certstorage/README.md @@ -0,0 +1,11 @@ +# CertStorage + +CertStorage example is important to improve the performance of an +HTTPS proxy server, which you can build using goproxy. +Without a `proxy.CertStore`, every HTTPS request will generate new TLS +certificates and this, repeated for hundreds of request, will destroy your CPU. + +A lot of people opened issues in the projects complaining about this, because +they didn't use a certificates cache. +The cache implementation is up to you, maybe you can cache only the +most used hostnames, if you want to. diff --git a/examples/certstorage/cache.go b/examples/certstorage/cache.go new file mode 100644 index 000000000..9522a13f4 --- /dev/null +++ b/examples/certstorage/cache.go @@ -0,0 +1,39 @@ +package main + +import ( + "crypto/tls" + "sync" +) + +// CertStorage is a simple certificate cache that keeps +// everything in memory. +type CertStorage struct { + certs map[string]*tls.Certificate + mtx sync.RWMutex +} + +func (cs *CertStorage) Fetch(hostname string, gen func() (*tls.Certificate, error)) (*tls.Certificate, error) { + cs.mtx.RLock() + cert, ok := cs.certs[hostname] + cs.mtx.RUnlock() + if ok { + return cert, nil + } + + cert, err := gen() + if err != nil { + return nil, err + } + + cs.mtx.Lock() + cs.certs[hostname] = cert + cs.mtx.Unlock() + + return cert, nil +} + +func NewCertStorage() *CertStorage { + return &CertStorage{ + certs: make(map[string]*tls.Certificate), + } +} diff --git a/examples/goproxy-certstorage/main.go b/examples/certstorage/main.go similarity index 87% rename from examples/goproxy-certstorage/main.go rename to examples/certstorage/main.go index a010a1438..20438589f 100644 --- a/examples/goproxy-certstorage/main.go +++ b/examples/certstorage/main.go @@ -13,14 +13,16 @@ func main() { flag.Parse() proxy := goproxy.NewProxyHttpServer() - proxy.CertStore = NewCertStorage() //设置storage - + proxy.CertStore = NewCertStorage() proxy.Verbose = *verbose proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + // Log requested URL log.Println(req.URL.String()) return req, nil }) + + // Start proxy server log.Fatal(http.ListenAndServe(*addr, proxy)) } diff --git a/examples/customca/README.md b/examples/customca/README.md new file mode 100644 index 000000000..3d4f1a4a5 --- /dev/null +++ b/examples/customca/README.md @@ -0,0 +1,10 @@ +# CustomCA + +This example shows you how to use a custom CA to sign the HTTPS MITM +requests (you can use your own certificate). +This certificate must be trusted by your system, or the client will fail, if it's +not recognized. +The custom certificate is used to read the request data of an HTTPS +connection. +If the client has some kind of SSL pinning to check the certificates, the +request will most likely fail, so make sure to remove it. \ No newline at end of file diff --git a/examples/goproxy-customca/cert.go b/examples/customca/cert.go similarity index 75% rename from examples/goproxy-customca/cert.go rename to examples/customca/cert.go index f4fb578f0..65dcd29ed 100644 --- a/examples/goproxy-customca/cert.go +++ b/examples/customca/cert.go @@ -1,13 +1,6 @@ package main -import ( - "crypto/tls" - "crypto/x509" - - "github.com/elazarl/goproxy" -) - -var caCert = []byte(`-----BEGIN CERTIFICATE----- +var _caCert = []byte(`-----BEGIN CERTIFICATE----- MIIDkzCCAnugAwIBAgIJAKe/ZGdfcHdPMA0GCSqGSIb3DQEBCwUAMGAxCzAJBgNV BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX aWRnaXRzIFB0eSBMdGQxGTAXBgNVBAMMEGRlbW8gZm9yIGdvcHJveHkwHhcNMTYw @@ -30,7 +23,7 @@ troTflFMD2/4O6MtBKbHxSmEG6H0FBYz5xUZhZq7WUH24V3xYsfge29/lOCd5/Xf A+j0RJc/lQ== -----END CERTIFICATE-----`) -var caKey = []byte(`-----BEGIN RSA PRIVATE KEY----- +var _caKey = []byte(`-----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA2+W48YZoch72zj0a+ZlyFVY2q2MWmqsEY9f/u53fAeTxvPE6 1/DnqsydnA3FnGvxw9Dz0oZO6xG+PZvp+lhN07NZbuXK1nie8IpxCa342axpu4C0 69lZwxikpGyJO4IL5ywp/qfb5a2DxPTAyQOQ8ROAaydoEmktRp25yicnQ2yeZW// @@ -57,19 +50,3 @@ BTubAoGBAMEhI/Wy9wAETuXwN84AhmPdQsyCyp37YKt2ZKaqu37x9v2iL8JTbPEz pdBzkA2Gc0Wdb6ekIzRrTsJQl+c/0m9byFHsRsxXW2HnezfOFX1H4qAmF6KWP0ub M8aIn6Rab4sNPSrvKGrU6rFpv/6M33eegzldVnV9ku6uPJI1fFTC -----END RSA PRIVATE KEY-----`) - -func setCA(caCert, caKey []byte) error { - goproxyCa, err := tls.X509KeyPair(caCert, caKey) - if err != nil { - return err - } - if goproxyCa.Leaf, err = x509.ParseCertificate(goproxyCa.Certificate[0]); err != nil { - return err - } - goproxy.GoproxyCa = goproxyCa - goproxy.OkConnect = &goproxy.ConnectAction{Action: goproxy.ConnectAccept, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} - goproxy.MitmConnect = &goproxy.ConnectAction{Action: goproxy.ConnectMitm, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} - goproxy.HTTPMitmConnect = &goproxy.ConnectAction{Action: goproxy.ConnectHTTPMitm, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} - goproxy.RejectConnect = &goproxy.ConnectAction{Action: goproxy.ConnectReject, TLSConfig: goproxy.TLSConfigFromCA(&goproxyCa)} - return nil -} diff --git a/examples/customca/main.go b/examples/customca/main.go new file mode 100644 index 000000000..30cc95c35 --- /dev/null +++ b/examples/customca/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "crypto/tls" + "crypto/x509" + "flag" + "log" + "net/http" + + "github.com/elazarl/goproxy" +) + +func main() { + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("addr", ":8080", "proxy listen address") + flag.Parse() + + cert, err := parseCA(_caCert, _caKey) + if err != nil { + log.Fatal(err) + } + + customCaMitm := &goproxy.ConnectAction{Action: goproxy.ConnectMitm, TLSConfig: goproxy.TLSConfigFromCA(cert)} + var customAlwaysMitm goproxy.FuncHttpsHandler = func(host string, ctx *goproxy.ProxyCtx) (*goproxy.ConnectAction, string) { + return customCaMitm, host + } + + proxy := goproxy.NewProxyHttpServer() + proxy.OnRequest().HandleConnect(customAlwaysMitm) + proxy.Verbose = *verbose + log.Fatal(http.ListenAndServe(*addr, proxy)) +} + +func parseCA(caCert, caKey []byte) (*tls.Certificate, error) { + parsedCert, err := tls.X509KeyPair(caCert, caKey) + if err != nil { + return nil, err + } + if parsedCert.Leaf, err = x509.ParseCertificate(parsedCert.Certificate[0]); err != nil { + return nil, err + } + return &parsedCert, nil +} diff --git a/examples/goproxy-certstorage/README.md b/examples/goproxy-certstorage/README.md deleted file mode 100644 index b8213681a..000000000 --- a/examples/goproxy-certstorage/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# certstorage - -## use certstorage - -## this could make https proxy faster diff --git a/examples/goproxy-certstorage/storage.go b/examples/goproxy-certstorage/storage.go deleted file mode 100644 index 5593d8546..000000000 --- a/examples/goproxy-certstorage/storage.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "crypto/tls" - "sync" -) - -type CertStorage struct { - certs sync.Map -} - -func (tcs *CertStorage) Fetch(hostname string, gen func() (*tls.Certificate, error)) (*tls.Certificate, error) { - var cert tls.Certificate - icert, ok := tcs.certs.Load(hostname) - if ok { - cert = icert.(tls.Certificate) - } else { - certp, err := gen() - if err != nil { - return nil, err - } - // store as concrete implementation - cert = *certp - tcs.certs.Store(hostname, cert) - } - return &cert, nil -} - -func NewCertStorage() *CertStorage { - tcs := &CertStorage{} - tcs.certs = sync.Map{} - - return tcs -} diff --git a/examples/goproxy-customca/main.go b/examples/goproxy-customca/main.go deleted file mode 100644 index 11171ef6e..000000000 --- a/examples/goproxy-customca/main.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "flag" - "log" - "net/http" - - "github.com/elazarl/goproxy" -) - -func main() { - verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") - addr := flag.String("addr", ":8080", "proxy listen address") - flag.Parse() - setCA(caCert, caKey) - proxy := goproxy.NewProxyHttpServer() - proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) - proxy.Verbose = *verbose - log.Fatal(http.ListenAndServe(*addr, proxy)) -} diff --git a/examples/goproxy-eavesdropper/main.go b/examples/goproxy-eavesdropper/main.go deleted file mode 100644 index ae2256284..000000000 --- a/examples/goproxy-eavesdropper/main.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "bufio" - "flag" - "log" - "net" - "net/http" - "regexp" - - "github.com/elazarl/goproxy" -) - -func orPanic(err error) { - if err != nil { - panic(err) - } -} - -func main() { - proxy := goproxy.NewProxyHttpServer() - proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("baidu.*:443$"))). - HandleConnect(goproxy.AlwaysReject) - proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))). - HandleConnect(goproxy.AlwaysMitm) - // enable curl -p for all hosts on port 80 - proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*:80$"))). - HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) { - defer func() { - if e := recover(); e != nil { - ctx.Logf("error connecting to remote: %v", e) - client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n")) - } - client.Close() - }() - clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client)) - remote, err := net.Dial("tcp", req.URL.Host) - orPanic(err) - client.Write([]byte("HTTP/1.1 200 Ok\r\n\r\n")) - remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote)) - for { - req, err := http.ReadRequest(clientBuf.Reader) - orPanic(err) - orPanic(req.Write(remoteBuf)) - orPanic(remoteBuf.Flush()) - resp, err := http.ReadResponse(remoteBuf.Reader, req) - orPanic(err) - orPanic(resp.Write(clientBuf.Writer)) - orPanic(clientBuf.Flush()) - } - }) - verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") - addr := flag.String("addr", ":8080", "proxy listen address") - flag.Parse() - proxy.Verbose = *verbose - log.Fatal(http.ListenAndServe(*addr, proxy)) -} diff --git a/examples/goproxy-jquery-version/README.md b/examples/goproxy-jquery-version/README.md deleted file mode 100644 index 6efba22ad..000000000 --- a/examples/goproxy-jquery-version/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# Content Analysis - -`goproxy-jquery-version` starts an HTTP proxy on :8080. It checks HTML -responses, looks for scripts referencing jQuery library and emits warnings if -different versions of the library are being used for a given host. - -Start it in one shell: - -```sh -goproxy-jquery-version -``` - -Fetch goproxy homepage in another: - -```sh -http_proxy=http://127.0.0.1:8080 wget -O - \ - http://ripper234.com/p/introducing-goproxy-light-http-proxy/ -``` - -Goproxy homepage uses jQuery and a mix of plugins. First the proxy reports the -first use of jQuery it detects for the domain. Then, because the regular -expression matching the jQuery sources is imprecise, it reports a mismatch with -a plugin reference: - -```sh -2015/04/11 11:23:02 [001] WARN: ripper234.com uses //ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js -2015/04/11 11:23:02 [001] WARN: In http://ripper234.com/p/introducing-goproxy-light-http-proxy/, \ - Contradicting jqueries //ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js \ - http://ripper234.wpengine.netdna-cdn.com/wp-content/plugins/wp-ajax-edit-comments/js/jquery.colorbox.min.js?ver=5.0.36 -``` - diff --git a/examples/goproxy-stats/README.md b/examples/goproxy-stats/README.md deleted file mode 100644 index a51d4c8eb..000000000 --- a/examples/goproxy-stats/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Gather Browsing Statistics - -`goproxy-stats` starts an HTTP proxy on :8080, counts the bytes received for -web resources and prints the cumulative sum per URL every 20 seconds. - -Start it in one shell: - -```sh -goproxy-stats -``` - -Fetch goproxy homepage in another: - -```sh -mkdir tmp -cd tmp -http_proxy=http://127.0.0.1:8080 wget -r -l 1 -H \ - http://ripper234.com/p/introducing-goproxy-light-http-proxy/ -``` - -Stop it after a moment. `goproxy-stats` should eventually print: -```sh -listening on :8080 -statistics -http://www.telerik.com/fiddler -> 84335 -http://msmvps.com/robots.txt -> 157 -http://eli.thegreenplace.net/robots.txt -> 294 -http://www.phdcomics.com/robots.txt -> 211 -http://resharper.blogspot.com/robots.txt -> 221 -http://idanz.blogli.co.il/robots.txt -> 271 -http://ripper234.com/p/introducing-goproxy-light-http-proxy/ -> 44407 -http://live.gnome.org/robots.txt -> 298 -http://ponetium.wordpress.com/robots.txt -> 178 -http://pilaheleg.blogli.co.il/robots.txt -> 321 -http://pilaheleg.wordpress.com/robots.txt -> 178 -http://blogli.co.il/ -> 9165 -http://nimrod-code.org/robots.txt -> 289 -http://www.joelonsoftware.com/robots.txt -> 1245 -http://top-performance.blogspot.com/robots.txt -> 227 -http://ooc-lang.org/robots.txt -> 345 -http://blogs.jetbrains.com/robots.txt -> 293 -``` - diff --git a/examples/goproxy-stats/main.go b/examples/goproxy-stats/main.go deleted file mode 100644 index 6865c19a6..000000000 --- a/examples/goproxy-stats/main.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "fmt" - "io" - "log" - . "net/http" - "time" - - "github.com/elazarl/goproxy" - goproxy_html "github.com/elazarl/goproxy/ext/html" -) - -type Count struct { - Id string - Count int64 -} -type CountReadCloser struct { - Id string - R io.ReadCloser - ch chan<- Count - nr int64 -} - -func (c *CountReadCloser) Read(b []byte) (n int, err error) { - n, err = c.R.Read(b) - c.nr += int64(n) - return -} -func (c CountReadCloser) Close() error { - c.ch <- Count{c.Id, c.nr} - return c.R.Close() -} - -func main() { - proxy := goproxy.NewProxyHttpServer() - timer := make(chan bool) - ch := make(chan Count, 10) - go func() { - for { - time.Sleep(20 * time.Second) - timer <- true - } - }() - go func() { - m := make(map[string]int64) - for { - select { - case c := <-ch: - m[c.Id] = m[c.Id] + c.Count - case <-timer: - fmt.Printf("statistics\n") - for k, v := range m { - fmt.Printf("%s -> %d\n", k, v) - } - } - } - }() - - // IsWebRelatedText filters on html/javascript/css resources - proxy.OnResponse(goproxy_html.IsWebRelatedText).DoFunc(func(resp *Response, ctx *goproxy.ProxyCtx) *Response { - resp.Body = &CountReadCloser{ctx.Req.URL.String(), resp.Body, ch, 0} - return resp - }) - fmt.Printf("listening on :8080\n") - log.Fatal(ListenAndServe(":8080", proxy)) -} diff --git a/examples/goproxy-yui-minify/yui.go b/examples/goproxy-yui-minify/yui.go deleted file mode 100644 index 7d83e9709..000000000 --- a/examples/goproxy-yui-minify/yui.go +++ /dev/null @@ -1,90 +0,0 @@ -// This example would minify standalone Javascript files (identified by their content type) -// using the command line utility YUI compressor http://yui.github.io/yuicompressor/ -// Example usage: -// -// ./yui -java /usr/local/bin/java -yuicompressor ~/Downloads/yuicompressor-2.4.8.jar -// $ curl -vx localhost:8080 http://golang.org/lib/godoc/godocs.js -// (function(){function g(){var u=$("#search");if(u.length===0){return}function t(){if(.... -// $ curl http://golang.org/lib/godoc/godocs.js | head -n 3 -// // Copyright 2012 The Go Authors. All rights reserved. -// // Use of this source code is governed by a BSD-style -// // license that can be found in the LICENSE file. -package main - -import ( - "flag" - "io" - "log" - "net/http" - "os" - "os/exec" - "path" - "strings" - - "github.com/elazarl/goproxy" -) - -func main() { - verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") - addr := flag.String("addr", ":8080", "proxy listen address") - java := flag.String("javapath", "java", "where the Java executable is located") - yuicompressor := flag.String("yuicompressor", "", "where the yuicompressor is located, assumed to be in CWD") - yuicompressordir := flag.String("yuicompressordir", ".", "a folder to search yuicompressor in, will be ignored if yuicompressor is set") - flag.Parse() - if *yuicompressor == "" { - files, err := os.ReadDir(*yuicompressordir) - if err != nil { - log.Fatal("Cannot find yuicompressor jar") - } - for _, file := range files { - if strings.HasPrefix(file.Name(), "yuicompressor") && strings.HasSuffix(file.Name(), ".jar") { - c := path.Join(*yuicompressordir, file.Name()) - yuicompressor = &c - break - } - } - } - if *yuicompressor == "" { - log.Fatal("Can't find yuicompressor jar, searched yuicompressor*.jar in dir ", *yuicompressordir) - } - if _, err := os.Stat(*yuicompressor); os.IsNotExist(err) { - log.Fatal("Can't find yuicompressor jar specified ", *yuicompressor) - } - proxy := goproxy.NewProxyHttpServer() - proxy.Verbose = *verbose - proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response { - contentType := resp.Header.Get("Content-Type") - if contentType == "application/javascript" || contentType == "application/x-javascript" { - // in real code, response should be streamed as well - var err error - cmd := exec.Command(*java, "-jar", *yuicompressor, "--type", "js") - cmd.Stdin = resp.Body - resp.Body, err = cmd.StdoutPipe() - if err != nil { - ctx.Warnf("Cannot minify content in %v: %v", ctx.Req.URL, err) - return goproxy.TextResponse(ctx.Req, "Error getting stdout pipe") - } - stderr, err := cmd.StderrPipe() - if err != nil { - ctx.Logf("Error obtaining stderr from yuicompress: %s", err) - return goproxy.TextResponse(ctx.Req, "Error getting stderr pipe") - } - if err := cmd.Start(); err != nil { - ctx.Warnf("Cannot minify content in %v: %v", ctx.Req.URL, err) - } - go func() { - defer stderr.Close() - const kb = 1024 - msg, err := io.ReadAll(&io.LimitedReader{stderr, 50 * kb}) - if len(msg) != 0 { - ctx.Logf("Error executing yuicompress: %s", string(msg)) - } - if err != nil { - ctx.Logf("Error reading stderr from yuicompress: %s", string(msg)) - } - }() - } - return resp - }) - log.Fatal(http.ListenAndServe(*addr, proxy)) -} diff --git a/examples/hijack/README.md b/examples/hijack/README.md new file mode 100644 index 000000000..f4458d267 --- /dev/null +++ b/examples/hijack/README.md @@ -0,0 +1,37 @@ +# Hijack +In this example we intercept the data of an HTTP request and decide to +modify them before sending to the client. +In this mode, we take over on the raw connection and we could send any +data that we want. + +Curl example: + +``` +$ curl -x localhost:8080 http://google.it -v -k -p + +* Host localhost:8080 was resolved. +* IPv6: ::1 +* IPv4: 127.0.0.1 +* Trying [::1]:8080... +* Connected to localhost (::1) port 8080 +* CONNECT tunnel: HTTP/1.1 negotiated +* allocate connect buffer +* Establish HTTP proxy tunnel to google.it:80 +> CONNECT google.it:80 HTTP/1.1 +> Host: google.it:80 +> User-Agent: curl/8.9.1 +> Proxy-Connection: Keep-Alive +> +< HTTP/1.1 200 Ok +< +* CONNECT phase completed +* CONNECT tunnel established, response 200 +> GET / HTTP/1.1 +> Host: google.it +> User-Agent: curl/8.9.1 +> Accept: */* +> +< HTTP/1.1 200 OK +< test: 1234 +< Content-Length: 0 +``` diff --git a/examples/hijack/main.go b/examples/hijack/main.go new file mode 100644 index 000000000..b8859d31b --- /dev/null +++ b/examples/hijack/main.go @@ -0,0 +1,46 @@ +package main + +import ( + "bufio" + "flag" + "log" + "net" + "net/http" + "regexp" + + "github.com/elazarl/goproxy" +) + +func main() { + proxy := goproxy.NewProxyHttpServer() + // Reject all requests to baidu + proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("baidu.*:443$"))). + HandleConnect(goproxy.AlwaysReject) + + // Instead of returning the Internet response, send custom data from + // our proxy server, using connection hijack + proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile("^.*$"))). + HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) { + client.Write([]byte("HTTP/1.1 200 Ok\r\n\r\n")) + + w := bufio.NewWriter(client) + + resp := &http.Response{ + StatusCode: http.StatusOK, + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{ + "test": {"1234"}, + }, + } + resp.Write(w) + w.Flush() + client.Close() + }) + + verbose := flag.Bool("v", false, "should every proxy request be logged to stdout") + addr := flag.String("addr", ":8080", "proxy listen address") + flag.Parse() + proxy.Verbose = *verbose + log.Fatal(http.ListenAndServe(*addr, proxy)) +} diff --git a/examples/html-parser/README.md b/examples/html-parser/README.md new file mode 100644 index 000000000..ed5b96213 --- /dev/null +++ b/examples/html-parser/README.md @@ -0,0 +1,34 @@ +# HTML Parser + +`html-parser` starts an HTTP proxy on :8080. +It checks HTML responses, looks for scripts referencing jQuery library +and log warnings if different versions of the library are being used +for a given host. +This is an example of how a proxy can parse the received responses and +manipulate them to do useful actions. + +Start the server: + +```sh +go build +html-parser +``` + +Make a test request in another shell: + +```sh +http_proxy=http://127.0.0.1:8080 wget -O - \ + http://ripper234.com/p/introducing-goproxy-light-http-proxy/ +``` + +Goproxy example homepage contains jQuery and a mix of JQuery plugins. +First the proxy reports the first use of jQuery it detects for the domain. +Then, because the regular expression matching the jQuery sources is imprecise, +it reports a mismatch with a plugin reference: + +```sh +2015/04/11 11:23:02 [001] WARN: ripper234.com uses //ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js +2015/04/11 11:23:02 [001] WARN: In http://ripper234.com/p/introducing-goproxy-light-http-proxy/, \ + Contradicting jqueries //ajax.googleapis.com/ajax/libs/jquery/1.11.1/jquery.min.js \ + http://ripper234.wpengine.netdna-cdn.com/wp-content/plugins/wp-ajax-edit-comments/js/jquery.colorbox.min.js?ver=5.0.36 +``` diff --git a/examples/goproxy-jquery-version/jquery1.html b/examples/html-parser/jquery1.html similarity index 100% rename from examples/goproxy-jquery-version/jquery1.html rename to examples/html-parser/jquery1.html diff --git a/examples/goproxy-jquery-version/jquery2.html b/examples/html-parser/jquery2.html similarity index 100% rename from examples/goproxy-jquery-version/jquery2.html rename to examples/html-parser/jquery2.html diff --git a/examples/goproxy-jquery-version/jquery_homepage.html b/examples/html-parser/jquery_homepage.html similarity index 100% rename from examples/goproxy-jquery-version/jquery_homepage.html rename to examples/html-parser/jquery_homepage.html diff --git a/examples/goproxy-jquery-version/jquery_test.go b/examples/html-parser/jquery_test.go similarity index 98% rename from examples/goproxy-jquery-version/jquery_test.go rename to examples/html-parser/jquery_test.go index 78b026f4f..1b67c4ed9 100644 --- a/examples/goproxy-jquery-version/jquery_test.go +++ b/examples/html-parser/jquery_test.go @@ -16,7 +16,7 @@ func equal(u, v []string) bool { if len(u) != len(v) { return false } - for i, _ := range u { + for i := range u { if u[i] != v[i] { return false } @@ -63,7 +63,7 @@ func TestDefectiveScriptParser(t *testing.T) { } func proxyWithLog() (*http.Client, *bytes.Buffer) { - proxy := NewJqueryVersionProxy() + proxy := NewJQueryVersionProxy() proxyServer := httptest.NewServer(proxy) buf := new(bytes.Buffer) proxy.Logger = log.New(buf, "", 0) diff --git a/examples/goproxy-jquery-version/main.go b/examples/html-parser/main.go similarity index 94% rename from examples/goproxy-jquery-version/main.go rename to examples/html-parser/main.go index a92dddeac..72223f1e5 100644 --- a/examples/goproxy-jquery-version/main.go +++ b/examples/html-parser/main.go @@ -31,7 +31,7 @@ func findScriptSrc(html string) []string { // NewJQueryVersionProxy creates a proxy checking responses HTML content, looks // for scripts referencing jQuery library and emits warnings if different // versions of the library are being used for a given host. -func NewJqueryVersionProxy() *goproxy.ProxyHttpServer { +func NewJQueryVersionProxy() *goproxy.ProxyHttpServer { proxy := goproxy.NewProxyHttpServer() m := make(map[string]string) jqueryMatcher := regexp.MustCompile(`(?i:jquery\.)`) @@ -59,6 +59,6 @@ func NewJqueryVersionProxy() *goproxy.ProxyHttpServer { } func main() { - proxy := NewJqueryVersionProxy() + proxy := NewJQueryVersionProxy() log.Fatal(http.ListenAndServe(":8080", proxy)) } diff --git a/examples/goproxy-jquery-version/php_man.html b/examples/html-parser/php_man.html similarity index 100% rename from examples/goproxy-jquery-version/php_man.html rename to examples/html-parser/php_man.html diff --git a/examples/goproxy-jquery-version/w3schools.html b/examples/html-parser/w3schools.html similarity index 100% rename from examples/goproxy-jquery-version/w3schools.html rename to examples/html-parser/w3schools.html diff --git a/examples/image-manipulation/README.md b/examples/image-manipulation/README.md new file mode 100644 index 000000000..86edd045a --- /dev/null +++ b/examples/image-manipulation/README.md @@ -0,0 +1,6 @@ +# Image Manipulation + +This example starts a proxy server that manipulate the received images, +to make them appear upside down. +This directly modify the response received from the response server, +and returns the new data to the proxy caller. diff --git a/examples/goproxy-upside-down-ternet/main.go b/examples/image-manipulation/main.go similarity index 83% rename from examples/goproxy-upside-down-ternet/main.go rename to examples/image-manipulation/main.go index 4b683fd32..4e4062766 100644 --- a/examples/goproxy-upside-down-ternet/main.go +++ b/examples/image-manipulation/main.go @@ -13,13 +13,13 @@ func main() { proxy.OnResponse().Do(goproxy_image.HandleImage(func(img image.Image, ctx *goproxy.ProxyCtx) image.Image { dx, dy := img.Bounds().Dx(), img.Bounds().Dy() - nimg := image.NewRGBA(img.Bounds()) + newImg := image.NewRGBA(img.Bounds()) for i := 0; i < dx; i++ { for j := 0; j <= dy; j++ { - nimg.Set(i, j, img.At(i, dy-j-1)) + newImg.Set(i, j, img.At(i, dy-j-1)) } } - return nimg + return newImg })) proxy.Verbose = true log.Fatal(http.ListenAndServe(":8080", proxy)) diff --git a/examples/remove-https/README.md b/examples/remove-https/README.md new file mode 100644 index 000000000..f72e54a6d --- /dev/null +++ b/examples/remove-https/README.md @@ -0,0 +1,6 @@ +# Remove HTTPS + +`remove-https` example forwards all the HTTPS request as HTTP requests, +effectively removing the https schema from the requests. +This example shows you how you can rewrite the request URL, when +needed. \ No newline at end of file diff --git a/examples/goproxy-sslstrip/sslstrip.go b/examples/remove-https/main.go similarity index 82% rename from examples/goproxy-sslstrip/sslstrip.go rename to examples/remove-https/main.go index b7e2dc9e5..29532c85a 100644 --- a/examples/goproxy-sslstrip/sslstrip.go +++ b/examples/remove-https/main.go @@ -1,9 +1,9 @@ package main import ( + "flag" "github.com/elazarl/goproxy" "log" - "flag" "net/http" ) @@ -13,7 +13,7 @@ func main() { flag.Parse() proxy := goproxy.NewProxyHttpServer() proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) - proxy.OnRequest().DoFunc(func (req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { + proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) { if req.URL.Scheme == "https" { req.URL.Scheme = "http" } diff --git a/examples/goproxy-no-reddit-at-worktime/README.md b/examples/request-filtering/README.md similarity index 57% rename from examples/goproxy-no-reddit-at-worktime/README.md rename to examples/request-filtering/README.md index 23b522405..51a3adef0 100644 --- a/examples/goproxy-no-reddit-at-worktime/README.md +++ b/examples/request-filtering/README.md @@ -1,15 +1,16 @@ # Request Filtering -`goproxy-no-reddit-at-work` starts an HTTP proxy on :8080. It denies requests -to "www.reddit.com" made between 8am to 5pm inclusive, local time. +`request-filtering` starts an HTTP proxy on :8080. It denies requests +to "www.reddit.com" made between 8am to 5pm inclusive, local server +time. -Start it in one shell: +Start the server: ```sh -$ goproxy-no-reddit-at-work +$ request-filtering ``` -Fetch reddit in another: +Make a test request in another shell: ```sh $ http_proxy=http://127.0.0.1:8080 wget -O - http://www.reddit.com @@ -18,4 +19,3 @@ Connecting to 127.0.0.1:8080... connected. Proxy request sent, awaiting response... 403 Forbidden 2015-04-11 16:59:01 ERROR 403: Forbidden. ``` - diff --git a/examples/goproxy-no-reddit-at-worktime/noreddit.go b/examples/request-filtering/noreddit.go similarity index 100% rename from examples/goproxy-no-reddit-at-worktime/noreddit.go rename to examples/request-filtering/noreddit.go diff --git a/examples/socket-keepalive/README.md b/examples/socket-keepalive/README.md new file mode 100644 index 000000000..cb392b9d0 --- /dev/null +++ b/examples/socket-keepalive/README.md @@ -0,0 +1,9 @@ +# Socket KeepAlive + +`socket-keepalive` example adds a custom net.Dialer that can be configured +by the user, enabling TCP keep alives in this example. +By default, Go already uses 15 seconds TCP keep alives for the connections, +so this example is not strictly required, as it is provided. +TCP keep alives are useful for a connection that can be idle for a while, +to avoid the TCP connection close. +The TCP connection is closed when the request context expires. \ No newline at end of file diff --git a/examples/goproxy-sokeepalive/sokeepalive.go b/examples/socket-keepalive/keepalive.go similarity index 90% rename from examples/goproxy-sokeepalive/sokeepalive.go rename to examples/socket-keepalive/keepalive.go index a50398e51..8db69f6c0 100644 --- a/examples/goproxy-sokeepalive/sokeepalive.go +++ b/examples/socket-keepalive/keepalive.go @@ -20,12 +20,8 @@ func main() { if c, ok := c.(*net.TCPConn); err == nil && ok { c.SetKeepAlive(true) go func() { - select { - case <-ctx.Done(): - { - c.Close() - } - } + <-ctx.Done() + c.Close() }() } return diff --git a/examples/websockets/README.md b/examples/websockets/README.md new file mode 100644 index 000000000..5149d2acc --- /dev/null +++ b/examples/websockets/README.md @@ -0,0 +1,9 @@ +# Websockets +`websockets` example shows an example of a WebSocket request made +through the proxy server. +The target server continuously send data to the client, and the proxy will +forward them to websocket client. +To run this example, it's enough to just run the main. + +Inside this folder there are also some self-signed certificates valid for +localhost, that are automatically used for the WebSocket server. diff --git a/examples/goproxy-websockets/localhost-key.pem b/examples/websockets/localhost-key.pem similarity index 100% rename from examples/goproxy-websockets/localhost-key.pem rename to examples/websockets/localhost-key.pem diff --git a/examples/goproxy-websockets/localhost.pem b/examples/websockets/localhost.pem similarity index 100% rename from examples/goproxy-websockets/localhost.pem rename to examples/websockets/localhost.pem diff --git a/examples/goproxy-websockets/main.go b/examples/websockets/main.go similarity index 65% rename from examples/goproxy-websockets/main.go rename to examples/websockets/main.go index ca703b771..29c0128ed 100644 --- a/examples/goproxy-websockets/main.go +++ b/examples/websockets/main.go @@ -9,80 +9,80 @@ import ( "net/url" "os" "os/signal" - "sync" "time" ) -var upgrader = websocket.Upgrader{} // use default options +var _upgrader = websocket.Upgrader{ + HandshakeTimeout: 10 * time.Second, +} func echo(w http.ResponseWriter, r *http.Request) { - c, err := upgrader.Upgrade(w, r, nil) + c, err := _upgrader.Upgrade(w, r, nil) if err != nil { - log.Print("upgrade:", err) + log.Printf("upgrade: %v\n", err) return } defer c.Close() + for { mt, message, err := c.ReadMessage() if err != nil { - log.Println("read:", err) + log.Printf("read: %v\n", err) break } - log.Printf("recv: %s", message) - err = c.WriteMessage(mt, message) - if err != nil { - log.Println("write:", err) + log.Printf("recv: %s\n", message) + if err := c.WriteMessage(mt, message); err != nil { + log.Printf("write: %v\n", err) break } } } -func StartEchoServer(wg *sync.WaitGroup) { +func StartEchoServer() { log.Println("Starting echo server") - wg.Add(1) go func() { http.HandleFunc("/", echo) err := http.ListenAndServeTLS(":12345", "localhost.pem", "localhost-key.pem", nil) if err != nil { - panic("ListenAndServe: " + err.Error()) + log.Fatal(err) } - wg.Done() }() } -func StartProxy(wg *sync.WaitGroup) { +func StartProxy() { log.Println("Starting proxy server") - wg.Add(1) go func() { proxy := goproxy.NewProxyHttpServer() proxy.OnRequest().HandleConnect(goproxy.AlwaysMitm) proxy.Verbose = true - err := http.ListenAndServe(":54321", proxy) - if err != nil { - log.Fatal(err.Error()) + if err := http.ListenAndServe(":54321", proxy); err != nil { + log.Fatal(err) } - wg.Done() }() } func main() { interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, os.Interrupt) - wg := &sync.WaitGroup{} - StartEchoServer(wg) - StartProxy(wg) + StartEchoServer() + StartProxy() - endpointUrl := "wss://localhost:12345" proxyUrl := "http://localhost:54321" + parsedProxy, err := url.Parse(proxyUrl) + if err != nil { + log.Fatal("unable to parse proxy URL") + } - surl, _ := url.Parse(proxyUrl) dialer := websocket.Dialer{ - Subprotocols: []string{"p1"}, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - Proxy: http.ProxyURL(surl), + Subprotocols: []string{"p1"}, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + Proxy: http.ProxyURL(parsedProxy), } + endpointUrl := "wss://localhost:12345" c, _, err := dialer.Dial(endpointUrl, nil) if err != nil { log.Fatal("dial:", err) @@ -92,7 +92,6 @@ func main() { done := make(chan struct{}) go func() { - defer c.Close() defer close(done) for { _, message, err := c.ReadMessage() @@ -109,13 +108,13 @@ func main() { for { select { - case t := <-ticker.C: - err := c.WriteMessage(websocket.TextMessage, []byte(t.String())) - if err != nil { + case t := <-ticker.C: // Message send + // Write current time to the websocket client every 1 second + if err := c.WriteMessage(websocket.TextMessage, []byte(t.String())); err != nil { log.Println("write:", err) return } - case <-interrupt: + case <-interrupt: // Server shutdown log.Println("interrupt") // To cleanly close a connection, a client should send a close // frame and wait for the server to close the connection. @@ -124,11 +123,11 @@ func main() { log.Println("write close:", err) return } + select { case <-done: case <-time.After(time.Second): } - c.Close() return } }