Integrate fixes from Stripe fork

counterecryptor.go

@@ -18,7 +18,7 @@ type CounterEncryptorRand struct {
 	ix      int
 }
 
-func NewCounterEncryptorRandFromKey(key interface{}, seed []byte) (r CounterEncryptorRand, err error) {
+func NewCounterEncryptorRandFromKey(key any, seed []byte) (r CounterEncryptorRand, err error) {
 	var keyBytes []byte
 	switch key := key.(type) {
 	case *rsa.PrivateKey:

ctx.go

@@ -18,7 +18,7 @@ type ProxyCtx struct {
 	Error error
 	// A handle for the user to keep data in the context, from the call of ReqHandler to the
 	// call of RespHandler
-	UserData interface{}
+	UserData any
 	// Will connect a request to a response
 	Session   int64
 	certStore CertStorage
@@ -46,8 +46,8 @@ func (ctx *ProxyCtx) RoundTrip(req *http.Request) (*http.Response, error) {
 	return ctx.Proxy.Tr.RoundTrip(req)
 }
 
-func (ctx *ProxyCtx) printf(msg string, argv ...interface{}) {
-	ctx.Proxy.Logger.Printf("[%03d] "+msg+"\n", append([]interface{}{ctx.Session & 0xFF}, argv...)...)
+func (ctx *ProxyCtx) printf(msg string, argv ...any) {
+	ctx.Proxy.Logger.Printf("[%03d] "+msg+"\n", append([]any{ctx.Session & 0xFFFF}, argv...)...)
 }
 
 // Logf prints a message to the proxy's log. Should be used in a ProxyHttpServer's filter
@@ -58,7 +58,7 @@ func (ctx *ProxyCtx) printf(msg string, argv ...interface{}) {
 //		ctx.Printf("So far %d requests",nr)
 //		return r, nil
 //	})
-func (ctx *ProxyCtx) Logf(msg string, argv ...interface{}) {
+func (ctx *ProxyCtx) Logf(msg string, argv ...any) {
 	if ctx.Proxy.Verbose {
 		ctx.printf("INFO: "+msg, argv...)
 	}
@@ -75,7 +75,7 @@ func (ctx *ProxyCtx) Logf(msg string, argv ...interface{}) {
 //		}
 //		return r, nil
 //	})
-func (ctx *ProxyCtx) Warnf(msg string, argv ...interface{}) {
+func (ctx *ProxyCtx) Warnf(msg string, argv ...any) {
 	ctx.printf("WARN: "+msg, argv...)
 }
 

examples/goproxy-httpdump/httpdump.go

@@ -56,7 +56,7 @@ type Meta struct {
 	from     string
 }
 
-func fprintf(nr *int64, err *error, w io.Writer, pat string, a ...interface{}) {
+func fprintf(nr *int64, err *error, w io.Writer, pat string, a ...any) {
 	if *err != nil {
 		return
 	}

https.go

@@ -12,7 +12,6 @@ import (
 	"os"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 )
 
@@ -137,17 +136,35 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request
 		targetTCP, targetOK := targetSiteCon.(halfClosable)
 		proxyClientTCP, clientOK := proxyClient.(halfClosable)
 		if targetOK && clientOK {
-			go copyAndClose(targetTCP, proxyClientTCP)
-			go copyAndClose(proxyClientTCP, targetTCP)
+			go copyAndClose(ctx, targetTCP, proxyClientTCP)
+			go copyAndClose(ctx, proxyClientTCP, targetTCP)
 		} else {
+			// There is a race with the runtime here. In the case where the
+			// connection to the target site times out, we cannot control which
+			// io.Copy loop will receive the timeout signal first. This means
+			// that in some cases the error passed to the ConnErrorHandler will
+			// be the timeout error, and in other cases it will be an error raised
+			// by the use of a closed network connection.
+			//
+			// 2020/05/28 23:42:17 [001] WARN: Error copying to client: read tcp 127.0.0.1:33742->127.0.0.1:34763: i/o timeout
+			// 2020/05/28 23:42:17 [001] WARN: Error copying to client: read tcp 127.0.0.1:45145->127.0.0.1:60494: use of closed network connection
+			//
+			// It's also not possible to synchronize these connection closures due to
+			// TCP connections which are half-closed. When this happens, only the one
+			// side of the connection breaks out of its io.Copy loop. The other side
+			// of the connection remains open until it either times out or is reset by
+			// the client.
 			go func() {
-				var wg sync.WaitGroup
-				wg.Add(2)
-				go copyOrWarn(targetSiteCon, proxyClient, &wg)
-				go copyOrWarn(proxyClient, targetSiteCon, &wg)
-				wg.Wait()
-				proxyClient.Close()
-				targetSiteCon.Close()
+				err := copyOrWarn(ctx, targetSiteCon, proxyClient)
+				if err != nil && proxy.ConnectionErrHandler != nil {
+					proxy.ConnectionErrHandler(w, ctx, err)
+				}
+				_ = targetSiteCon.Close()
+			}()
+
+			go func() {
+				_ = copyOrWarn(ctx, proxyClient, targetSiteCon)
+				_ = proxyClient.Close()
 			}()
 		}
 
@@ -226,7 +243,7 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request
 			clientTlsReader := bufio.NewReader(rawClientTls)
 			for !isEof(clientTlsReader) {
 				req, err := http.ReadRequest(clientTlsReader)
-				var ctx = &ProxyCtx{Req: req, Session: atomic.AddInt64(&proxy.sess, 1), Proxy: proxy, UserData: ctx.UserData}
+				var ctx = &ProxyCtx{Req: req, Session: atomic.AddInt64(&proxy.sess, 1), Proxy: proxy, UserData: ctx.UserData, RoundTripper: ctx.RoundTripper}
 				if err != nil && err != io.EOF {
 					return
 				}
@@ -380,22 +397,36 @@ func (proxy *ProxyHttpServer) handleHttps(w http.ResponseWriter, r *http.Request
 }
 
 func httpError(w io.WriteCloser, ctx *ProxyCtx, err error) {
-	errStr := fmt.Sprintf("HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s", len(err.Error()), err.Error())
-	if _, err := io.WriteString(w, errStr); err != nil {
-		ctx.Warnf("Error responding to client: %s", err)
+	if ctx.Proxy.ConnectionErrHandler != nil {
+		ctx.Proxy.ConnectionErrHandler(w, ctx, err)
+	} else {
+		errStr := fmt.Sprintf("HTTP/1.1 502 Bad Gateway\r\nContent-Type: text/plain\r\nContent-Length: %d\r\n\r\n%s", len(err.Error()), err.Error())
+		if _, err := io.WriteString(w, errStr); err != nil {
+			ctx.Warnf("Error responding to client: %s", err)
+		}
 	}
 	if err := w.Close(); err != nil {
 		ctx.Warnf("Error closing client connection: %s", err)
 	}
 }
 
-func copyOrWarn(dst io.Writer, src io.Reader, wg *sync.WaitGroup) {
-	_, _ = io.Copy(dst, src)
-	wg.Done()
+func copyOrWarn(ctx *ProxyCtx, dst io.Writer, src io.Reader) error {
+	_, err := io.Copy(dst, src)
+	if err != nil && errors.Is(err, net.ErrClosed) {
+		// Discard closed connection errors
+		err = nil
+	} else if err != nil {
+		ctx.Warnf("Error copying to client: %s", err)
+	}
+	return err
 }
 
-func copyAndClose(dst, src halfClosable) {
-	_, _ = io.Copy(dst, src)
+func copyAndClose(ctx *ProxyCtx, dst, src halfClosable) {
+	_, err := io.Copy(dst, src)
+	if err != nil && !errors.Is(err, net.ErrClosed) {
+		ctx.Warnf("Error copying to client: %s", err.Error())
+	}
+
 	dst.CloseWrite()
 	src.CloseRead()
 }

logger.go

@@ -1,5 +1,5 @@
 package goproxy
 
 type Logger interface {
-	Printf(format string, v ...interface{})
+	Printf(format string, v ...any)
 }

proxy.go

@@ -27,13 +27,27 @@ type ProxyHttpServer struct {
 	respHandlers    []RespHandler
 	httpsHandlers   []HttpsHandler
 	Tr              *http.Transport
+	// ConnectionErrHandler will be invoked to return a custom response
+	// to clients (written using conn parameter), when goproxy fails to connect
+	// to a target proxy.
+	// The error is passed as function parameter and not inside the proxy
+	// context, to avoid race conditions.
+	ConnectionErrHandler func(conn io.Writer, ctx *ProxyCtx, err error)
 	// ConnectDial will be used to create TCP connections for CONNECT requests
 	// if nil Tr.Dial will be used
 	ConnectDial        func(network string, addr string) (net.Conn, error)
 	ConnectDialWithReq func(req *http.Request, network string, addr string) (net.Conn, error)
 	CertStore          CertStorage
 	KeepHeader         bool
 	AllowHTTP2         bool
+	// KeepAcceptEncoding, if true, prevents the proxy from dropping
+	// Accept-Encoding headers from the client.
+	//
+	// Note that the outbound http.Transport may still choose to add
+	// Accept-Encoding: gzip if the client did not explicitly send an
+	// Accept-Encoding header. To disable this behavior, set
+	// Tr.DisableCompression to true.
+	KeepAcceptEncoding bool
 }
 
 var hasPort = regexp.MustCompile(`:\d+$`)
@@ -83,9 +97,11 @@ func (proxy *ProxyHttpServer) filterResponse(respOrig *http.Response, ctx *Proxy
 func RemoveProxyHeaders(ctx *ProxyCtx, r *http.Request) {
 	r.RequestURI = "" // this must be reset when serving a request with the client
 	ctx.Logf("Sending request %v %v", r.Method, r.URL.String())
-	// If no Accept-Encoding header exists, Transport will add the headers it can accept
-	// and would wrap the response body with the relevant reader.
-	r.Header.Del("Accept-Encoding")
+	if !ctx.Proxy.KeepAcceptEncoding {
+		// If no Accept-Encoding header exists, Transport will add the headers it can accept
+		// and would wrap the response body with the relevant reader.
+		r.Header.Del("Accept-Encoding")
+	}
 	// curl can add that, see
 	// https://jdebp.eu./FGA/web-proxy-connection-header.html
 	r.Header.Del("Proxy-Connection")
@@ -153,7 +169,6 @@ func (proxy *ProxyHttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request)
 			if err != nil {
 				ctx.Error = err
 				resp = proxy.filterResponse(nil, ctx)
-
 			}
 			if resp != nil {
 				ctx.Logf("Received response %v", resp.Status)