diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d4995272..ea09c0617 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +* Use retriable HTTP client in proxy mode. [#883](https://github.com/elastic/package-registry/pull/883) + ### Deprecated ### Known Issues diff --git a/go.mod b/go.mod index 01f85fcf2..e6cf059f7 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/felixge/httpsnoop v1.0.3 github.com/fsouza/fake-gcs-server v1.40.2 github.com/gorilla/mux v1.8.0 + github.com/hashicorp/go-retryablehttp v0.7.1 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 github.com/magefile/mage v1.14.0 github.com/pkg/errors v0.9.1 @@ -42,6 +43,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect github.com/googleapis/gax-go/v2 v2.4.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/jcchavezs/porto v0.3.0 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect diff --git a/go.sum b/go.sum index db2ec1481..5ef340328 100644 --- a/go.sum +++ b/go.sum @@ -242,6 +242,12 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= diff --git a/proxymode/logger.go b/proxymode/logger.go new file mode 100644 index 000000000..fbfd8979f --- /dev/null +++ b/proxymode/logger.go @@ -0,0 +1,53 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package proxymode + +import ( + "github.com/hashicorp/go-retryablehttp" + "go.uber.org/zap" +) + +type zapLoggerAdapter struct { + target *zap.Logger +} + +var _ retryablehttp.LeveledLogger = new(zapLoggerAdapter) + +func withZapLoggerAdapter(target *zap.Logger) retryablehttp.LeveledLogger { + return &zapLoggerAdapter{ + target: target, + } +} + +func (a zapLoggerAdapter) Error(msg string, keysAndValues ...interface{}) { + a.target.Error(msg, keysAndValuesAsZapFields(keysAndValues...)...) +} + +func (a zapLoggerAdapter) Info(msg string, keysAndValues ...interface{}) { + a.target.Info(msg, keysAndValuesAsZapFields(keysAndValues...)...) +} + +func (a zapLoggerAdapter) Debug(msg string, keysAndValues ...interface{}) { + a.target.Debug(msg, keysAndValuesAsZapFields(keysAndValues...)...) +} + +func (a zapLoggerAdapter) Warn(msg string, keysAndValues ...interface{}) { + a.target.Warn(msg, keysAndValuesAsZapFields(keysAndValues...)...) +} + +// keysAndValuesAsZapFields function transforms the LeveledLogger arguments to the zap.Logger interface. +func keysAndValuesAsZapFields(keysAndValues ...interface{}) []zap.Field { + fields := make([]zap.Field, len(keysAndValues)/2) + var j int + for i := 0; i < len(keysAndValues); i += 2 { + key, ok := keysAndValues[i].(string) + if !ok { + continue // something is wrong with the key, string expected + } + fields[j] = zap.Any(key, keysAndValues[i+1]) + j++ + } + return fields +} diff --git a/proxymode/proxymode.go b/proxymode/proxymode.go index 46ed3f596..c2751ad6b 100644 --- a/proxymode/proxymode.go +++ b/proxymode/proxymode.go @@ -5,13 +5,16 @@ package proxymode import ( + "context" "encoding/json" "fmt" "net/http" "net/url" + "strings" "time" "github.com/gorilla/mux" + "github.com/hashicorp/go-retryablehttp" "github.com/pkg/errors" "go.uber.org/zap" @@ -22,7 +25,7 @@ import ( type ProxyMode struct { options ProxyOptions - httpClient *http.Client + httpClient *retryablehttp.Client destinationURL *url.URL resolver *proxyResolver } @@ -48,13 +51,21 @@ func NewProxyMode(options ProxyOptions) (*ProxyMode, error) { return &pm, nil } - pm.httpClient = &http.Client{ - Timeout: 10 * time.Second, - Transport: &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - IdleConnTimeout: 90 * time.Second, + pm.httpClient = &retryablehttp.Client{ + HTTPClient: &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + }, }, + Logger: withZapLoggerAdapter(util.Logger()), + RetryWaitMin: 1 * time.Second, + RetryWaitMax: 15 * time.Second, + RetryMax: 4, + CheckRetry: proxyRetryPolicy, + Backoff: retryablehttp.DefaultBackoff, } var err error @@ -67,6 +78,28 @@ func NewProxyMode(options ProxyOptions) (*ProxyMode, error) { return &pm, nil } +// proxyRetryPolicy function extends the DefaultRetryPolicy to check if the HTTP response content-type +// is application/json. We found occurrences of requests being rejected by an intermittent proxy and causing +// the json.Decoder to fail. +func proxyRetryPolicy(ctx context.Context, resp *http.Response, err error) (bool, error) { + shouldRetry, err := retryablehttp.DefaultRetryPolicy(ctx, resp, err) + if shouldRetry { + return shouldRetry, err + } + + // Chaining Package Registry servers (proxies) is allowed. HTTP client must get to the end of the chain. + locationHeader := resp.Header.Get("location") + if locationHeader != "" { + return false, nil + } + + contentType := resp.Header.Get("content-type") + if strings.HasPrefix(contentType, "application/json") { + return false, nil + } + return true, fmt.Errorf("unexpected content type: %s", contentType) +} + func (pm *ProxyMode) Enabled() bool { return pm.options.Enabled } @@ -79,7 +112,7 @@ func (pm *ProxyMode) Search(r *http.Request) (packages.Packages, error) { proxyURL.Scheme = pm.destinationURL.Scheme proxyURL.User = pm.destinationURL.User - proxyRequest, err := http.NewRequest(http.MethodGet, proxyURL.String(), nil) + proxyRequest, err := retryablehttp.NewRequest(http.MethodGet, proxyURL.String(), nil) if err != nil { return nil, errors.Wrap(err, "can't create proxy request") } @@ -109,7 +142,7 @@ func (pm *ProxyMode) Categories(r *http.Request) ([]packages.Category, error) { proxyURL.Scheme = pm.destinationURL.Scheme proxyURL.User = pm.destinationURL.User - proxyRequest, err := http.NewRequest(http.MethodGet, proxyURL.String(), nil) + proxyRequest, err := retryablehttp.NewRequest(http.MethodGet, proxyURL.String(), nil) if err != nil { return nil, errors.Wrap(err, "can't create proxy request") } @@ -144,7 +177,7 @@ func (pm *ProxyMode) Package(r *http.Request) (*packages.Package, error) { urlPath := fmt.Sprintf("/package/%s/%s/", packageName, packageVersion) proxyURL := pm.destinationURL.ResolveReference(&url.URL{Path: urlPath}) - proxyRequest, err := http.NewRequest(http.MethodGet, proxyURL.String(), nil) + proxyRequest, err := retryablehttp.NewRequest(http.MethodGet, proxyURL.String(), nil) if err != nil { return nil, errors.Wrap(err, "can't create proxy request") }