From 1c1172bda63f796f2f37c276444c8595fa7685af Mon Sep 17 00:00:00 2001 From: Ivan Ilves Date: Wed, 8 Nov 2017 09:19:20 +0100 Subject: [PATCH 1/2] Retry failed registry HTTP requests --- main.go | 29 ++++++++++++++--------- tag/remote/remote.go | 55 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 65 insertions(+), 19 deletions(-) diff --git a/main.go b/main.go index d639ad5..b1cd263 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "strings" + "time" "github.com/jessevdk/go-flags" @@ -19,17 +20,19 @@ import ( // Options represents configuration options we extract from passed command line arguments type Options struct { - DockerJSON string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials" env:"DOCKER_JSON"` - Pull bool `short:"p" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"` - Push bool `short:"P" long:"push" description:"Push Docker images matched by filter to some registry (See 'push-registry')" env:"PUSH"` - PushRegistry string `short:"r" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"` - PushPrefix string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"` - PushUpdate bool `short:"U" long:"push-update" description:"Update our pushed images if remote image digest changes" env:"PUSH_UPDATE"` - ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"` - InsecureRegistryEx string `short:"I" long:"insecure-registry-ex" description:"Expression to match insecure registry hostnames" env:"INSECURE_REGISTRY_EX"` - TraceRequests bool `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"` - DoNotFail bool `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"` - Version bool `short:"V" long:"version" description:"Show version and exit"` + DockerJSON string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials" env:"DOCKER_JSON"` + Pull bool `short:"p" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"` + Push bool `short:"P" long:"push" description:"Push Docker images matched by filter to some registry (See 'push-registry')" env:"PUSH"` + PushRegistry string `short:"r" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"` + PushPrefix string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"` + PushUpdate bool `short:"U" long:"push-update" description:"Update our pushed images if remote image digest changes" env:"PUSH_UPDATE"` + ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"` + RetryRequests int `short:"y" long:"retry-requests" default:"2" description:"Number of retries for failed registry HTTP requests" env:"RETRY_REQUESTS"` + RetryDelay time.Duration `short:"D" long:"retry-delay" default:"30s" description:"Delay between retries of failed registry requests" env:"RETRY_DELAY"` + InsecureRegistryEx string `short:"I" long:"insecure-registry-ex" description:"Expression to match insecure registry hostnames" env:"INSECURE_REGISTRY_EX"` + TraceRequests bool `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"` + DoNotFail bool `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"` + Version bool `short:"V" long:"version" description:"Show version and exit"` Positional struct { Repositories []string `positional-arg-name:"REPO1 REPO2 REPOn" description:"Docker repositories to operate on, e.g.: alpine nginx~/1\\.13\\.5$/ busybox~/1.27.2/"` } `positional-args:"yes" required:"yes"` @@ -74,6 +77,10 @@ func parseFlags() (*Options, error) { remote.ConcurrentRequests = o.ConcurrentRequests + remote.RetryRequests = o.RetryRequests + + remote.RetryDelay = o.RetryDelay + if o.InsecureRegistryEx != "" { docker.InsecureRegistryEx = o.InsecureRegistryEx } diff --git a/tag/remote/remote.go b/tag/remote/remote.go index 6442c10..151ac53 100644 --- a/tag/remote/remote.go +++ b/tag/remote/remote.go @@ -23,13 +23,15 @@ import ( // ConcurrentRequests defines maximum number of concurrent requests we could maintain against the registry var ConcurrentRequests = 32 +// RetryRequests is a number of retries we do in case of request failure +var RetryRequests = 0 + +// RetryDelay is a delay between retries of failed requests to the registry +var RetryDelay = 5 * time.Second + // TraceRequests defines if we should print out HTTP request URLs and response headers/bodies var TraceRequests = false -func getAuthorizationType(authorization string) string { - return strings.Split(authorization, " ")[0] -} - func getRequestID() string { data := make([]byte, 10) @@ -73,7 +75,7 @@ func httpRequest(url, authorization, mode string) (*http.Response, error) { return nil, err } if resp.StatusCode != 200 { - return nil, errors.New("Bad response status: " + resp.Status + " >> " + url) + return resp, errors.New("Bad response status: " + resp.Status + " >> " + url) } if TraceRequests { @@ -91,6 +93,43 @@ func httpRequest(url, authorization, mode string) (*http.Response, error) { return resp, nil } +func httpRetriableRequest(url, authorization, mode string) (*http.Response, error) { + tries := 1 + + if RetryRequests > 0 { + tries = tries + RetryRequests + } + + var resp *http.Response + var err error + + for try := 1; try <= tries; try++ { + resp, err := httpRequest(url, authorization, mode) + + if err == nil { + return resp, nil + } + + if resp.StatusCode >= 400 && resp.StatusCode < 500 { + return nil, err + } + + if try < tries { + fmt.Printf( + "Will retry '%s' [%s] in a %v\n=> Error: %s\n", + url, + mode, + RetryDelay, + err.Error(), + ) + + time.Sleep(RetryDelay) + } + } + + return resp, err +} + type tagNameInfo struct { TagNames []string `json:"tags"` } @@ -109,7 +148,7 @@ func parseTagNamesJSON(data io.ReadCloser) ([]string, error) { func fetchTagNames(registry, repoPath, authorization string) ([]string, error) { url := docker.WebSchema(registry) + registry + "/v2/" + repoPath + "/tags/list" - resp, err := httpRequest(url, authorization, "v2") + resp, err := httpRetriableRequest(url, authorization, "v2") if err != nil { return nil, err } @@ -144,7 +183,7 @@ func extractMetadataFromHistory(s string) (imageMetadata, error) { } func fetchMetadata(url, authorization string) (imageMetadata, error) { - resp, err := httpRequest(url, authorization, "v1") + resp, err := httpRetriableRequest(url, authorization, "v1") if err != nil { return imageMetadata{}, nil } @@ -171,7 +210,7 @@ func fetchMetadata(url, authorization string) (imageMetadata, error) { } func fetchDigest(url, authorization string) (string, error) { - resp, err := httpRequest(url, authorization, "v2") + resp, err := httpRetriableRequest(url, authorization, "v2") if err != nil { return "", err } From c60b4d203743cc5511d0613a6338fdbb281da237 Mon Sep 17 00:00:00 2001 From: Ivan Ilves Date: Thu, 9 Nov 2017 17:57:53 +0100 Subject: [PATCH 2/2] Added backoff delay to save the Internet! --- tag/remote/remote.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tag/remote/remote.go b/tag/remote/remote.go index 151ac53..f16ee98 100644 --- a/tag/remote/remote.go +++ b/tag/remote/remote.go @@ -124,6 +124,8 @@ func httpRetriableRequest(url, authorization, mode string) (*http.Response, erro ) time.Sleep(RetryDelay) + + RetryDelay += RetryDelay } }