From 42850d6ea05719e3c11900c280e78ad8c6dfea42 Mon Sep 17 00:00:00 2001 From: Ivan Ilves Date: Tue, 17 Oct 2017 21:45:50 +0200 Subject: [PATCH] :fire: useless functionality and improve structure --- README.md | 5 +- app/app.go | 114 --------------------------- auth/auth.go | 2 +- docker/client/client.go | 4 +- docker/config/config.go | 25 ++---- docker/config/config_test.go | 21 ----- main.go | 105 +++++++++++++++--------- tag/registry/registry.go | 2 +- util/util.go | 50 ++++++++++++ app/app_test.go => util/util_test.go | 2 +- 10 files changed, 134 insertions(+), 196 deletions(-) delete mode 100644 app/app.go create mode 100644 util/util.go rename app/app_test.go => util/util_test.go (99%) diff --git a/README.md b/README.md index c966c26..c371d8f 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,11 @@ You could use `lstags`, if you ... ## How do I use it myself? I run `lstags` inside a Cron Job on my Kubernetes worker nodes to poll my own Docker registry for a new [stable] images. ``` -lstags --pull -u myuser -p mypass registry.ivanilves.local/tools/sicario~/v1\\.[0-9]+$/ +lstags --pull registry.ivanilves.local/tools/sicario~/v1\\.[0-9]+$/ ``` +**NB!** In case you use private registry with authentication, make sure your Docker client knows how to authenticate against it! +`lstags` will reuse credentials saved by Docker client in its `config.json` file, one usually found at `~/.docker/config.json` + ... and following cronjob runs on my CI server to ensure I always have latest Ubuntu 14.04 and 16.04 images to play with: ``` lstags --pull ubuntu~/^1[46]\\.04$/ diff --git a/app/app.go b/app/app.go deleted file mode 100644 index c316671..0000000 --- a/app/app.go +++ /dev/null @@ -1,114 +0,0 @@ -package app - -import ( - "errors" - "os" - "regexp" - "strings" - - "github.com/jessevdk/go-flags" -) - -const dockerHub = "registry.hub.docker.com" - -// Options represents configuration options we extract from passed command line arguments -type Options struct { - DefaultRegistry string `short:"r" long:"default-registry" default:"registry.hub.docker.com" description:"Default Docker registry to use" env:"DEFAULT_REGISTRY"` - DockerJSON string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials (use it, please <3)" env:"DOCKER_JSON"` - Username string `short:"u" long:"username" default:"" description:"Override Docker registry username (not recommended, please use JSON file)" env:"USERNAME"` - Password string `short:"p" long:"password" default:"" description:"Override Docker registry password (not recommended, please use JSON file)" env:"PASSWORD"` - ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"` - Pull bool `short:"P" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"` - PushRegistry string `short:"U" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"` - PushPrefix string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"` - InsecureRegistry bool `short:"i" long:"insecure-registry" description:"Use insecure plain-HTTP connection to registries (not recommended!)" env:"INSECURE_REGISTRY"` - TraceRequests bool `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"` - DoNotFail bool `short:"N" long:"do-not-fail" description:"Do not fail on errors (could be dangerous!)" env:"DO_NOT_FAIL"` - Version bool `short:"V" long:"version" description:"Show version and exit"` - Positional struct { - Repositories []string `positional-arg-name:"REPO1 REPO2 REPOn" description:"Docker repositories to operate on, e.g.: alpine nginx~/1\\.13\\.5$/ busybox~/1.27.2/"` - } `positional-args:"yes" required:"yes"` -} - -// ParseFlags parses command line arguments and applies some additional post-processing -func ParseFlags() (*Options, error) { - var err error - - o := &Options{} - - _, err = flags.Parse(o) - if err != nil { - os.Exit(1) - } - - err = o.postprocess() - if err != nil { - return nil, err - } - - return o, nil -} - -func (o *Options) postprocess() error { - if !o.Version && len(o.Positional.Repositories) == 0 { - return errors.New("Need at least one repository name, e.g. 'nginx~/^1\\\\.13/' or 'mesosphere/chronos'") - } - - if o.PushRegistry != "" { - o.Pull = true - } - - return nil -} - -// GetWebSchema gets web schema we will use to talk to Docker registry (HTTP||HTTPS) -func (o *Options) GetWebSchema() string { - if o.InsecureRegistry { - return "http://" - } - - return "https://" -} - -// SeparateFilterAndRepo separates repository name from optional regex filter -func SeparateFilterAndRepo(repoWithFilter string) (string, string, error) { - parts := strings.Split(repoWithFilter, "~") - - repository := parts[0] - - if len(parts) < 2 { - return repository, ".*", nil - } - - if len(parts) > 2 { - return "", "", errors.New("Unable to trim filter from repository (too many '~'!): " + repoWithFilter) - } - - f := parts[1] - - if !strings.HasPrefix(f, "/") || !strings.HasSuffix(f, "/") { - return "", "", errors.New("Filter should be passed in a form: /REGEXP/") - } - - filter := f[1 : len(f)-1] - - return repository, filter, nil -} - -// DoesMatch wraps over regexp.MatchString to cowardly escape errors -func DoesMatch(s, ex string) bool { - matched, err := regexp.MatchString(ex, s) - if err != nil { - return false - } - - return matched -} - -// GeneratePathFromHostname generates "/"-delimited path from a hostname[:port] -func GeneratePathFromHostname(hostname string) string { - allParts := strings.Split(hostname, ":") - hostPart := allParts[0] - - return "/" + strings.Replace(hostPart, ".", "/", -1) -} diff --git a/auth/auth.go b/auth/auth.go index ffee8be..3ab832f 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -11,7 +11,7 @@ import ( ) // WebSchema defines how do we connect to remote web servers -var WebSchema = "https://" +const WebSchema = "https://" // TokenResponse is an abstraction for aggregated token-related information we get from authentication services type TokenResponse interface { diff --git a/docker/client/client.go b/docker/client/client.go index 5d36bd9..db653bd 100644 --- a/docker/client/client.go +++ b/docker/client/client.go @@ -62,7 +62,7 @@ func buildImageListOptions(repo string) (types.ImageListOptions, error) { // Pull pulls Docker image specified func (dc *DockerClient) Pull(ref string) error { - registryAuth, _ := dc.cnf.GetRegistryAuth( + registryAuth := dc.cnf.GetRegistryAuth( docker.GetRegistry(ref), ) @@ -83,7 +83,7 @@ func (dc *DockerClient) Pull(ref string) error { // Push pushes Docker image specified func (dc *DockerClient) Push(ref string) error { - registryAuth, _ := dc.cnf.GetRegistryAuth( + registryAuth := dc.cnf.GetRegistryAuth( docker.GetRegistry(ref), ) diff --git a/docker/config/config.go b/docker/config/config.go index 028c63c..f36ba58 100644 --- a/docker/config/config.go +++ b/docker/config/config.go @@ -8,12 +8,6 @@ import ( "strings" ) -// DefaultUsername is the username we use if none is defined in config -var DefaultUsername string - -// DefaultPassword is the password we use if none is defined in config -var DefaultPassword string - // DefaultDockerJSON is the defalt path for Docker JSON config file var DefaultDockerJSON = "~/.docker/config.json" @@ -29,11 +23,6 @@ type Auth struct { B64Auth string `json:"auth"` } -// AreDefaultCredentialsDefined tells if default username & password are defined -func AreDefaultCredentialsDefined() bool { - return DefaultUsername != "" || DefaultPassword != "" -} - // IsEmpty return true if structure has no relevant data inside func (c *Config) IsEmpty() bool { return len(c.Auths) == 0 @@ -41,21 +30,23 @@ func (c *Config) IsEmpty() bool { // GetCredentials gets per-registry credentials from loaded Docker config func (c *Config) GetCredentials(registry string) (string, string, bool) { - _, defined := c.usernames[registry] - if !defined { - return DefaultUsername, DefaultUsername, false + if _, defined := c.usernames[registry]; !defined { + return "", "", false } return c.usernames[registry], c.passwords[registry], true } // GetRegistryAuth gets per-registry base64 authentication string -func (c *Config) GetRegistryAuth(registry string) (string, bool) { +func (c *Config) GetRegistryAuth(registry string) string { username, password, defined := c.GetCredentials(registry) + if !defined { + return "" + } - jsonString := fmt.Sprintf("{ \"username\": \"%s\", \"password\": \"%s\" }", username, password) + jsonString := fmt.Sprintf(`{ "username": "%s", "password": "%s" }`, username, password) - return base64.StdEncoding.EncodeToString([]byte(jsonString)), defined + return base64.StdEncoding.EncodeToString([]byte(jsonString)) } // Load loads a Config object from Docker JSON configuration file specified diff --git a/docker/config/config_test.go b/docker/config/config_test.go index db8b864..f417a63 100644 --- a/docker/config/config_test.go +++ b/docker/config/config_test.go @@ -8,27 +8,6 @@ const configFile = "../../fixtures/docker/config.json" const irrelevantConfigFile = "../../fixtures/docker/config.json.irrelevant" const invalidConfigFile = "../../fixtures/docker/config.json.invalid" -func TestAreDefaultCredentialsDefined(t *testing.T) { - if AreDefaultCredentialsDefined() { - t.Fatalf( - "By default no credentials should be defined, but they are: username '%s', password '%s'", - DefaultUsername, - DefaultPassword, - ) - } - - DefaultUsername = "user" - DefaultPassword = "pass" - - if !AreDefaultCredentialsDefined() { - t.Fatalf( - "Credentials should be defined now, but we get contrary: username '%s', password '%s'", - DefaultUsername, - DefaultPassword, - ) - } -} - func TestLoad(t *testing.T) { examples := map[string]string{ "registry.company.io": "user1:pass1", diff --git a/main.go b/main.go index 80319e9..9302860 100644 --- a/main.go +++ b/main.go @@ -1,10 +1,12 @@ package main import ( + "errors" "fmt" "os" - "github.com/ivanilves/lstags/app" + "github.com/jessevdk/go-flags" + "github.com/ivanilves/lstags/auth" "github.com/ivanilves/lstags/docker" dockerclient "github.com/ivanilves/lstags/docker/client" @@ -12,30 +14,42 @@ import ( "github.com/ivanilves/lstags/tag" "github.com/ivanilves/lstags/tag/local" "github.com/ivanilves/lstags/tag/registry" + "github.com/ivanilves/lstags/util" ) -var doNotFail bool +// Options represents configuration options we extract from passed command line arguments +type Options struct { + DockerJSON string `short:"j" long:"docker-json" default:"~/.docker/config.json" description:"JSON file with credentials" env:"DOCKER_JSON"` + Pull bool `short:"P" long:"pull" description:"Pull Docker images matched by filter (will use local Docker deamon)" env:"PULL"` + PushRegistry string `short:"U" long:"push-registry" description:"[Re]Push pulled images to a specified remote registry" env:"PUSH_REGISTRY"` + PushPrefix string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"` + ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"` + TraceRequests bool `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"` + DoNotFail bool `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"` + Version bool `short:"V" long:"version" description:"Show version and exit"` + Positional struct { + Repositories []string `positional-arg-name:"REPO1 REPO2 REPOn" description:"Docker repositories to operate on, e.g.: alpine nginx~/1\\.13\\.5$/ busybox~/1.27.2/"` + } `positional-args:"yes" required:"yes"` +} + +var doNotFail = false -func suicide(err error) { +func suicide(err error, critical bool) { fmt.Printf("%s\n", err.Error()) - if !doNotFail { + if doNotFail || critical { os.Exit(1) } } -func getVersion() string { - return VERSION -} +func parseFlags() (*Options, error) { + var err error -func getAuthorization(t auth.TokenResponse) string { - return t.Method() + " " + t.Token() -} + o := &Options{} -func main() { - o, err := app.ParseFlags() + _, err = flags.Parse(o) if err != nil { - suicide(err) + os.Exit(1) // YES! Just exit! } if o.Version { @@ -43,21 +57,40 @@ func main() { os.Exit(0) } - dockerconfig.DefaultUsername = o.Username - dockerconfig.DefaultPassword = o.Password + if len(o.Positional.Repositories) == 0 { + return nil, errors.New("Need at least one repository name, e.g. 'nginx~/^1\\\\.13/' or 'mesosphere/chronos'") + } - dockerConfig, err := dockerconfig.Load(o.DockerJSON) - if err != nil { - suicide(err) + if o.PushRegistry != "" { + o.Pull = true } registry.TraceRequests = o.TraceRequests - auth.WebSchema = o.GetWebSchema() - registry.WebSchema = o.GetWebSchema() - doNotFail = o.DoNotFail + return o, nil +} + +func getVersion() string { + return VERSION +} + +func getAuthorization(t auth.TokenResponse) string { + return t.Method() + " " + t.Token() +} + +func main() { + o, err := parseFlags() + if err != nil { + suicide(err, true) + } + + dockerConfig, err := dockerconfig.Load(o.DockerJSON) + if err != nil { + suicide(err, true) + } + const format = "%-12s %-45s %-15s %-25s %s\n" fmt.Printf(format, "", "", "<(local) ID>", "", "") @@ -65,11 +98,9 @@ func main() { pullCount := 0 pushCount := 0 - pullAuths := make(map[string]string) - dc, err := dockerclient.New(dockerConfig) if err != nil { - suicide(err) + suicide(err, true) } type tagResult struct { @@ -82,10 +113,10 @@ func main() { trc := make(chan tagResult, repoCount) for _, r := range o.Positional.Repositories { - go func(r string, o *app.Options, trc chan tagResult) { - repository, filter, err := app.SeparateFilterAndRepo(r) + go func(r string, o *Options, trc chan tagResult) { + repository, filter, err := util.SeparateFilterAndRepo(r) if err != nil { - suicide(err) + suicide(err, true) } registryName := docker.GetRegistry(repository) @@ -95,27 +126,25 @@ func main() { username, password, _ := dockerConfig.GetCredentials(registryName) - pullAuths[repoLocalName], _ = dockerConfig.GetRegistryAuth(registryName) - tresp, err := auth.NewToken(registryName, repoRegistryName, username, password) if err != nil { - suicide(err) + suicide(err, true) } authorization := getAuthorization(tresp) registryTags, err := registry.FetchTags(registryName, repoRegistryName, authorization, o.ConcurrentRequests) if err != nil { - suicide(err) + suicide(err, true) } imageSummaries, err := dc.ListImagesForRepo(repoLocalName) if err != nil { - suicide(err) + suicide(err, true) } localTags, err := local.FetchTags(repoLocalName, imageSummaries) if err != nil { - suicide(err) + suicide(err, true) } sortedKeys, names, joinedTags := tag.Join(registryTags, localTags) @@ -126,7 +155,7 @@ func main() { tg := joinedTags[name] - if !app.DoesMatch(tg.GetName(), filter) { + if !util.DoesMatch(tg.GetName(), filter) { continue } @@ -177,7 +206,7 @@ func main() { fmt.Printf("PULLING %s\n", ref) err := dc.Pull(ref) if err != nil { - suicide(err) + suicide(err, false) } done <- true @@ -207,7 +236,7 @@ func main() { for _, tg := range tags { prefix := o.PushPrefix if prefix == "" { - prefix = app.GeneratePathFromHostname(registry) + prefix = util.GeneratePathFromHostname(registry) } srcRef := repo + ":" + tg.GetName() @@ -217,12 +246,12 @@ func main() { err := dc.Tag(srcRef, dstRef) if err != nil { - suicide(err) + suicide(err, true) } err = dc.Push(dstRef) if err != nil { - suicide(err) + suicide(err, false) } done <- true diff --git a/tag/registry/registry.go b/tag/registry/registry.go index c8fb2a0..55d6d64 100644 --- a/tag/registry/registry.go +++ b/tag/registry/registry.go @@ -18,7 +18,7 @@ import ( ) // WebSchema defines how do we connect to remote web servers -var WebSchema = "https://" +const WebSchema = "https://" // TraceRequests defines if we should print out HTTP request URLs and response headers/bodies var TraceRequests = false diff --git a/util/util.go b/util/util.go new file mode 100644 index 0000000..75c61fc --- /dev/null +++ b/util/util.go @@ -0,0 +1,50 @@ +package util + +import ( + "errors" + "regexp" + "strings" +) + +// SeparateFilterAndRepo separates repository name from optional regex filter +func SeparateFilterAndRepo(repoWithFilter string) (string, string, error) { + parts := strings.Split(repoWithFilter, "~") + + repository := parts[0] + + if len(parts) < 2 { + return repository, ".*", nil + } + + if len(parts) > 2 { + return "", "", errors.New("Unable to trim filter from repository (too many '~'!): " + repoWithFilter) + } + + f := parts[1] + + if !strings.HasPrefix(f, "/") || !strings.HasSuffix(f, "/") { + return "", "", errors.New("Filter should be passed in a form: /REGEXP/") + } + + filter := f[1 : len(f)-1] + + return repository, filter, nil +} + +// DoesMatch wraps over regexp.MatchString to cowardly escape errors +func DoesMatch(s, ex string) bool { + matched, err := regexp.MatchString(ex, s) + if err != nil { + return false + } + + return matched +} + +// GeneratePathFromHostname generates "/"-delimited path from a hostname[:port] +func GeneratePathFromHostname(hostname string) string { + allParts := strings.Split(hostname, ":") + hostPart := allParts[0] + + return "/" + strings.Replace(hostPart, ".", "/", -1) +} diff --git a/app/app_test.go b/util/util_test.go similarity index 99% rename from app/app_test.go rename to util/util_test.go index 2ee4a56..3b4543a 100644 --- a/app/app_test.go +++ b/util/util_test.go @@ -1,4 +1,4 @@ -package app +package util import ( "testing"