diff --git a/.travis.yml b/.travis.yml index 4b00216..769be18 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,9 @@ before_script: - git remote set-url --push origin https://${GITHUB_TOKEN}@github.com/ivanilves/lstags.git script: - - make package-test - - make integration-test - - make shell-test + - make unit-test + - make whitebox-integration-test + - make blackbox-integration-test - make lint - make vet diff --git a/Makefile b/Makefile index 9cf163b..5262583 100644 --- a/Makefile +++ b/Makefile @@ -8,21 +8,36 @@ prepare: dep: dep ensure -v -test: package-test integration-test +test: unit-test whitebox-integration-test -package-test: +unit-test: @find \ -mindepth 2 -type f ! -path "./vendor/*" -name "*_test.go" \ | xargs dirname \ | xargs -i sh -c "pushd {}; go test -v || exit 1; popd" -integration-test: +whitebox-integration-test: go test -v env: env -shell-test: build shell-test-alpine shell-test-wrong-image shell-test-pull-public shell-test-pull-private +docker-json: + test -n "${DOCKER_JSON}" && mkdir -p `dirname "${DOCKER_JSON}"` && touch "${DOCKER_JSON}" && chmod 0600 "${DOCKER_JSON}" \ + && echo "{ \"auths\": { \"registry.hub.docker.com\": { \"auth\": \"${DOCKERHUB_AUTH}\" } } }" >${DOCKER_JSON} + +start-local-registry: + test ${REGISTRY_PORT} && docker run -d -p ${REGISTRY_PORT}:5000 --name lstags-registry registry:2 + +stop-local-registry: + docker rm -f lstags-registry + +blackbox-integration-test: build \ + shell-test-alpine \ + shell-test-wrong-image \ + shell-test-pull-public \ + shell-test-pull-private \ + shell-test-push-local shell-test-alpine: ./lstags alpine | egrep "\salpine:latest" @@ -35,18 +50,25 @@ shell-test-pull-public: ./lstags --pull ${DOCKERHUB_PUBLIC_REPO}~/latest/ shell-test-pull-private: DOCKER_JSON:=tmp/docker.json.private-repo -shell-test-pull-private: - mkdir -p tmp +shell-test-pull-private: docker-json if [[ -n "${DOCKERHUB_PRIVATE_REPO}" && -n "${DOCKERHUB_AUTH}" ]]; then\ - touch "${DOCKER_JSON}" && chmod 0600 "${DOCKER_JSON}" \ - && echo "{ \"auths\": { \"registry.hub.docker.com\": { \"auth\": \"${DOCKERHUB_AUTH}\" } } }" >"${DOCKER_JSON}"\ - && ./lstags -j "${DOCKER_JSON}" --pull ${DOCKERHUB_PRIVATE_REPO}~/latest/; else echo "DOCKERHUB_PRIVATE_REPO or DOCKERHUB_AUTH not set!";\ + ./lstags -j "${DOCKER_JSON}" --pull ${DOCKERHUB_PRIVATE_REPO}~/latest/; \ + else \ + echo "DOCKERHUB_PRIVATE_REPO or DOCKERHUB_AUTH not set!"; \ fi -lint: ERRORS:=$(shell find . -name "*.go" ! -path "./vendor/*" | xargs -i golint {}) +shell-test-push-local: REGISTRY_PORT:=5757 +shell-test-push-local: + ${MAKE} --no-print-directory stop-local-registry &>/dev/null | true + ${MAKE} --no-print-directory start-local-registry REGISTRY_PORT=${REGISTRY_PORT} + ./lstags --push-registry=localhost:${REGISTRY_PORT} --push-prefix=/qa alpine~/3.6/ + ./lstags localhost:${REGISTRY_PORT}/qa/library/alpine + ${MAKE} --no-print-directory stop-local-registry + +lint: ERRORS:=$(shell find . -name "*.go" ! -path "./vendor/*" | xargs -i golint {} | tr '`' '|') lint: fail-on-errors -vet: ERRORS:=$(shell find . -name "*.go" ! -path "./vendor/*" | xargs -i go tool vet {}) +vet: ERRORS:=$(shell find . -name "*.go" ! -path "./vendor/*" | xargs -i go tool vet {} | tr '`' '|') vet: fail-on-errors fail-on-errors: diff --git a/auth/auth.go b/auth/auth.go index 24d8c7d..9aca357 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -8,11 +8,9 @@ import ( "github.com/ivanilves/lstags/auth/basic" "github.com/ivanilves/lstags/auth/bearer" "github.com/ivanilves/lstags/auth/none" + "github.com/ivanilves/lstags/docker" ) -// WebSchema defines how do we connect to remote web servers -const WebSchema = "https://" - // TokenResponse is an abstraction for aggregated token-related information we get from authentication services type TokenResponse interface { Method() string @@ -68,7 +66,7 @@ func validateParams(method string, params map[string]string) (map[string]string, // * detects authentication type (e.g. Bearer or Basic) // * delegates actual authentication to type-specific implementation func NewToken(registry, repository, username, password string) (TokenResponse, error) { - url := WebSchema + registry + "/v2" + url := docker.WebSchema(registry) + registry + "/v2" resp, err := http.Get(url) if err != nil { diff --git a/docker/client/client.go b/docker/client/client.go index db653bd..e27e681 100644 --- a/docker/client/client.go +++ b/docker/client/client.go @@ -89,7 +89,7 @@ func (dc *DockerClient) Push(ref string) error { pushOptions := types.ImagePushOptions{RegistryAuth: registryAuth} if registryAuth == "" { - pushOptions = types.ImagePushOptions{} + pushOptions = types.ImagePushOptions{RegistryAuth: "IA=="} } resp, err := dc.cli.ImagePush(context.Background(), ref, pushOptions) diff --git a/docker/docker.go b/docker/docker.go index 4bd7d2d..771bb7e 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -2,11 +2,16 @@ package docker import ( "strings" + + "github.com/ivanilves/lstags/util" ) // DefaultRegistry is a registry we use if none could be resolved from image ref var DefaultRegistry = "registry.hub.docker.com" +// InsecureRegistryEx contains a regex string to match insecure registries +var InsecureRegistryEx = "^(127\\..*|::1|localhost)(:[0-9]+)?$" + // GetRegistry tries to get Docker registry name from a repository or reference // .. if it is not possible it returns default registry name (usually Docker Hub) func GetRegistry(repoOrRef string) string { @@ -64,3 +69,12 @@ func GetRepoPath(repository, registry string) string { return repository } + +// WebSchema tells us if we should use HTTP or HTTPS +func WebSchema(registry string) string { + if util.DoesMatch(registry, InsecureRegistryEx) { + return "http://" + } + + return "https://" +} diff --git a/docker/docker_test.go b/docker/docker_test.go index bb28e39..188bd06 100644 --- a/docker/docker_test.go +++ b/docker/docker_test.go @@ -95,3 +95,54 @@ func TestGetRepoPath(t *testing.T) { } } } + +func TestWebSchema(t *testing.T) { + examples := map[string]string{ + "localhost": "http://", + "localhost:4000": "http://", + "127.0.0.1": "http://", + "127.0.0.1:5000": "http://", + "remotehost": "https://", + "reg.hype.io": "https://", + "reg.hype.io:3128": "https://", + } + + for input, expected := range examples { + output := WebSchema(input) + + if output != expected { + t.Fatalf( + "Got unexpected schema '%s' for registry hostname '%s', while expecting for '%s'", + output, + input, + expected, + ) + } + } + + InsecureRegistryEx = ".*" + + for input := range examples { + output := WebSchema(input) + + if output != "http://" { + t.Fatalf( + "Expected schema 'http://' for registry hostname '%s' in this case", + input, + ) + } + } + + InsecureRegistryEx = "i.do.not.match.anything" + + for input := range examples { + output := WebSchema(input) + + if output != "https://" { + t.Fatalf( + "Expected schema 'https://' for registry hostname '%s' in this case", + input, + ) + } + } +} diff --git a/main.go b/main.go index b502840..78e5c09 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ type Options struct { PushPrefix string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"` PushUpdate bool `short:"U" long:"push-update" description:"Update our pushed images if remote image digest changes" env:"PUSH_UPDATE"` ConcurrentRequests int `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"` + InsecureRegistryEx string `short:"I" long:"insecure-registry-ex" description:"Expression to match insecure registry hostnames" env:"INSECURE_REGISTRY_EX"` TraceRequests bool `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"` DoNotFail bool `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"` Version bool `short:"V" long:"version" description:"Show version and exit"` @@ -40,7 +41,7 @@ var doNotFail = false func suicide(err error, critical bool) { fmt.Printf("%s\n", err.Error()) - if doNotFail || critical { + if !doNotFail || critical { os.Exit(1) } } @@ -72,6 +73,10 @@ func parseFlags() (*Options, error) { return nil, errors.New("You either '--pull' or '--push', not both") } + if o.InsecureRegistryEx != "" { + docker.InsecureRegistryEx = o.InsecureRegistryEx + } + remote.TraceRequests = o.TraceRequests doNotFail = o.DoNotFail diff --git a/tag/remote/remote.go b/tag/remote/remote.go index b6889ca..be3e3d4 100644 --- a/tag/remote/remote.go +++ b/tag/remote/remote.go @@ -14,13 +14,11 @@ import ( "strings" "time" + "github.com/ivanilves/lstags/docker" "github.com/ivanilves/lstags/tag" "github.com/ivanilves/lstags/util" ) -// WebSchema defines how do we connect to remote web servers -const WebSchema = "https://" - // TraceRequests defines if we should print out HTTP request URLs and response headers/bodies var TraceRequests = false @@ -105,7 +103,7 @@ func parseTagNamesJSON(data io.ReadCloser) ([]string, error) { } func fetchTagNames(registry, repo, authorization string) ([]string, error) { - url := WebSchema + registry + "/v2/" + repo + "/tags/list" + url := docker.WebSchema(registry) + registry + "/v2/" + repo + "/tags/list" resp, err := httpRequest(url, authorization, "v2") if err != nil { @@ -183,7 +181,7 @@ func fetchDigest(url, authorization string) (string, error) { } func fetchDetails(registry, repo, tagName, authorization string) (string, imageMetadata, error) { - url := WebSchema + registry + "/v2/" + repo + "/manifests/" + tagName + url := docker.WebSchema(registry) + registry + "/v2/" + repo + "/manifests/" + tagName dc := make(chan string, 0) mc := make(chan imageMetadata, 0)