Merge pull request #81 from ivanilves/implement/insecure-registry

[Re]Implement insecure registry handling

.travis.yml

@@ -16,9 +16,9 @@ before_script:
   - git remote set-url --push origin https://${GITHUB_TOKEN}@github.com/ivanilves/lstags.git
 
 script:
-  - make package-test
-  - make integration-test
-  - make shell-test
+  - make unit-test
+  - make whitebox-integration-test
+  - make blackbox-integration-test
   - make lint
   - make vet
 

Makefile

@@ -8,21 +8,36 @@ prepare:
 dep:
 	dep ensure -v
 
-test: package-test integration-test
+test: unit-test whitebox-integration-test
 
-package-test:
+unit-test:
 	@find \
 		-mindepth 2 -type f ! -path "./vendor/*" -name "*_test.go" \
 		| xargs dirname \
 		| xargs -i sh -c "pushd {}; go test -v || exit 1; popd"
 
-integration-test:
+whitebox-integration-test:
 	go test -v
 
 env:
 	env
 
-shell-test: build shell-test-alpine shell-test-wrong-image shell-test-pull-public shell-test-pull-private
+docker-json:
+	test -n "${DOCKER_JSON}" && mkdir -p `dirname "${DOCKER_JSON}"` && touch "${DOCKER_JSON}" && chmod 0600 "${DOCKER_JSON}" \
+		&& echo "{ \"auths\": { \"registry.hub.docker.com\": { \"auth\": \"${DOCKERHUB_AUTH}\" } } }" >${DOCKER_JSON}
+
+start-local-registry:
+	test ${REGISTRY_PORT} && docker run -d -p ${REGISTRY_PORT}:5000 --name lstags-registry registry:2
+
+stop-local-registry:
+	docker rm -f lstags-registry
+
+blackbox-integration-test: build \
+	shell-test-alpine \
+	shell-test-wrong-image \
+	shell-test-pull-public \
+	shell-test-pull-private \
+	shell-test-push-local
 
 shell-test-alpine:
 	./lstags alpine | egrep "\salpine:latest"
@@ -35,18 +50,25 @@ shell-test-pull-public:
 	./lstags --pull ${DOCKERHUB_PUBLIC_REPO}~/latest/
 
 shell-test-pull-private: DOCKER_JSON:=tmp/docker.json.private-repo
-shell-test-pull-private:
-	mkdir -p tmp
+shell-test-pull-private: docker-json
 	if [[ -n "${DOCKERHUB_PRIVATE_REPO}" && -n "${DOCKERHUB_AUTH}" ]]; then\
-		touch "${DOCKER_JSON}" && chmod 0600 "${DOCKER_JSON}" \
-		&& echo "{ \"auths\": { \"registry.hub.docker.com\": { \"auth\": \"${DOCKERHUB_AUTH}\" } } }" >"${DOCKER_JSON}"\
-		&& ./lstags -j "${DOCKER_JSON}" --pull ${DOCKERHUB_PRIVATE_REPO}~/latest/; else echo "DOCKERHUB_PRIVATE_REPO or DOCKERHUB_AUTH not set!";\
+		./lstags -j "${DOCKER_JSON}" --pull ${DOCKERHUB_PRIVATE_REPO}~/latest/; \
+		else \
+		echo "DOCKERHUB_PRIVATE_REPO or DOCKERHUB_AUTH not set!"; \
 	fi
 
-lint: ERRORS:=$(shell find . -name "*.go" ! -path "./vendor/*" | xargs -i golint {})
+shell-test-push-local: REGISTRY_PORT:=5757
+shell-test-push-local:
+	${MAKE} --no-print-directory stop-local-registry &>/dev/null | true
+	${MAKE} --no-print-directory start-local-registry REGISTRY_PORT=${REGISTRY_PORT}
+	./lstags --push-registry=localhost:${REGISTRY_PORT} --push-prefix=/qa alpine~/3.6/
+	./lstags localhost:${REGISTRY_PORT}/qa/library/alpine
+	${MAKE} --no-print-directory stop-local-registry
+
+lint: ERRORS:=$(shell find . -name "*.go" ! -path "./vendor/*" | xargs -i golint {} | tr '`' '|')
 lint: fail-on-errors
 
-vet: ERRORS:=$(shell find . -name "*.go" ! -path "./vendor/*" | xargs -i go tool vet {})
+vet: ERRORS:=$(shell find . -name "*.go" ! -path "./vendor/*" | xargs -i go tool vet {} | tr '`' '|')
 vet: fail-on-errors
 
 fail-on-errors:

auth/auth.go

@@ -8,11 +8,9 @@ import (
 	"github.com/ivanilves/lstags/auth/basic"
 	"github.com/ivanilves/lstags/auth/bearer"
 	"github.com/ivanilves/lstags/auth/none"
+	"github.com/ivanilves/lstags/docker"
 )
 
-// WebSchema defines how do we connect to remote web servers
-const WebSchema = "https://"
-
 // TokenResponse is an abstraction for aggregated token-related information we get from authentication services
 type TokenResponse interface {
 	Method() string
@@ -68,7 +66,7 @@ func validateParams(method string, params map[string]string) (map[string]string,
 // * detects authentication type (e.g. Bearer or Basic)
 // * delegates actual authentication to type-specific implementation
 func NewToken(registry, repository, username, password string) (TokenResponse, error) {
-	url := WebSchema + registry + "/v2"
+	url := docker.WebSchema(registry) + registry + "/v2"
 
 	resp, err := http.Get(url)
 	if err != nil {

docker/client/client.go

@@ -89,7 +89,7 @@ func (dc *DockerClient) Push(ref string) error {
 
 	pushOptions := types.ImagePushOptions{RegistryAuth: registryAuth}
 	if registryAuth == "" {
-		pushOptions = types.ImagePushOptions{}
+		pushOptions = types.ImagePushOptions{RegistryAuth: "IA=="}
 	}
 
 	resp, err := dc.cli.ImagePush(context.Background(), ref, pushOptions)

docker/docker.go

@@ -2,11 +2,16 @@ package docker
 
 import (
 	"strings"
+
+	"github.com/ivanilves/lstags/util"
 )
 
 // DefaultRegistry is a registry we use if none could be resolved from image ref
 var DefaultRegistry = "registry.hub.docker.com"
 
+// InsecureRegistryEx contains a regex string to match insecure registries
+var InsecureRegistryEx = "^(127\\..*|::1|localhost)(:[0-9]+)?$"
+
 // GetRegistry tries to get Docker registry name from a repository or reference
 // .. if it is not possible it returns default registry name (usually Docker Hub)
 func GetRegistry(repoOrRef string) string {
@@ -64,3 +69,12 @@ func GetRepoPath(repository, registry string) string {
 
 	return repository
 }
+
+// WebSchema tells us if we should use HTTP or HTTPS
+func WebSchema(registry string) string {
+	if util.DoesMatch(registry, InsecureRegistryEx) {
+		return "http://"
+	}
+
+	return "https://"
+}

docker/docker_test.go

@@ -95,3 +95,54 @@ func TestGetRepoPath(t *testing.T) {
 		}
 	}
 }
+
+func TestWebSchema(t *testing.T) {
+	examples := map[string]string{
+		"localhost":        "http://",
+		"localhost:4000":   "http://",
+		"127.0.0.1":        "http://",
+		"127.0.0.1:5000":   "http://",
+		"remotehost":       "https://",
+		"reg.hype.io":      "https://",
+		"reg.hype.io:3128": "https://",
+	}
+
+	for input, expected := range examples {
+		output := WebSchema(input)
+
+		if output != expected {
+			t.Fatalf(
+				"Got unexpected schema '%s' for registry hostname '%s', while expecting for '%s'",
+				output,
+				input,
+				expected,
+			)
+		}
+	}
+
+	InsecureRegistryEx = ".*"
+
+	for input := range examples {
+		output := WebSchema(input)
+
+		if output != "http://" {
+			t.Fatalf(
+				"Expected schema 'http://' for registry hostname '%s' in this case",
+				input,
+			)
+		}
+	}
+
+	InsecureRegistryEx = "i.do.not.match.anything"
+
+	for input := range examples {
+		output := WebSchema(input)
+
+		if output != "https://" {
+			t.Fatalf(
+				"Expected schema 'https://' for registry hostname '%s' in this case",
+				input,
+			)
+		}
+	}
+}

main.go

@@ -27,6 +27,7 @@ type Options struct {
 	PushPrefix         string `short:"R" long:"push-prefix" description:"[Re]Push pulled images with a specified repo path prefix" env:"PUSH_PREFIX"`
 	PushUpdate         bool   `short:"U" long:"push-update" description:"Update our pushed images if remote image digest changes" env:"PUSH_UPDATE"`
 	ConcurrentRequests int    `short:"c" long:"concurrent-requests" default:"32" description:"Limit of concurrent requests to the registry" env:"CONCURRENT_REQUESTS"`
+	InsecureRegistryEx string `short:"I" long:"insecure-registry-ex" description:"Expression to match insecure registry hostnames" env:"INSECURE_REGISTRY_EX"`
 	TraceRequests      bool   `short:"T" long:"trace-requests" description:"Trace Docker registry HTTP requests" env:"TRACE_REQUESTS"`
 	DoNotFail          bool   `short:"N" long:"do-not-fail" description:"Do not fail on non-critical errors (could be dangerous!)" env:"DO_NOT_FAIL"`
 	Version            bool   `short:"V" long:"version" description:"Show version and exit"`
@@ -40,7 +41,7 @@ var doNotFail = false
 func suicide(err error, critical bool) {
 	fmt.Printf("%s\n", err.Error())
 
-	if doNotFail || critical {
+	if !doNotFail || critical {
 		os.Exit(1)
 	}
 }
@@ -72,6 +73,10 @@ func parseFlags() (*Options, error) {
 		return nil, errors.New("You either '--pull' or '--push', not both")
 	}
 
+	if o.InsecureRegistryEx != "" {
+		docker.InsecureRegistryEx = o.InsecureRegistryEx
+	}
+
 	remote.TraceRequests = o.TraceRequests
 
 	doNotFail = o.DoNotFail

tag/remote/remote.go

@@ -14,13 +14,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ivanilves/lstags/docker"
 	"github.com/ivanilves/lstags/tag"
 	"github.com/ivanilves/lstags/util"
 )
 
-// WebSchema defines how do we connect to remote web servers
-const WebSchema = "https://"
-
 // TraceRequests defines if we should print out HTTP request URLs and response headers/bodies
 var TraceRequests = false
 
@@ -105,7 +103,7 @@ func parseTagNamesJSON(data io.ReadCloser) ([]string, error) {
 }
 
 func fetchTagNames(registry, repo, authorization string) ([]string, error) {
-	url := WebSchema + registry + "/v2/" + repo + "/tags/list"
+	url := docker.WebSchema(registry) + registry + "/v2/" + repo + "/tags/list"
 
 	resp, err := httpRequest(url, authorization, "v2")
 	if err != nil {
@@ -183,7 +181,7 @@ func fetchDigest(url, authorization string) (string, error) {
 }
 
 func fetchDetails(registry, repo, tagName, authorization string) (string, imageMetadata, error) {
-	url := WebSchema + registry + "/v2/" + repo + "/manifests/" + tagName
+	url := docker.WebSchema(registry) + registry + "/v2/" + repo + "/manifests/" + tagName
 
 	dc := make(chan string, 0)
 	mc := make(chan imageMetadata, 0)