Merge pull request #86 from ivanilves/just-another-refactor

Just another refactoring for greater good!

Makefile

@@ -1,5 +1,7 @@
 all: prepare dep test lint vet build
 
+offline: unit-test lint vet build
+
 prepare:
 	go get -u -v \
 		github.com/golang/dep/cmd/dep \

main.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/jessevdk/go-flags"
 
-	"github.com/ivanilves/lstags/auth"
 	"github.com/ivanilves/lstags/docker"
 	dockerclient "github.com/ivanilves/lstags/docker/client"
 	dockerconfig "github.com/ivanilves/lstags/docker/config"
@@ -73,6 +72,8 @@ func parseFlags() (*Options, error) {
 		return nil, errors.New("You either '--pull' or '--push', not both")
 	}
 
+	remote.ConcurrentRequests = o.ConcurrentRequests
+
 	if o.InsecureRegistryEx != "" {
 		docker.InsecureRegistryEx = o.InsecureRegistryEx
 	}
@@ -127,17 +128,12 @@ func main() {
 
 			username, password, _ := dockerConfig.GetCredentials(registry)
 
-			tr, err := auth.NewToken(registry, repoPath, username, password)
-			if err != nil {
-				suicide(err, true)
-			}
-
-			remoteTags, err := remote.FetchTags(registry, repoPath, tr.AuthHeader(), o.ConcurrentRequests, filter)
+			remoteTags, err := remote.FetchTags(registry, repoPath, filter, username, password)
 			if err != nil {
 				suicide(err, true)
 			}
 
-			localTags, err := local.FetchTags(repoName, dc)
+			localTags, err := local.FetchTags(repoName, filter, dc)
 			if err != nil {
 				suicide(err, true)
 			}
@@ -151,10 +147,6 @@ func main() {
 
 				tg := joinedTags[name]
 
-				if !util.DoesMatch(tg.GetName(), filter) {
-					continue
-				}
-
 				if tg.NeedsPull() {
 					pullTags = append(pullTags, tg)
 					pullCount++
@@ -179,12 +171,7 @@ func main() {
 
 				username, password, _ := dockerConfig.GetCredentials(o.PushRegistry)
 
-				tr, err := auth.NewToken(o.PushRegistry, pushRepoPath, username, password)
-				if err != nil {
-					suicide(err, true)
-				}
-
-				alreadyPushedTags, err := remote.FetchTags(o.PushRegistry, pushRepoPath, tr.AuthHeader(), o.ConcurrentRequests, filter)
+				alreadyPushedTags, err := remote.FetchTags(o.PushRegistry, pushRepoPath, username, password, filter)
 				if err != nil {
 					if !strings.Contains(err.Error(), "404 Not Found") {
 						suicide(err, true)
@@ -199,10 +186,6 @@ func main() {
 
 					tg := joinedTags[name]
 
-					if !util.DoesMatch(tg.GetName(), filter) {
-						continue
-					}
-
 					if tg.NeedsPush(o.PushUpdate) {
 						pushTags = append(pushTags, tg)
 						pushCount++

main_test.go

@@ -13,7 +13,6 @@ import (
 
 	"os"
 
-	"github.com/ivanilves/lstags/auth"
 	"github.com/ivanilves/lstags/tag/remote"
 )
 
@@ -24,12 +23,7 @@ const dockerJSON = "./fixtures/docker/config.json"
 func TestDockerHubWithPublicRepo(t *testing.T) {
 	const repo = "library/alpine"
 
-	tr, err := auth.NewToken(dockerHub, repo, "", "")
-	if err != nil {
-		t.Fatalf("Failed to get DockerHub public repo token: %s", err.Error())
-	}
-
-	tags, err := remote.FetchTags(dockerHub, repo, tr.AuthHeader(), 128, ".*")
+	tags, err := remote.FetchTags(dockerHub, repo, ".*", "", "")
 	if err != nil {
 		t.Fatalf("Failed to list DockerHub public repo (%s) tags: %s", repo, err.Error())
 	}
@@ -55,12 +49,7 @@ func TestDockerHubWithPrivateRepo(t *testing.T) {
 	pass := os.Getenv("DOCKERHUB_PASSWORD")
 	repo := os.Getenv("DOCKERHUB_PRIVATE_REPO")
 
-	tr, err := auth.NewToken(dockerHub, repo, user, pass)
-	if err != nil {
-		t.Fatalf("Failed to get DockerHub private repo token: %s", err.Error())
-	}
-
-	tags, err := remote.FetchTags(dockerHub, repo, tr.AuthHeader(), 128, ".*")
+	tags, err := remote.FetchTags(dockerHub, repo, ".*", user, pass)
 	if err != nil {
 		t.Fatalf("Failed to list DockerHub private repo (%s) tags: %s", repo, err.Error())
 	}

tag/local/local.go

@@ -5,10 +5,11 @@ import (
 
 	dockerclient "github.com/ivanilves/lstags/docker/client"
 	"github.com/ivanilves/lstags/tag"
+	"github.com/ivanilves/lstags/util"
 )
 
 // FetchTags looks up Docker repo tags and IDs present on local Docker daemon
-func FetchTags(repoName string, dc *dockerclient.DockerClient) (map[string]*tag.Tag, error) {
+func FetchTags(repoName, filter string, dc *dockerclient.DockerClient) (map[string]*tag.Tag, error) {
 	imageSummaries, err := dc.ListImagesForRepo(repoName)
 	if err != nil {
 		return nil, err
@@ -25,6 +26,10 @@ func FetchTags(repoName string, dc *dockerclient.DockerClient) (map[string]*tag.
 		}
 
 		for _, tagName := range tagNames {
+			if !util.DoesMatch(tagName, filter) {
+				continue
+			}
+
 			tg, err := tag.New(tagName, repoDigest)
 			if err != nil {
 				return nil, err

auth/auth.go → tag/remote/auth/auth.go

@@ -5,10 +5,10 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/ivanilves/lstags/auth/basic"
-	"github.com/ivanilves/lstags/auth/bearer"
-	"github.com/ivanilves/lstags/auth/none"
 	"github.com/ivanilves/lstags/docker"
+	"github.com/ivanilves/lstags/tag/remote/auth/basic"
+	"github.com/ivanilves/lstags/tag/remote/auth/bearer"
+	"github.com/ivanilves/lstags/tag/remote/auth/none"
 )
 
 // TokenResponse is an abstraction for aggregated token-related information we get from authentication services
@@ -65,7 +65,7 @@ func validateParams(method string, params map[string]string) (map[string]string,
 // NewToken is a high-level function which:
 // * detects authentication type (e.g. Bearer or Basic)
 // * delegates actual authentication to type-specific implementation
-func NewToken(registry, repository, username, password string) (TokenResponse, error) {
+func NewToken(registry, repoPath, username, password string) (TokenResponse, error) {
 	url := docker.WebSchema(registry) + registry + "/v2"
 
 	resp, err := http.Get(url)
@@ -88,7 +88,7 @@ func NewToken(registry, repository, username, password string) (TokenResponse, e
 	case "Basic":
 		return basic.RequestToken(url, username, password)
 	case "Bearer":
-		return bearer.RequestToken(params["realm"], params["service"], repository, username, password)
+		return bearer.RequestToken(params["realm"], params["service"], repoPath, username, password)
 	default:
 		return nil, errors.New("Unknown authentication method: " + method)
 	}

auth/bearer/bearer.go → tag/remote/auth/bearer/bearer.go

@@ -45,8 +45,8 @@ func decodeTokenResponse(data io.ReadCloser) (*TokenResponse, error) {
 }
 
 // RequestToken requests Bearer token from authentication service
-func RequestToken(realm, service, repository, username, password string) (*TokenResponse, error) {
-	url := realm + "?service=" + service + "&scope=repository:" + repository + ":pull"
+func RequestToken(realm, service, repoPath, username, password string) (*TokenResponse, error) {
+	url := realm + "?service=" + service + "&scope=repository:" + repoPath + ":pull"
 
 	hc := &http.Client{}
 	req, err := http.NewRequest("GET", url, nil)

tag/remote/remote.go

@@ -16,9 +16,13 @@ import (
 
 	"github.com/ivanilves/lstags/docker"
 	"github.com/ivanilves/lstags/tag"
+	"github.com/ivanilves/lstags/tag/remote/auth"
 	"github.com/ivanilves/lstags/util"
 )
 
+// ConcurrentRequests defines maximum number of concurrent requests we could maintain against the registry
+var ConcurrentRequests = 32
+
 // TraceRequests defines if we should print out HTTP request URLs and response headers/bodies
 var TraceRequests = false
 
@@ -102,8 +106,8 @@ func parseTagNamesJSON(data io.ReadCloser) ([]string, error) {
 	return tn.TagNames, nil
 }
 
-func fetchTagNames(registry, repo, authorization string) ([]string, error) {
-	url := docker.WebSchema(registry) + registry + "/v2/" + repo + "/tags/list"
+func fetchTagNames(registry, repoPath, authorization string) ([]string, error) {
+	url := docker.WebSchema(registry) + registry + "/v2/" + repoPath + "/tags/list"
 
 	resp, err := httpRequest(url, authorization, "v2")
 	if err != nil {
@@ -180,8 +184,8 @@ func fetchDigest(url, authorization string) (string, error) {
 	return digests[0], nil
 }
 
-func fetchDetails(registry, repo, tagName, authorization string) (string, imageMetadata, error) {
-	url := docker.WebSchema(registry) + registry + "/v2/" + repo + "/manifests/" + tagName
+func fetchDetails(registry, repoPath, tagName, authorization string) (string, imageMetadata, error) {
+	url := docker.WebSchema(registry) + registry + "/v2/" + repoPath + "/manifests/" + tagName
 
 	dc := make(chan string, 0)
 	mc := make(chan imageMetadata, 0)
@@ -234,19 +238,19 @@ type detailResponse struct {
 	Error       error
 }
 
-func validateConcurrentRequests(concurrentRequests int) (int, error) {
+func validateConcurrentRequests() (int, error) {
 	const min = 1
 	const max = 128
 
-	if concurrentRequests < min {
+	if ConcurrentRequests < min {
 		return 0, errors.New("Concurrent requests limit could not be lower than " + strconv.Itoa(min))
 	}
 
-	if concurrentRequests > max {
+	if ConcurrentRequests > max {
 		return 0, errors.New("Concurrent requests limit could not be higher than " + strconv.Itoa(max))
 	}
 
-	return concurrentRequests, nil
+	return ConcurrentRequests, nil
 }
 
 func calculateBatchSteps(count, limit int) (int, int) {
@@ -268,14 +272,21 @@ func calculateBatchStepSize(stepNumber, stepsTotal, remain, limit int) int {
 	return limit
 }
 
-// FetchTags looks up Docker repo tags present on remote Docker registry
-func FetchTags(registry, repo, authorization string, concurrentRequests int, filter string) (map[string]*tag.Tag, error) {
-	batchLimit, err := validateConcurrentRequests(concurrentRequests)
+// FetchTags looks up Docker repoPath tags present on remote Docker registry
+func FetchTags(registry, repoPath, filter, username, password string) (map[string]*tag.Tag, error) {
+	batchLimit, err := validateConcurrentRequests()
+	if err != nil {
+		return nil, err
+	}
+
+	tr, err := auth.NewToken(registry, repoPath, username, password)
 	if err != nil {
 		return nil, err
 	}
 
-	allTagNames, err := fetchTagNames(registry, repo, authorization)
+	authorization := tr.AuthHeader()
+
+	allTagNames, err := fetchTagNames(registry, repoPath, authorization)
 	if err != nil {
 		return nil, err
 	}
@@ -299,11 +310,11 @@ func FetchTags(registry, repo, authorization string, concurrentRequests int, fil
 		ch := make(chan detailResponse, stepSize)
 
 		for s := 1; s <= stepSize; s++ {
-			go func(registry, repo, tagName, authorization string, ch chan detailResponse) {
-				digest, metadata, err := fetchDetails(registry, repo, tagName, authorization)
+			go func(registry, repoPath, tagName, authorization string, ch chan detailResponse) {
+				digest, metadata, err := fetchDetails(registry, repoPath, tagName, authorization)
 
 				ch <- detailResponse{TagName: tagName, Digest: digest, Created: metadata.Created, ContainerID: metadata.ContainerID, Error: err}
-			}(registry, repo, tagNames[tagIndex], authorization, ch)
+			}(registry, repoPath, tagNames[tagIndex], authorization, ch)
 
 			tagIndex++
 		}