Merge pull request #142 from ivanilves/auth-must-respect-concurrency

CHORE: Auth must respect `--concurrent-requests`
ivanilves · Apr 21, 2018 · ee02329 · ee02329
2 parents 57dcc84 + 70d392c
commit ee02329
Show file tree

Hide file tree

Showing 2 changed files with 108 additions and 33 deletions.
diff --git a/api/v1/v1.go b/api/v1/v1.go
@@ -58,63 +58,101 @@ func fn(labels ...string) string {
 	return fmt.Sprintf("[%s():%s]", shortname, strings.Join(labels, ":"))
 }
 
+func getBatchedSlices(batchSize int, unbatched ...string) [][]string {
+	batchedSlices := make([][]string, 0)
+
+	index := 0
+
+	for range unbatched {
+		batchedSlice := make([]string, 0)
+
+		for c := 0; c < batchSize; c++ {
+			batchedSlice = append(batchedSlice, unbatched[index])
+
+			index++
+
+			if index == len(unbatched) {
+				break
+			}
+		}
+
+		batchedSlices = append(batchedSlices, batchedSlice)
+
+		if index == len(unbatched) {
+			break
+		}
+	}
+
+	return batchedSlices
+}
+
 // CollectTags collects information on tags present in remote registry and [local] Docker daemon,
 // makes required comparisons between them and spits organized info back as collection.Collection
 func (api *API) CollectTags(refs ...string) (*collection.Collection, error) {
 	if len(refs) == 0 {
 		return nil, fmt.Errorf("no image references passed")
 	}
 
-	log.Debugf("%s references: %+v", fn(), refs)
-
-	repos, err := repository.ParseRefs(refs)
+	_, err := repository.ParseRefs(refs)
 	if err != nil {
 		return nil, err
 	}
-	for _, repo := range repos {
-		log.Debugf("%s repository: %+v", fn(), repo)
-	}
 
-	done := make(chan error, len(repos))
 	tags := make(map[string][]*tag.Tag)
 
-	for _, repo := range repos {
-		go func(repo *repository.Repository, done chan error) {
-			log.Infof("ANALYZE %s", repo.Ref())
+	batchedSlicesOfRefs := getBatchedSlices(api.config.ConcurrentRequests, refs...)
 
-			username, password, _ := api.dockerClient.Config().GetCredentials(repo.Registry())
+	for bindex, brefs := range batchedSlicesOfRefs {
+		log.Infof("BATCH %d of %d", bindex+1, len(batchedSlicesOfRefs))
 
-			remoteTags, err := remote.FetchTags(repo, username, password)
-			if err != nil {
-				done <- err
-				return
-			}
-			log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)
+		log.Debugf("%s references: %+v", fn(), brefs)
+
+		repos, _ := repository.ParseRefs(brefs)
+		for _, repo := range repos {
+			log.Debugf("%s repository: %+v", fn(), repo)
+		}
 
-			localTags, _ := local.FetchTags(repo, api.dockerClient)
+		done := make(chan error, len(repos))
 
-			log.Debugf("%s local tags: %+v", fn(repo.Ref()), localTags)
+		for _, repo := range repos {
+			go func(repo *repository.Repository, done chan error) {
+				log.Infof("ANALYZE %s", repo.Ref())
 
-			sortedKeys, tagNames, joinedTags := tag.Join(
-				remoteTags,
-				localTags,
-				repo.Tags(),
-			)
-			log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags)
+				username, password, _ := api.dockerClient.Config().GetCredentials(repo.Registry())
 
-			tags[repo.Ref()] = tag.Collect(sortedKeys, tagNames, joinedTags)
+				remoteTags, err := remote.FetchTags(repo, username, password)
+				if err != nil {
+					done <- err
+					return
+				}
+				log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)
 
-			done <- nil
+				localTags, _ := local.FetchTags(repo, api.dockerClient)
 
-			log.Infof("FETCHED %s", repo.Ref())
+				log.Debugf("%s local tags: %+v", fn(repo.Ref()), localTags)
 
-			return
-		}(repo, done)
-	}
+				sortedKeys, tagNames, joinedTags := tag.Join(
+					remoteTags,
+					localTags,
+					repo.Tags(),
+				)
+				log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags)
 
-	if err := wait.Until(done); err != nil {
-		return nil, err
+				tags[repo.Ref()] = tag.Collect(sortedKeys, tagNames, joinedTags)
+
+				done <- nil
+
+				log.Infof("FETCHED %s", repo.Ref())
+
+				return
+			}(repo, done)
+		}
+
+		if err := wait.Until(done); err != nil {
+			return nil, err
+		}
 	}
+
 	log.Debugf("%s tags: %+v", fn(), tags)
 
 	return collection.New(refs, tags)

diff --git a/api/v1/v1_test.go b/api/v1/v1_test.go
@@ -181,3 +181,40 @@ func TestGetPushPrefix(t *testing.T) {
 		assert.Equal(expected, actual)
 	}
 }
+
+func TestGetBatchedSlices(t *testing.T) {
+	var unbatched = []string{
+		"unbatched/repo01",
+		"unbatched/repo02",
+		"unbatched/repo03",
+		"unbatched/repo04",
+		"unbatched/repo05",
+		"unbatched/repo06",
+		"unbatched/repo07",
+		"unbatched/repo08",
+		"unbatched/repo09",
+		"unbatched/repo10",
+	}
+
+	var testCases = map[int][][]string{
+		1:   [][]string{{"unbatched/repo01"}, {"unbatched/repo02"}, {"unbatched/repo03"}, {"unbatched/repo04"}, {"unbatched/repo05"}, {"unbatched/repo06"}, {"unbatched/repo07"}, {"unbatched/repo08"}, {"unbatched/repo09"}, {"unbatched/repo10"}},
+		3:   [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03"}, {"unbatched/repo04", "unbatched/repo05", "unbatched/repo06"}, {"unbatched/repo07", "unbatched/repo08", "unbatched/repo09"}, {"unbatched/repo10"}},
+		7:   [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03", "unbatched/repo04", "unbatched/repo05", "unbatched/repo06", "unbatched/repo07"}, {"unbatched/repo08", "unbatched/repo09", "unbatched/repo10"}},
+		10:  [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03", "unbatched/repo04", "unbatched/repo05", "unbatched/repo06", "unbatched/repo07", "unbatched/repo08", "unbatched/repo09", "unbatched/repo10"}},
+		11:  [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03", "unbatched/repo04", "unbatched/repo05", "unbatched/repo06", "unbatched/repo07", "unbatched/repo08", "unbatched/repo09", "unbatched/repo10"}},
+		100: [][]string{{"unbatched/repo01", "unbatched/repo02", "unbatched/repo03", "unbatched/repo04", "unbatched/repo05", "unbatched/repo06", "unbatched/repo07", "unbatched/repo08", "unbatched/repo09", "unbatched/repo10"}},
+	}
+
+	var assert = assert.New(t)
+
+	for batchSize, expectedBatchedSlices := range testCases {
+		actualBatchedSlices := getBatchedSlices(batchSize, unbatched...)
+
+		assert.Equalf(
+			expectedBatchedSlices,
+			actualBatchedSlices,
+			"unexpected result for batch size: %d",
+			batchSize,
+		)
+	}
+}