wip zot catalog pagination

Signed-off-by: Eusebiu Petu <[email protected]>

pkg/api/routes.go

@@ -47,6 +47,7 @@ import (
 	mTypes "zotregistry.dev/zot/pkg/meta/types"
 	zreg "zotregistry.dev/zot/pkg/regexp"
 	reqCtx "zotregistry.dev/zot/pkg/requestcontext"
+	"zotregistry.dev/zot/pkg/storage"
 	storageCommon "zotregistry.dev/zot/pkg/storage/common"
 	storageTypes "zotregistry.dev/zot/pkg/storage/types"
 	"zotregistry.dev/zot/pkg/test/inject"
@@ -1776,12 +1777,116 @@ func (rh *RouteHandler) ListRepositories(response http.ResponseWriter, request *
 		return
 	}
 
+	q := request.URL.Query()
+
+	lastEntry := q.Get("last")
+	maxEntries, err := strconv.Atoi(q.Get("n"))
+	if err != nil {
+		maxEntries = -1
+	}
+
+	if lastEntry != "" || maxEntries > 0 {
+		storePath := rh.c.StoreController.GetStorePath(lastEntry)
+
+		combineRepoList := make([]string, 0)
+
+		var moreEntries bool = false
+
+		var remainder int
+
+		singleStore := rh.c.StoreController.DefaultStore
+		if singleStore != nil && storePath == storage.DefaultStorePath { // route is default
+			var err error
+
+			var repos []string
+
+			repos, moreEntries, err = singleStore.GetNextRepositories(lastEntry, maxEntries)
+			if err != nil {
+				response.WriteHeader(http.StatusInternalServerError)
+
+				return
+			}
+
+			// compute remainder
+			remainder = maxEntries - len(repos)
+
+			if moreEntries {
+				// maxEntries has been hit
+				lastEntry = repos[len(repos)-1]
+			} else {
+				// reset for the next substores
+				lastEntry = ""
+			}
+
+			combineRepoList = append(combineRepoList, repos...)
+		}
+
+		subStore := rh.c.StoreController.SubStore
+		for subPath, imgStore := range subStore {
+			if subPath != storePath || remainder <= 0 {
+				continue
+			}
+
+			var err error
+
+			var repos []string
+
+			repos, moreEntries, err = imgStore.GetNextRepositories(lastEntry, remainder)
+			if err != nil {
+				response.WriteHeader(http.StatusInternalServerError)
+
+				return
+			}
+
+			// compute remainder
+			remainder = maxEntries - len(repos)
+
+			if moreEntries {
+				// maxEntries has been hit
+				lastEntry = repos[len(repos)-1]
+			} else {
+				// reset for the next substores
+				lastEntry = ""
+			}
+
+			combineRepoList = append(combineRepoList, repos...)
+		}
+
+		repos := make([]string, 0)
+		// authz context
+		userAc, err := reqCtx.UserAcFromContext(request.Context())
+		if err != nil {
+			response.WriteHeader(http.StatusInternalServerError)
+
+			return
+		}
+
+		if userAc != nil {
+			for _, r := range combineRepoList {
+				if userAc.Can(constants.ReadPermission, r) {
+					repos = append(repos, r)
+				}
+			}
+		} else {
+			repos = combineRepoList
+		}
+
+		is := RepositoryList{Repositories: repos}
+
+		zcommon.WriteJSON(response, http.StatusOK, is)
+
+		return
+	}
+
 	combineRepoList := make([]string, 0)
 
-	subStore := rh.c.StoreController.SubStore
+	singleStore := rh.c.StoreController.DefaultStore
+	if singleStore != nil { // route is default
+		var err error
 
-	for _, imgStore := range subStore {
-		repos, err := imgStore.GetRepositories()
+		var repos []string
+
+		repos, err = singleStore.GetRepositories()
 		if err != nil {
 			response.WriteHeader(http.StatusInternalServerError)
 
@@ -1791,16 +1896,17 @@ func (rh *RouteHandler) ListRepositories(response http.ResponseWriter, request *
 		combineRepoList = append(combineRepoList, repos...)
 	}
 
-	singleStore := rh.c.StoreController.DefaultStore
-	if singleStore != nil {
-		repos, err := singleStore.GetRepositories()
+	subStore := rh.c.StoreController.SubStore
+	for _, imgStore := range subStore {
+		repos, err := imgStore.GetRepositories()
 		if err != nil {
 			response.WriteHeader(http.StatusInternalServerError)
 
 			return
 		}
 
 		combineRepoList = append(combineRepoList, repos...)
+
 	}
 
 	repos := make([]string, 0)

pkg/storage/imagestore/imagestore.go

@@ -265,6 +265,75 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) {
 	return true, nil
 }
 
+func (is *ImageStore) GetNextRepositories(lastRepo string, maxEntries int) ([]string, bool, error) {
+	var lockLatency time.Time
+
+	dir := is.rootDir
+
+	is.RLock(&lockLatency)
+	defer is.RUnlock(&lockLatency)
+
+	stores := make([]string, 0)
+
+	moreEntries := false
+	entries := 0
+	found := false
+	err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error {
+		if entries == maxEntries {
+			moreEntries = true
+
+			return io.EOF
+		}
+
+		if !fileInfo.IsDir() {
+			return nil
+		}
+
+		// skip .sync and .uploads dirs no need to try to validate them
+		if strings.HasSuffix(fileInfo.Path(), syncConstants.SyncBlobUploadDir) ||
+			strings.HasSuffix(fileInfo.Path(), ispec.ImageBlobsDir) ||
+			strings.HasSuffix(fileInfo.Path(), storageConstants.BlobUploadDir) {
+			return driver.ErrSkipDir
+		}
+
+		rel, err := filepath.Rel(is.rootDir, fileInfo.Path())
+		if err != nil {
+			return nil //nolint:nilerr // ignore paths that are not under root dir
+		}
+
+		if ok, err := is.ValidateRepo(rel); !ok || err != nil {
+			return nil //nolint:nilerr // ignore invalid repos
+		}
+
+		if lastRepo == "" || lastRepo == rel {
+			found = true
+		}
+
+		if found {
+			entries++
+
+			stores = append(stores, rel)
+		}
+
+		return nil
+	})
+
+	// if the root directory is not yet created then return an empty slice of repositories
+	driverErr := &driver.Error{}
+	if errors.As(err, &driver.PathNotFoundError{}) {
+		is.log.Debug().Msg("empty rootDir")
+
+		return stores, moreEntries, nil
+	}
+
+	if errors.Is(err, io.EOF) ||
+		(errors.As(err, driverErr) && errors.Is(driverErr.Detail, io.EOF)) {
+		return stores, moreEntries, nil
+	}
+
+	return stores, moreEntries, err
+}
+
 // GetRepositories returns a list of all the repositories under this store.
 func (is *ImageStore) GetRepositories() ([]string, error) {
 	var lockLatency time.Time

pkg/storage/storage_controller.go

@@ -7,8 +7,9 @@ import (
 )
 
 const (
-	CosignType   = "cosign"
-	NotationType = "notation"
+	CosignType       = "cosign"
+	NotationType     = "notation"
+	DefaultStorePath = "/"
 )
 
 type StoreController struct {
@@ -29,6 +30,21 @@ func GetRoutePrefix(name string) string {
 	return "/" + names[0]
 }
 
+func (sc StoreController) GetStorePath(name string) string {
+	if sc.SubStore != nil && name != "" {
+		subStorePath := GetRoutePrefix(name)
+
+		_, ok := sc.SubStore[subStorePath]
+		if !ok {
+			return DefaultStorePath
+		}
+
+		return subStorePath
+	}
+
+	return DefaultStorePath
+}
+
 func (sc StoreController) GetImageStore(name string) storageTypes.ImageStore {
 	if sc.SubStore != nil {
 		// SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name

pkg/storage/types/types.go

@@ -30,6 +30,7 @@ type ImageStore interface { //nolint:interfacebloat
 	ValidateRepo(name string) (bool, error)
 	GetRepositories() ([]string, error)
 	GetNextRepository(repo string) (string, error)
+	GetNextRepositories(repo string, maxEntries int) ([]string, bool, error)
 	GetImageTags(repo string) ([]string, error)
 	GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error)
 	PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error)

pkg/test/mocks/image_store_mock.go

@@ -12,16 +12,17 @@ import (
 )
 
 type MockedImageStore struct {
-	NameFn              func() string
-	DirExistsFn         func(d string) bool
-	RootDirFn           func() string
-	InitRepoFn          func(name string) error
-	ValidateRepoFn      func(name string) (bool, error)
-	GetRepositoriesFn   func() ([]string, error)
-	GetNextRepositoryFn func(repo string) (string, error)
-	GetImageTagsFn      func(repo string) ([]string, error)
-	GetImageManifestFn  func(repo string, reference string) ([]byte, godigest.Digest, string, error)
-	PutImageManifestFn  func(repo string, reference string, mediaType string, body []byte) (godigest.Digest,
+	NameFn                func() string
+	DirExistsFn           func(d string) bool
+	RootDirFn             func() string
+	InitRepoFn            func(name string) error
+	ValidateRepoFn        func(name string) (bool, error)
+	GetRepositoriesFn     func() ([]string, error)
+	GetNextRepositoryFn   func(repo string) (string, error)
+	GetNextRepositoriesFn func(lastRepo string, maxEntries int) ([]string, bool, error)
+	GetImageTagsFn        func(repo string) ([]string, error)
+	GetImageManifestFn    func(repo string, reference string) ([]byte, godigest.Digest, string, error)
+	PutImageManifestFn    func(repo string, reference string, mediaType string, body []byte) (godigest.Digest,
 		godigest.Digest, error)
 	DeleteImageManifestFn  func(repo string, reference string, detectCollision bool) error
 	BlobUploadPathFn       func(repo string, uuid string) string
@@ -138,6 +139,14 @@ func (is MockedImageStore) GetNextRepository(repo string) (string, error) {
 	return "", nil
 }
 
+func (is MockedImageStore) GetNextRepositories(lastRepo string, maxEntries int) ([]string, bool, error) {
+	if is.GetNextRepositoriesFn != nil {
+		return is.GetNextRepositoriesFn(lastRepo, maxEntries)
+	}
+
+	return []string{}, false, nil
+}
+
 func (is MockedImageStore) GetImageManifest(repo string, reference string) ([]byte, godigest.Digest, string, error) {
 	if is.GetImageManifestFn != nil {
 		return is.GetImageManifestFn(repo, reference)