diff --git a/pkg/cli/search_cmd_referrers_test.go b/pkg/cli/search_cmd_referrers_test.go index aa3cdcc634..26f9c67465 100644 --- a/pkg/cli/search_cmd_referrers_test.go +++ b/pkg/cli/search_cmd_referrers_test.go @@ -21,9 +21,9 @@ import ( ) func ref[T any](input T) *T { - obj := input + ref := input - return &obj + return &ref } const ( diff --git a/pkg/common/common.go b/pkg/common/common.go index 5989d7a78e..a6f67ffea8 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -6,6 +6,7 @@ import ( "fmt" "io/fs" "os" + "strings" "syscall" "time" "unicode/utf8" @@ -101,3 +102,13 @@ func MarshalThroughStruct(obj interface{}, throughStruct interface{}) ([]byte, e return toJSON, nil } + +func ContainsString(strSlice []string, str string) bool { + for _, val := range strSlice { + if strings.EqualFold(val, str) { + return true + } + } + + return false +} diff --git a/pkg/common/pagination/image_pagination.go b/pkg/common/pagination/image_pagination.go new file mode 100644 index 0000000000..f0f4a6ecc8 --- /dev/null +++ b/pkg/common/pagination/image_pagination.go @@ -0,0 +1,164 @@ +package pagination + +import ( + "fmt" + "sort" + "time" + + zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" + gql_gen "zotregistry.io/zot/pkg/extensions/search/gql_generated" +) + +type ImageSummariesPageFinder struct { + limit int + offset int + sortBy SortCriteria + pageBuffer []*gql_gen.ImageSummary +} + +func NewImgSumPageFinder(limit, offset int, sortBy SortCriteria) (*ImageSummariesPageFinder, error) { + if sortBy == "" { + sortBy = AlphabeticAsc + } + + if limit < 0 { + return nil, zerr.ErrLimitIsNegative + } + + if offset < 0 { + return nil, zerr.ErrOffsetIsNegative + } + + if _, found := ImgSumSortFuncs()[sortBy]; !found { + return nil, fmt.Errorf("sorting repos by '%s' is not supported %w", + sortBy, zerr.ErrSortCriteriaNotSupported) + } + + return &ImageSummariesPageFinder{ + limit: limit, + offset: offset, + sortBy: sortBy, + pageBuffer: []*gql_gen.ImageSummary{}, + }, nil +} + +func (pf *ImageSummariesPageFinder) Add(imgSum *gql_gen.ImageSummary) { + pf.pageBuffer = append(pf.pageBuffer, imgSum) +} + +func (pf *ImageSummariesPageFinder) Page() ([]*gql_gen.ImageSummary, zcommon.PageInfo) { + if len(pf.pageBuffer) == 0 { + return []*gql_gen.ImageSummary{}, zcommon.PageInfo{} + } + + pageInfo := zcommon.PageInfo{} + + sort.Slice(pf.pageBuffer, ImgSumSortFuncs()[pf.sortBy](pf.pageBuffer)) + + // the offset and limit are calculated in terms of repos counted + start := pf.offset + end := pf.offset + pf.limit + + // we'll return an empty array when the offset is greater than the number of elements + if start >= len(pf.pageBuffer) { + start = len(pf.pageBuffer) + end = start + } + + if end >= len(pf.pageBuffer) { + end = len(pf.pageBuffer) + } + + page := pf.pageBuffer[start:end] + + pageInfo.ItemCount = len(page) + + if start == 0 && end == 0 { + page = pf.pageBuffer + pageInfo.ItemCount = len(page) + } + + pageInfo.TotalCount = len(pf.pageBuffer) + + return page, pageInfo +} + +func ImgSumSortFuncs() map[SortCriteria]func(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool { + return map[SortCriteria]func(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool{ + AlphabeticAsc: ImgSortByAlphabeticAsc, + AlphabeticDsc: ImgSortByAlphabeticDsc, + UpdateTime: ImgSortByUpdateTime, + Relevance: ImgSortByRelevance, + Downloads: ImgSortByDownloads, + } +} + +func ImgSortByAlphabeticAsc(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool { + return func(i, j int) bool { //nolint: varnamelen + if *pageBuffer[i].RepoName < *pageBuffer[j].RepoName { + return true + } + + if *pageBuffer[i].RepoName == *pageBuffer[j].RepoName { + return *pageBuffer[i].Tag < *pageBuffer[j].Tag + } + + return false + } +} + +func ImgSortByAlphabeticDsc(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool { + return func(i, j int) bool { //nolint: varnamelen + if *pageBuffer[i].RepoName > *pageBuffer[j].RepoName { + return true + } + + if *pageBuffer[i].RepoName == *pageBuffer[j].RepoName { + return *pageBuffer[i].Tag > *pageBuffer[j].Tag + } + + return false + } +} + +func ImgSortByRelevance(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool { + return func(i, j int) bool { //nolint: varnamelen + if *pageBuffer[i].RepoName < *pageBuffer[j].RepoName { + return true + } + + if *pageBuffer[i].RepoName == *pageBuffer[j].RepoName { + return *pageBuffer[i].Tag < *pageBuffer[j].Tag + } + + return false + } +} + +// SortByUpdateTime sorting descending by time. +func ImgSortByUpdateTime(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool { + repos2LastUpdated := map[string]time.Time{} + + for _, img := range pageBuffer { + lastUpdated, ok := repos2LastUpdated[*img.RepoName] + + if !ok || lastUpdated.Before(*img.LastUpdated) { + repos2LastUpdated[*img.RepoName] = *img.LastUpdated + } + } + + return func(i, j int) bool { + iRepoTime, jRepoTime := repos2LastUpdated[*pageBuffer[i].RepoName], repos2LastUpdated[*pageBuffer[j].RepoName] + + return (iRepoTime.After(jRepoTime) || iRepoTime.Equal(jRepoTime)) && + pageBuffer[i].LastUpdated.After(*pageBuffer[j].LastUpdated) + } +} + +// SortByDownloads returns a comparison function for descendant sorting by downloads. +func ImgSortByDownloads(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool { + return func(i, j int) bool { + return *pageBuffer[i].DownloadCount > *pageBuffer[j].DownloadCount + } +} diff --git a/pkg/common/pagination/model.go b/pkg/common/pagination/model.go new file mode 100644 index 0000000000..690c352584 --- /dev/null +++ b/pkg/common/pagination/model.go @@ -0,0 +1,18 @@ +package pagination + +type SortCriteria string + +type PageInput struct { + Limit int + Offset int + SortBy SortCriteria +} + +const ( + Relevance = SortCriteria("RELEVANCE") + UpdateTime = SortCriteria("UPDATE_TIME") + AlphabeticAsc = SortCriteria("ALPHABETIC_ASC") + AlphabeticDsc = SortCriteria("ALPHABETIC_DSC") + Stars = SortCriteria("STARS") + Downloads = SortCriteria("DOWNLOADS") +) diff --git a/pkg/common/pagination/pagination_test.go b/pkg/common/pagination/pagination_test.go new file mode 100644 index 0000000000..e1d64ba83c --- /dev/null +++ b/pkg/common/pagination/pagination_test.go @@ -0,0 +1,294 @@ +package pagination_test + +import ( + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.io/zot/pkg/common/pagination" + "zotregistry.io/zot/pkg/extensions/search/gql_generated" +) + +func TestImgSumPagination(t *testing.T) { + Convey("NewImgSumPageFinder errors", t, func() { + _, err := pagination.NewImgSumPageFinder(-1, 0, pagination.AlphabeticAsc) + So(err, ShouldNotBeNil) + + _, err = pagination.NewImgSumPageFinder(0, -1, pagination.AlphabeticAsc) + So(err, ShouldNotBeNil) + + _, err = pagination.NewImgSumPageFinder(0, 0, "unknown") + So(err, ShouldNotBeNil) + }) + + Convey("Sort Functions", t, func() { + Convey("ImgSortByAlphabeticAsc", func() { + // Case: repo1 is < repo2 + pageBuff := []*gql_generated.ImageSummary{ + {RepoName: ref("repo1:1")}, + {RepoName: ref("repo2:2")}, + } + + sortFunc := pagination.ImgSortByAlphabeticAsc(pageBuff) + So(sortFunc(0, 1), ShouldBeTrue) + }) + + Convey("ImgSortByAlphabeticDsc", func() { + // Case: repo1 is < repo2 + pageBuff := []*gql_generated.ImageSummary{ + {RepoName: ref("repo1:1")}, + {RepoName: ref("repo2:2")}, + } + + sortFunc := pagination.ImgSortByAlphabeticDsc(pageBuff) + So(sortFunc(0, 1), ShouldBeFalse) + }) + + Convey("ImgSortByRelevance", func() { + // Case: repo1 is < repo2 + pageBuff := []*gql_generated.ImageSummary{ + {RepoName: ref("repo1:1")}, + {RepoName: ref("repo2:2")}, + } + + sortFunc := pagination.ImgSortByRelevance(pageBuff) + So(sortFunc(0, 1), ShouldBeTrue) + }) + }) +} + +func TestRepoSumPagination(t *testing.T) { + Convey("NewRepoSumPageFinder errors", t, func() { + _, err := pagination.NewRepoSumPageFinder(-1, 0, pagination.AlphabeticAsc) + So(err, ShouldNotBeNil) + + _, err = pagination.NewRepoSumPageFinder(0, -1, pagination.AlphabeticAsc) + So(err, ShouldNotBeNil) + + _, err = pagination.NewRepoSumPageFinder(0, 0, "unknown") + So(err, ShouldNotBeNil) + }) +} + +func ref[T any](input T) *T { + obj := input + + return &obj +} + +func TestPagination(t *testing.T) { + Convey("Image Pagination", t, func() { + Convey("Sort functions", func() { + imgSum1 := gql_generated.ImageSummary{ + RepoName: ref("1"), + Tag: ref("1"), + LastUpdated: ref(time.Date(2010, 1, 1, 1, 1, 1, 1, time.UTC)), + DownloadCount: ref(33), + } + + imgSum2 := gql_generated.ImageSummary{ + RepoName: ref("1"), + Tag: ref("latest"), + LastUpdated: ref(time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC)), + DownloadCount: ref(11), + } + + imgSum3 := gql_generated.ImageSummary{ + RepoName: ref("3"), + Tag: ref("1"), + LastUpdated: ref(time.Date(2011, 1, 1, 1, 1, 1, 1, time.UTC)), + DownloadCount: ref(22), + } + + imgSum4 := gql_generated.ImageSummary{ + RepoName: ref("4"), + Tag: ref("latest"), + LastUpdated: ref(time.Date(2012, 1, 1, 1, 1, 1, 1, time.UTC)), + DownloadCount: ref(44), + } + + // ImgSortByAlphabeticAsc + imagePageFinder, err := pagination.NewImgSumPageFinder(4, 0, pagination.AlphabeticAsc) + So(err, ShouldBeNil) + imagePageFinder.Add(&imgSum1) + imagePageFinder.Add(&imgSum2) + imagePageFinder.Add(&imgSum3) + imagePageFinder.Add(&imgSum4) + page, _ := imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.ImageSummary{ + &imgSum1, &imgSum2, &imgSum3, &imgSum4, + }) + + // ImgSortByAlphabeticDsc + imagePageFinder, err = pagination.NewImgSumPageFinder(4, 0, pagination.AlphabeticDsc) + So(err, ShouldBeNil) + imagePageFinder.Add(&imgSum1) + imagePageFinder.Add(&imgSum2) + imagePageFinder.Add(&imgSum3) + imagePageFinder.Add(&imgSum4) + page, _ = imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.ImageSummary{ + &imgSum4, &imgSum3, &imgSum2, &imgSum1, + }) + + // ImgSortByRelevance + imagePageFinder, err = pagination.NewImgSumPageFinder(4, 0, pagination.Relevance) + So(err, ShouldBeNil) + imagePageFinder.Add(&imgSum1) + imagePageFinder.Add(&imgSum2) + imagePageFinder.Add(&imgSum3) + imagePageFinder.Add(&imgSum4) + page, _ = imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.ImageSummary{ + &imgSum1, &imgSum2, &imgSum3, &imgSum4, + }) + + // ImgSortByUpdateTime + imagePageFinder, err = pagination.NewImgSumPageFinder(4, 0, pagination.UpdateTime) + So(err, ShouldBeNil) + imagePageFinder.Add(&imgSum1) + imagePageFinder.Add(&imgSum2) + imagePageFinder.Add(&imgSum3) + imagePageFinder.Add(&imgSum4) + page, _ = imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.ImageSummary{ + &imgSum2, &imgSum1, &imgSum4, &imgSum3, + }) + + // ImgSortByDownloads + imagePageFinder, err = pagination.NewImgSumPageFinder(4, 0, pagination.Downloads) + So(err, ShouldBeNil) + imagePageFinder.Add(&imgSum1) + imagePageFinder.Add(&imgSum2) + imagePageFinder.Add(&imgSum3) + imagePageFinder.Add(&imgSum4) + page, _ = imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.ImageSummary{ + &imgSum4, &imgSum1, &imgSum3, &imgSum2, + }) + }) + + Convey("Errors", func() { + imagePageFinder, err := pagination.NewImgSumPageFinder(2, 0, "") + So(err, ShouldBeNil) + So(imagePageFinder, ShouldNotBeNil) + + _, err = pagination.NewImgSumPageFinder(-1, 0, "") + So(err, ShouldNotBeNil) + + _, err = pagination.NewImgSumPageFinder(1, -1, "") + So(err, ShouldNotBeNil) + + _, err = pagination.NewImgSumPageFinder(1, -1, "bad sort func") + So(err, ShouldNotBeNil) + }) + }) + + Convey("Repos Pagination", t, func() { + Convey("Sort functions", func() { + repoSum1 := gql_generated.RepoSummary{ + Name: ref("1"), + LastUpdated: ref(time.Date(2010, 1, 1, 1, 1, 1, 1, time.UTC)), + DownloadCount: ref(33), + Rank: ref(1), + } + + repoSum2 := gql_generated.RepoSummary{ + Name: ref("2"), + LastUpdated: ref(time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC)), + DownloadCount: ref(11), + Rank: ref(2), + } + + repoSum3 := gql_generated.RepoSummary{ + Name: ref("3"), + LastUpdated: ref(time.Date(2011, 1, 1, 1, 1, 1, 1, time.UTC)), + DownloadCount: ref(22), + Rank: ref(3), + } + + repoSum4 := gql_generated.RepoSummary{ + Name: ref("4"), + LastUpdated: ref(time.Date(2012, 1, 1, 1, 1, 1, 1, time.UTC)), + DownloadCount: ref(44), + Rank: ref(4), + } + + // ImgSortByAlphabeticAsc + imagePageFinder, err := pagination.NewRepoSumPageFinder(4, 0, pagination.AlphabeticAsc) + So(err, ShouldBeNil) + imagePageFinder.Add(&repoSum1) + imagePageFinder.Add(&repoSum2) + imagePageFinder.Add(&repoSum3) + imagePageFinder.Add(&repoSum4) + page, _ := imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.RepoSummary{ + &repoSum1, &repoSum2, &repoSum3, &repoSum4, + }) + + // ImgSortByAlphabeticDsc + imagePageFinder, err = pagination.NewRepoSumPageFinder(4, 0, pagination.AlphabeticDsc) + So(err, ShouldBeNil) + imagePageFinder.Add(&repoSum1) + imagePageFinder.Add(&repoSum2) + imagePageFinder.Add(&repoSum3) + imagePageFinder.Add(&repoSum4) + page, _ = imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.RepoSummary{ + &repoSum4, &repoSum3, &repoSum2, &repoSum1, + }) + + // ImgSortByRelevance + imagePageFinder, err = pagination.NewRepoSumPageFinder(4, 0, pagination.Relevance) + So(err, ShouldBeNil) + imagePageFinder.Add(&repoSum1) + imagePageFinder.Add(&repoSum2) + imagePageFinder.Add(&repoSum3) + imagePageFinder.Add(&repoSum4) + page, _ = imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.RepoSummary{ + &repoSum1, &repoSum2, &repoSum3, &repoSum4, + }) + + // ImgSortByUpdateTime + imagePageFinder, err = pagination.NewRepoSumPageFinder(4, 0, pagination.UpdateTime) + So(err, ShouldBeNil) + imagePageFinder.Add(&repoSum1) + imagePageFinder.Add(&repoSum2) + imagePageFinder.Add(&repoSum3) + imagePageFinder.Add(&repoSum4) + page, _ = imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.RepoSummary{ + &repoSum2, &repoSum4, &repoSum3, &repoSum1, + }) + + // ImgSortByDownloads + imagePageFinder, err = pagination.NewRepoSumPageFinder(4, 0, pagination.Downloads) + So(err, ShouldBeNil) + imagePageFinder.Add(&repoSum1) + imagePageFinder.Add(&repoSum2) + imagePageFinder.Add(&repoSum3) + imagePageFinder.Add(&repoSum4) + page, _ = imagePageFinder.Page() + So(page, ShouldEqual, []*gql_generated.RepoSummary{ + &repoSum4, &repoSum1, &repoSum3, &repoSum2, + }) + }) + + Convey("Errors", func() { + repoPageFinder, err := pagination.NewRepoSumPageFinder(2, 0, "") + So(err, ShouldBeNil) + So(repoPageFinder, ShouldNotBeNil) + + _, err = pagination.NewRepoSumPageFinder(-1, 0, "") + So(err, ShouldNotBeNil) + + _, err = pagination.NewRepoSumPageFinder(1, -1, "") + So(err, ShouldNotBeNil) + + _, err = pagination.NewRepoSumPageFinder(1, -1, "bad sort func") + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/pkg/common/pagination/repo_pagination.go b/pkg/common/pagination/repo_pagination.go new file mode 100644 index 0000000000..a25a8938b5 --- /dev/null +++ b/pkg/common/pagination/repo_pagination.go @@ -0,0 +1,126 @@ +package pagination + +import ( + "fmt" + "sort" + + zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" + gql_gen "zotregistry.io/zot/pkg/extensions/search/gql_generated" +) + +type RepoSummariesPageFinder struct { + limit int + offset int + sortBy SortCriteria + pageBuffer []*gql_gen.RepoSummary +} + +func NewRepoSumPageFinder(limit, offset int, sortBy SortCriteria) (*RepoSummariesPageFinder, error) { + if sortBy == "" { + sortBy = AlphabeticAsc + } + + if limit < 0 { + return nil, zerr.ErrLimitIsNegative + } + + if offset < 0 { + return nil, zerr.ErrOffsetIsNegative + } + + if _, found := RepoSumSortFuncs()[sortBy]; !found { + return nil, fmt.Errorf("sorting repos by '%s' is not supported %w", + sortBy, zerr.ErrSortCriteriaNotSupported) + } + + return &RepoSummariesPageFinder{ + limit: limit, + offset: offset, + sortBy: sortBy, + pageBuffer: []*gql_gen.RepoSummary{}, + }, nil +} + +func (pf *RepoSummariesPageFinder) Add(imgSum *gql_gen.RepoSummary) { + pf.pageBuffer = append(pf.pageBuffer, imgSum) +} + +func (pf *RepoSummariesPageFinder) Page() ([]*gql_gen.RepoSummary, zcommon.PageInfo) { + if len(pf.pageBuffer) == 0 { + return []*gql_gen.RepoSummary{}, zcommon.PageInfo{} + } + + pageInfo := zcommon.PageInfo{} + + sort.Slice(pf.pageBuffer, RepoSumSortFuncs()[pf.sortBy](pf.pageBuffer)) + + // the offset and limit are calculated in terms of repos counted + start := pf.offset + end := pf.offset + pf.limit + + // we'll return an empty array when the offset is greater than the number of elements + if start >= len(pf.pageBuffer) { + start = len(pf.pageBuffer) + end = start + } + + if end >= len(pf.pageBuffer) { + end = len(pf.pageBuffer) + } + + page := pf.pageBuffer[start:end] + + pageInfo.ItemCount = len(page) + + if start == 0 && end == 0 { + page = pf.pageBuffer + pageInfo.ItemCount = len(page) + } + + pageInfo.TotalCount = len(pf.pageBuffer) + + return page, pageInfo +} + +func RepoSumSortFuncs() map[SortCriteria]func(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool { + return map[SortCriteria]func(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool{ + AlphabeticAsc: RepoSortByAlphabeticAsc, + AlphabeticDsc: RepoSortByAlphabeticDsc, + Relevance: RepoSortByRelevance, + UpdateTime: RepoSortByUpdateTime, + Downloads: RepoSortByDownloads, + } +} + +func RepoSortByAlphabeticAsc(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool { + return func(i, j int) bool { + return *pageBuffer[i].Name < *pageBuffer[j].Name + } +} + +func RepoSortByAlphabeticDsc(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool { + return func(i, j int) bool { + return *pageBuffer[i].Name > *pageBuffer[j].Name + } +} + +func RepoSortByRelevance(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool { + return func(i, j int) bool { + return *pageBuffer[i].Rank < *pageBuffer[j].Rank + } +} + +// SortByUpdateTime sorting descending by time. +func RepoSortByUpdateTime(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool { + return func(i, j int) bool { + return pageBuffer[i].LastUpdated.After(*pageBuffer[j].LastUpdated) + } +} + +// SortByDownloads returns a comparison function for descendant sorting by downloads. +func RepoSortByDownloads(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool { + return func(i, j int) bool { + return *pageBuffer[i].DownloadCount > *pageBuffer[j].DownloadCount + } +} diff --git a/pkg/extensions/extension_mgmt.go b/pkg/extensions/extension_mgmt.go index 325ed074da..7d365643a8 100644 --- a/pkg/extensions/extension_mgmt.go +++ b/pkg/extensions/extension_mgmt.go @@ -250,7 +250,7 @@ func EnablePeriodicSignaturesVerification(config *config.Config, taskScheduler * repos, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true - }, mTypes.PageInput{}) + }) if err != nil { return } @@ -297,9 +297,7 @@ func (gen *taskGeneratorSigValidity) Reset() { gen.repoIndex = -1 ctx := context.Background() - repos, err := gen.metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { - return true - }, mTypes.PageInput{}) + repos, err := gen.metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }) if err != nil { return } diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index c75269016a..38091380ae 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -12,12 +12,14 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + "zotregistry.io/zot/pkg/common/pagination" "zotregistry.io/zot/pkg/extensions/search/convert" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/boltdb" mTypes "zotregistry.io/zot/pkg/meta/types" + "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" ) @@ -59,8 +61,7 @@ func TestConvertErrors(t *testing.T) { err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - repoMetas, manifestMetaMap, _, _, err := metaDB.SearchRepos(context.Background(), "", mTypes.Filter{}, - mTypes.PageInput{}) + repoMetas, manifestMetaMap, _, err := metaDB.SearchRepos(context.Background(), "") So(err, ShouldBeNil) ctx := graphql.WithResponseContext(context.Background(), @@ -414,3 +415,266 @@ func TestGetSignaturesInfo(t *testing.T) { So(*signaturesSummary[0].Tool, ShouldEqual, "notation") }) } + +func TestAcceptedByFilter(t *testing.T) { + Convey("Images", t, func() { + Convey("Os not found", func() { + found := convert.ImgSumAcceptedByFilter( + &gql_generated.ImageSummary{ + Manifests: []*gql_generated.ManifestSummary{ + {Platform: &gql_generated.Platform{Os: ref("os1")}}, + {Platform: &gql_generated.Platform{Os: ref("os2")}}, + }, + }, + mTypes.Filter{Os: []*string{ref("os3")}}, + ) + + So(found, ShouldBeFalse) + }) + + Convey("Has to be signed ", func() { + found := convert.ImgSumAcceptedByFilter( + &gql_generated.ImageSummary{ + Manifests: []*gql_generated.ManifestSummary{ + {IsSigned: ref(false)}, + }, + IsSigned: ref(false), + }, + mTypes.Filter{HasToBeSigned: ref(true)}, + ) + + So(found, ShouldBeFalse) + }) + }) + + Convey("Repos", t, func() { + Convey("Os not found", func() { + found := convert.RepoSumAcceptedByFilter( + &gql_generated.RepoSummary{ + Platforms: []*gql_generated.Platform{ + {Os: ref("os1")}, + {Os: ref("os2")}, + }, + }, + mTypes.Filter{Os: []*string{ref("os3")}}, + ) + + So(found, ShouldBeFalse) + }) + + Convey("Arch not found", func() { + found := convert.RepoSumAcceptedByFilter( + &gql_generated.RepoSummary{ + Platforms: []*gql_generated.Platform{ + {Arch: ref("Arch")}, + }, + }, + mTypes.Filter{Arch: []*string{ref("arch_not_found")}}, + ) + + So(found, ShouldBeFalse) + }) + + Convey("Has to be signed ", func() { + found := convert.ImgSumAcceptedByFilter( + &gql_generated.ImageSummary{ + Manifests: []*gql_generated.ManifestSummary{ + {IsSigned: ref(false)}, + }, + IsSigned: ref(false), + }, + mTypes.Filter{HasToBeSigned: ref(true)}, + ) + + So(found, ShouldBeFalse) + }) + }) +} + +func ref[T any](val T) *T { + ref := val + + return &ref +} + +func TestPaginatedConvert(t *testing.T) { + ctx := context.Background() + + Convey("PaginatedRepoMeta2Summaries filtering and sorting", t, func() { + var ( + badBothImage = test.CreateImageWith().DefaultLayers().ImageConfig( + ispec.Image{Platform: ispec.Platform{OS: "bad-os", Architecture: "bad-arch"}}).Build() + badOsImage = test.CreateImageWith().DefaultLayers().ImageConfig( + ispec.Image{Platform: ispec.Platform{OS: "bad-os", Architecture: "good-arch"}}).Build() + badArchImage = test.CreateImageWith().DefaultLayers().ImageConfig( + ispec.Image{Platform: ispec.Platform{OS: "good-os", Architecture: "bad-arch"}}).Build() + goodImage = test.CreateImageWith().DefaultLayers().ImageConfig( + ispec.Image{Platform: ispec.Platform{OS: "good-os", Architecture: "good-arch"}}).Build() + + randomImage1 = test.CreateRandomImage() + randomImage2 = test.CreateRandomImage() + + badMultiArch = test.CreateMultiarchWith().Images( + []test.Image{badBothImage, badOsImage, badArchImage, randomImage1}).Build() + goodMultiArch = test.CreateMultiarchWith().Images( + []test.Image{badOsImage, badArchImage, randomImage2, goodImage}).Build() + ) + + reposMeta, manifestMetaMap, indexDataMap := test.GetMetadataForRepos( + test.Repo{ + Name: "repo1-only-images", + Images: []test.RepoImage{ + {Image: goodImage, Tag: "goodImage"}, + {Image: badOsImage, Tag: "badOsImage"}, + {Image: badArchImage, Tag: "badArchImage"}, + {Image: badBothImage, Tag: "badBothImage"}, + }, + IsBookmarked: true, + IsStarred: true, + }, + test.Repo{ + Name: "repo2-only-bad-images", + Images: []test.RepoImage{ + {Image: randomImage1, Tag: "randomImage1"}, + {Image: randomImage2, Tag: "randomImage2"}, + {Image: badBothImage, Tag: "badBothImage"}, + }, + IsBookmarked: true, + IsStarred: true, + }, + test.Repo{ + Name: "repo3-only-multiarch", + MultiArchImages: []test.RepoMultiArchImage{ + {MultiarchImage: badMultiArch, Tag: "badMultiArch"}, + {MultiarchImage: goodMultiArch, Tag: "goodMultiArch"}, + }, + IsBookmarked: true, + IsStarred: true, + }, + test.Repo{ + Name: "repo4-not-bookmarked-or-starred", + Images: []test.RepoImage{ + {Image: goodImage, Tag: "goodImage"}, + }, + MultiArchImages: []test.RepoMultiArchImage{ + {MultiarchImage: goodMultiArch, Tag: "goodMultiArch"}, + }, + }, + test.Repo{ + Name: "repo5-signed", + Images: []test.RepoImage{ + {Image: goodImage, Tag: "goodImage"}, // is fake signed by the image below + {Image: test.CreateFakeTestSignature(goodImage.DescriptorRef())}, + }, + }, + ) + + skipCVE := convert.SkipQGLField{Vulnerabilities: true} + + // Test different combinations of the filter + + reposSum, pageInfo, err := convert.PaginatedRepoMeta2Summaries( + ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + mTypes.Filter{ + Os: []*string{ref("good-os")}, + Arch: []*string{ref("good-arch")}, + IsBookmarked: ref(true), + IsStarred: ref(true), + }, + pagination.PageInput{SortBy: pagination.AlphabeticAsc}, + ) + So(err, ShouldBeNil) + So(len(reposSum), ShouldEqual, 2) + So(*reposSum[0].Name, ShouldResemble, "repo1-only-images") + So(*reposSum[1].Name, ShouldResemble, "repo3-only-multiarch") + So(pageInfo.ItemCount, ShouldEqual, 2) + + reposSum, pageInfo, err = convert.PaginatedRepoMeta2Summaries( + ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + mTypes.Filter{ + Os: []*string{ref("good-os")}, + Arch: []*string{ref("good-arch")}, + IsBookmarked: ref(true), + IsStarred: ref(true), + HasToBeSigned: ref(true), + }, + pagination.PageInput{SortBy: pagination.AlphabeticAsc}, + ) + So(err, ShouldBeNil) + So(len(reposSum), ShouldEqual, 0) + So(pageInfo.ItemCount, ShouldEqual, 0) + + reposSum, pageInfo, err = convert.PaginatedRepoMeta2Summaries( + ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + mTypes.Filter{ + HasToBeSigned: ref(true), + }, + pagination.PageInput{SortBy: pagination.AlphabeticAsc}, + ) + So(err, ShouldBeNil) + So(len(reposSum), ShouldEqual, 1) + So(*reposSum[0].Name, ShouldResemble, "repo5-signed") + So(pageInfo.ItemCount, ShouldEqual, 1) + + // no filter + reposSum, pageInfo, err = convert.PaginatedRepoMeta2Summaries( + ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticAsc}, + ) + So(err, ShouldBeNil) + So(len(reposSum), ShouldEqual, 5) + So(*reposSum[0].Name, ShouldResemble, "repo1-only-images") + So(*reposSum[1].Name, ShouldResemble, "repo2-only-bad-images") + So(*reposSum[2].Name, ShouldResemble, "repo3-only-multiarch") + So(*reposSum[3].Name, ShouldResemble, "repo4-not-bookmarked-or-starred") + So(*reposSum[4].Name, ShouldResemble, "repo5-signed") + So(pageInfo.ItemCount, ShouldEqual, 5) + + // no filter opposite sorting + reposSum, pageInfo, err = convert.PaginatedRepoMeta2Summaries( + ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticDsc}, + ) + So(err, ShouldBeNil) + So(len(reposSum), ShouldEqual, 5) + So(*reposSum[0].Name, ShouldResemble, "repo5-signed") + So(*reposSum[1].Name, ShouldResemble, "repo4-not-bookmarked-or-starred") + So(*reposSum[2].Name, ShouldResemble, "repo3-only-multiarch") + So(*reposSum[3].Name, ShouldResemble, "repo2-only-bad-images") + So(*reposSum[4].Name, ShouldResemble, "repo1-only-images") + So(pageInfo.ItemCount, ShouldEqual, 5) + + // add pagination + reposSum, pageInfo, err = convert.PaginatedRepoMeta2Summaries( + ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + mTypes.Filter{ + Os: []*string{ref("good-os")}, + Arch: []*string{ref("good-arch")}, + IsBookmarked: ref(true), + IsStarred: ref(true), + }, + pagination.PageInput{Limit: 1, Offset: 0, SortBy: pagination.AlphabeticAsc}, + ) + So(err, ShouldBeNil) + So(len(reposSum), ShouldEqual, 1) + So(*reposSum[0].Name, ShouldResemble, "repo1-only-images") + So(pageInfo.ItemCount, ShouldEqual, 1) + So(pageInfo.TotalCount, ShouldEqual, 2) + + reposSum, pageInfo, err = convert.PaginatedRepoMeta2Summaries( + ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + mTypes.Filter{ + Os: []*string{ref("good-os")}, + Arch: []*string{ref("good-arch")}, + IsBookmarked: ref(true), + IsStarred: ref(true), + }, + pagination.PageInput{Limit: 1, Offset: 1, SortBy: pagination.AlphabeticAsc}, + ) + So(err, ShouldBeNil) + So(len(reposSum), ShouldEqual, 1) + So(*reposSum[0].Name, ShouldResemble, "repo3-only-multiarch") + So(pageInfo.ItemCount, ShouldEqual, 1) + So(pageInfo.TotalCount, ShouldEqual, 2) + }) +} diff --git a/pkg/extensions/search/convert/metadb.go b/pkg/extensions/search/convert/metadb.go index 6c0c1bb9a6..acc0b9b047 100644 --- a/pkg/extensions/search/convert/metadb.go +++ b/pkg/extensions/search/convert/metadb.go @@ -15,7 +15,8 @@ import ( "github.com/vektah/gqlparser/v2/gqlerror" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/common" + zcommon "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/common/pagination" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/gql_generated" @@ -134,9 +135,32 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMetadata, StarCount: &repoStarCount, IsBookmarked: &repoIsUserBookMarked, IsStarred: &repoIsUserStarred, + Rank: &repoMeta.Rank, } } +func PaginatedRepoMeta2Summaries(ctx context.Context, repoMetas []mTypes.RepoMetadata, + manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData, + skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, +) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) { + reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) + if err != nil { + return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err + } + + for _, repoMeta := range repoMetas { + repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo) + + if RepoSumAcceptedByFilter(repoSummary, filter) { + reposPageFinder.Add(repoSummary) + } + } + + page, pageInfo := reposPageFinder.Page() + + return page, pageInfo, nil +} + func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time, lastUpdatedImageSummary *gql_generated.ImageSummary, imageSummary *gql_generated.ImageSummary, ) *gql_generated.ImageSummary { @@ -265,6 +289,7 @@ func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest Labels: &annotations.Labels, Source: &annotations.Source, Vendor: &annotations.Vendor, + Authors: &annotations.Authors, Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{ MaxSeverity: &imageCveSummary.MaxSeverity, Count: &imageCveSummary.Count, @@ -306,8 +331,8 @@ func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest go repoName = repo configDigest = manifestContent.Config.Digest.String() configSize = manifestContent.Config.Size - artifactType = common.GetManifestArtifactType(manifestContent) - imageLastUpdated = common.GetImageLastUpdated(configContent) + artifactType = zcommon.GetManifestArtifactType(manifestContent) + imageLastUpdated = zcommon.GetImageLastUpdated(configContent) downloadCount = repoMeta.Statistics[digest.String()].DownloadCount isSigned = false ) @@ -474,8 +499,8 @@ func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descri manifestDigestStr = digest.String() configDigest = manifestContent.Config.Digest.String() configSize = manifestContent.Config.Size - artifactType = common.GetManifestArtifactType(manifestContent) - imageLastUpdated = common.GetImageLastUpdated(configContent) + artifactType = zcommon.GetManifestArtifactType(manifestContent) + imageLastUpdated = zcommon.GetImageLastUpdated(configContent) downloadCount = manifestMeta.DownloadCount isSigned = false ) @@ -599,6 +624,36 @@ func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMetadata, return imageSummaries } +func PaginatedRepoMeta2ImageSummaries(ctx context.Context, reposMeta []mTypes.RepoMetadata, + manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData, + skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, +) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) { + imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) + if err != nil { + return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err + } + + for _, repoMeta := range reposMeta { + for tag := range repoMeta.Tags { + descriptor := repoMeta.Tags[tag] + + imageSummary, _, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, skip.Vulnerabilities, + repoMeta, manifestMetaMap, indexDataMap, cveInfo) + if err != nil { + continue + } + + if ImgSumAcceptedByFilter(imageSummary, filter) { + imagePageFinder.Add(imageSummary) + } + } + } + + page, pageInfo := imagePageFinder.Page() + + return page, pageInfo, nil +} + func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMetadata, manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData, skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger, diff --git a/pkg/extensions/search/convert/utils.go b/pkg/extensions/search/convert/utils.go new file mode 100644 index 0000000000..d305dcbbad --- /dev/null +++ b/pkg/extensions/search/convert/utils.go @@ -0,0 +1,115 @@ +package convert + +import ( + zcommon "zotregistry.io/zot/pkg/common" + gql_gen "zotregistry.io/zot/pkg/extensions/search/gql_generated" + mTypes "zotregistry.io/zot/pkg/meta/types" +) + +func ImgSumAcceptedByFilter(imageSummary *gql_gen.ImageSummary, filter mTypes.Filter) bool { + if filter.Arch != nil { + foundArch := false + for _, arch := range filter.Arch { + foundArch = foundArch || zcommon.ContainsString(imgArchList(imageSummary), *arch) + } + + if !foundArch { + return false + } + } + + if filter.Os != nil { + foundOs := false + for _, os := range filter.Os { + foundOs = foundOs || zcommon.ContainsString(imgOsList(imageSummary), *os) + } + + if !foundOs { + return false + } + } + + if filter.HasToBeSigned != nil && *filter.HasToBeSigned && !*imageSummary.IsSigned { + return false + } + + return true +} + +func imgOsList(imageSummary *gql_gen.ImageSummary) []string { + list := []string{} + + for _, manifest := range imageSummary.Manifests { + list = append(list, *manifest.Platform.Os) + } + + return list +} + +func imgArchList(imageSummary *gql_gen.ImageSummary) []string { + list := []string{} + + for _, manifest := range imageSummary.Manifests { + list = append(list, *manifest.Platform.Arch) + } + + return list +} + +func RepoSumAcceptedByFilter(repoSummary *gql_gen.RepoSummary, filter mTypes.Filter) bool { + if filter.Arch != nil { + foundArch := false + for _, arch := range filter.Arch { + foundArch = foundArch || zcommon.ContainsString(repoArchList(repoSummary.Platforms), *arch) + } + + if !foundArch { + return false + } + } + + if filter.Os != nil { + foundOs := false + for _, os := range filter.Os { + foundOs = foundOs || zcommon.ContainsString(repoOsList(repoSummary.Platforms), *os) + } + + if !foundOs { + return false + } + } + + if filter.HasToBeSigned != nil && *filter.HasToBeSigned && !*repoSummary.NewestImage.IsSigned { + return false + } + + if filter.IsBookmarked != nil && *filter.IsBookmarked != *repoSummary.IsBookmarked { + return false + } + + if filter.IsStarred != nil && *filter.IsStarred != *repoSummary.IsStarred { + return false + } + + return true +} + +func repoOsList(platforms []*gql_gen.Platform) []string { + list := []string{} + + for _, platform := range platforms { + list = append(list, *platform.Os) + } + + return list +} + +func repoArchList(platforms []*gql_gen.Platform) []string { + list := []string{} + + for _, platform := range platforms { + list = append(list, *platform.Arch) + } + + return list +} diff --git a/pkg/extensions/search/gql_generated/generated.go b/pkg/extensions/search/gql_generated/generated.go index cfcb200f54..cc25d8bbed 100644 --- a/pkg/extensions/search/gql_generated/generated.go +++ b/pkg/extensions/search/gql_generated/generated.go @@ -195,6 +195,7 @@ type ComplexityRoot struct { Name func(childComplexity int) int NewestImage func(childComplexity int) int Platforms func(childComplexity int) int + Rank func(childComplexity int) int Size func(childComplexity int) int StarCount func(childComplexity int) int Vendors func(childComplexity int) int @@ -988,6 +989,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.RepoSummary.Platforms(childComplexity), true + case "RepoSummary.Rank": + if e.complexity.RepoSummary.Rank == nil { + break + } + + return e.complexity.RepoSummary.Rank(childComplexity), true + case "RepoSummary.Size": if e.complexity.RepoSummary.Size == nil { break @@ -1439,6 +1447,10 @@ type RepoSummary { True if the repository is stared by the current user, fale otherwise """ IsStarred: Boolean + """ + Rank represents how good the match was between the querried repo name and this repo summary. + """ + Rank: Int } """ @@ -2909,6 +2921,8 @@ func (ec *executionContext) fieldContext_GlobalSearchResult_Repos(ctx context.Co return ec.fieldContext_RepoSummary_IsBookmarked(ctx, field) case "IsStarred": return ec.fieldContext_RepoSummary_IsStarred(ctx, field) + case "Rank": + return ec.fieldContext_RepoSummary_Rank(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type RepoSummary", field.Name) }, @@ -5333,6 +5347,8 @@ func (ec *executionContext) fieldContext_PaginatedReposResult_Results(ctx contex return ec.fieldContext_RepoSummary_IsBookmarked(ctx, field) case "IsStarred": return ec.fieldContext_RepoSummary_IsStarred(ctx, field) + case "Rank": + return ec.fieldContext_RepoSummary_Rank(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type RepoSummary", field.Name) }, @@ -6806,6 +6822,8 @@ func (ec *executionContext) fieldContext_RepoInfo_Summary(ctx context.Context, f return ec.fieldContext_RepoSummary_IsBookmarked(ctx, field) case "IsStarred": return ec.fieldContext_RepoSummary_IsStarred(ctx, field) + case "Rank": + return ec.fieldContext_RepoSummary_Rank(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type RepoSummary", field.Name) }, @@ -7271,6 +7289,47 @@ func (ec *executionContext) fieldContext_RepoSummary_IsStarred(ctx context.Conte return fc, nil } +func (ec *executionContext) _RepoSummary_Rank(ctx context.Context, field graphql.CollectedField, obj *RepoSummary) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_RepoSummary_Rank(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Rank, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.(*int) + fc.Result = res + return ec.marshalOInt2áš–int(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_RepoSummary_Rank(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "RepoSummary", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + return nil, errors.New("field of type Int does not have child fields") + }, + } + return fc, nil +} + func (ec *executionContext) _SignatureSummary_Tool(ctx context.Context, field graphql.CollectedField, obj *SignatureSummary) (ret graphql.Marshaler) { fc, err := ec.fieldContext_SignatureSummary_Tool(ctx, field) if err != nil { @@ -10421,6 +10480,8 @@ func (ec *executionContext) _RepoSummary(ctx context.Context, sel ast.SelectionS out.Values[i] = ec._RepoSummary_IsBookmarked(ctx, field, obj) case "IsStarred": out.Values[i] = ec._RepoSummary_IsStarred(ctx, field, obj) + case "Rank": + out.Values[i] = ec._RepoSummary_Rank(ctx, field, obj) default: panic("unknown field " + strconv.Quote(field.Name)) } diff --git a/pkg/extensions/search/gql_generated/models_gen.go b/pkg/extensions/search/gql_generated/models_gen.go index 0d8a03099d..de48ff492b 100644 --- a/pkg/extensions/search/gql_generated/models_gen.go +++ b/pkg/extensions/search/gql_generated/models_gen.go @@ -293,6 +293,8 @@ type RepoSummary struct { IsBookmarked *bool `json:"IsBookmarked,omitempty"` // True if the repository is stared by the current user, fale otherwise IsStarred *bool `json:"IsStarred,omitempty"` + // Rank represents how good the match was between the querried repo name and this repo summary. + Rank *int `json:"Rank,omitempty"` } // Contains details about the signature diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index 60e43b511e..e8b592db4e 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -19,6 +19,7 @@ import ( zerr "zotregistry.io/zot/errors" zcommon "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/common/pagination" "zotregistry.io/zot/pkg/extensions/search/convert" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" @@ -114,8 +115,6 @@ func FilterByDigest(digest string) mTypes.FilterFunc { func getImageListForDigest(ctx context.Context, digest string, metaDB mTypes.MetaDB, cveInfo cveinfo.CveInfo, requestedPage *gql_generated.PageInput, ) (*gql_generated.PaginatedImagesResult, error) { - imageList := make([]*gql_generated.ImageSummary, 0) - if requestedPage == nil { requestedPage = &gql_generated.PageInput{} } @@ -124,30 +123,28 @@ func getImageListForDigest(ctx context.Context, digest string, metaDB mTypes.Met Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"), } - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), ), } // get all repos - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx, - FilterByDigest(digest), mTypes.Filter{}, pageInput) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByDigest(digest)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - for _, repoMeta := range reposMeta { - imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, - skip, cveInfo) - - imageList = append(imageList, imageSummaries...) + imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, + indexDataMap, skip, cveInfo, mTypes.Filter{}, pageInput) + if err != nil { + return &gql_generated.PaginatedImagesResult{}, err } return &gql_generated.PaginatedImagesResult{ - Results: imageList, + Results: imageSummaries, Page: &gql_generated.PageInfo{ TotalCount: pageInfo.TotalCount, ItemCount: pageInfo.ItemCount, @@ -380,6 +377,33 @@ func FilterByTagInfo(tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { } } +func FilterByRepoAndTagInfo(repo string, tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { + return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { + if repoMeta.Name != repo { + return false + } + + manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + + for _, tagInfo := range tagsInfo { + switch tagInfo.Descriptor.MediaType { + case ispec.MediaTypeImageManifest: + if tagInfo.Descriptor.Digest.String() == manifestDigest { + return true + } + case ispec.MediaTypeImageIndex: + for _, manifestDesc := range tagInfo.Manifests { + if manifestDesc.Digest.String() == manifestDigest { + return true + } + } + } + } + + return false + } +} + func getImageListForCVE( ctx context.Context, cveID string, @@ -393,10 +417,7 @@ func getImageListForCVE( // Infinite page to make sure we scan all repos in advance, before filtering results // The CVE scan logic is called from here, not in the actual filter, // this is because we shouldn't keep the DB locked while we wait on scan results - reposMeta, err := metaDB.GetMultipleRepoMeta(ctx, - func(repoMeta mTypes.RepoMetadata) bool { return true }, - mTypes.PageInput{Limit: 0, Offset: 0, SortBy: mTypes.SortCriteria(gql_generated.SortCriteriaUpdateTime)}, - ) + reposMeta, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -419,8 +440,6 @@ func getImageListForCVE( affectedImages = append(affectedImages, tagsInfo...) } - imageList := make([]*gql_generated.ImageSummary, 0) - // We're not interested in other vulnerabilities skip := convert.SkipQGLField{Vulnerabilities: true} @@ -440,29 +459,28 @@ func getImageListForCVE( } // Actual page requested by user - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } // get all repos - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx, - FilterByTagInfo(affectedImages), localFilter, pageInput) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByTagInfo(affectedImages)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - for _, repoMeta := range reposMeta { - imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo) - - imageList = append(imageList, imageSummaries...) + imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, + indexDataMap, skip, cveInfo, localFilter, pageInput) + if err != nil { + return &gql_generated.PaginatedImagesResult{}, err } return &gql_generated.PaginatedImagesResult{ - Results: imageList, + Results: imageSummaries, Page: &gql_generated.PageInfo{ TotalCount: pageInfo.TotalCount, ItemCount: pageInfo.ItemCount, @@ -514,33 +532,28 @@ func getImageListWithCVEFixed( } // Actual page requested by user - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } // get all repos - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx, - FilterByTagInfo(tagsInfo), localFilter, pageInput, - ) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByRepoAndTagInfo(repo, tagsInfo)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - for _, repoMeta := range reposMeta { - if repoMeta.Name != repo { - continue - } - - imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo) - imageList = append(imageList, imageSummaries...) + imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, + indexDataMap, skip, cveInfo, localFilter, pageInput) + if err != nil { + return &gql_generated.PaginatedImagesResult{}, err } return &gql_generated.PaginatedImagesResult{ - Results: imageList, + Results: imageSummaries, Page: &gql_generated.PageInfo{ TotalCount: pageInfo.TotalCount, ItemCount: pageInfo.ItemCount, @@ -555,7 +568,6 @@ func repoListWithNewestImage( requestedPage *gql_generated.PageInput, metaDB mTypes.MetaDB, ) (*gql_generated.PaginatedReposResult, error) { - repos := []*gql_generated.RepoSummary{} paginatedRepos := &gql_generated.PaginatedReposResult{} if requestedPage == nil { @@ -566,29 +578,30 @@ func repoListWithNewestImage( Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Results.NewestImage.Vulnerabilities"), } - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.SearchRepos(ctx, "", mTypes.Filter{}, pageInput) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, "") if err != nil { return &gql_generated.PaginatedReposResult{}, err } - for _, repoMeta := range reposMeta { - repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap, - skip, cveInfo) - repos = append(repos, repoSummary) + repos, pageInfo, err := convert.PaginatedRepoMeta2Summaries(ctx, reposMeta, manifestMetaMap, indexDataMap, + skip, cveInfo, mTypes.Filter{}, pageInput) + if err != nil { + return &gql_generated.PaginatedReposResult{}, err } paginatedRepos.Page = &gql_generated.PageInfo{ TotalCount: pageInfo.TotalCount, ItemCount: pageInfo.ItemCount, } + paginatedRepos.Results = repos return paginatedRepos, nil @@ -640,9 +653,6 @@ func getFilteredPaginatedRepos( requestedPage *gql_generated.PageInput, metaDB mTypes.MetaDB, ) (*gql_generated.PaginatedReposResult, error) { - repos := []*gql_generated.RepoSummary{} - paginatedRepos := &gql_generated.PaginatedReposResult{} - if requestedPage == nil { requestedPage = &gql_generated.PageInput{} } @@ -651,32 +661,32 @@ func getFilteredPaginatedRepos( Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Results.NewestImage.Vulnerabilities"), } - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterRepos(ctx, filterFn, pageInput) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterRepos(ctx, filterFn) if err != nil { - return paginatedRepos, err - } - - for _, repoMeta := range reposMeta { - repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap, - skip, cveInfo) - repos = append(repos, repoSummary) + return &gql_generated.PaginatedReposResult{}, err } - paginatedRepos.Page = &gql_generated.PageInfo{ - TotalCount: pageInfo.TotalCount, - ItemCount: pageInfo.ItemCount, + repos, pageInfo, err := convert.PaginatedRepoMeta2Summaries(ctx, reposMeta, manifestMetaMap, indexDataMap, + skip, cveInfo, mTypes.Filter{}, pageInput) + if err != nil { + return &gql_generated.PaginatedReposResult{}, err } - paginatedRepos.Results = repos - return paginatedRepos, nil + return &gql_generated.PaginatedReposResult{ + Results: repos, + Page: &gql_generated.PageInfo{ + TotalCount: pageInfo.TotalCount, + ItemCount: pageInfo.ItemCount, + }, + }, nil } func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filter *gql_generated.Filter, @@ -684,7 +694,6 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte ) (*gql_generated.PaginatedReposResult, []*gql_generated.ImageSummary, []*gql_generated.LayerSummary, error, ) { preloads := convert.GetPreloads(ctx) - repos := []*gql_generated.RepoSummary{} paginatedRepos := gql_generated.PaginatedReposResult{} images := []*gql_generated.ImageSummary{} layers := []*gql_generated.LayerSummary{} @@ -709,24 +718,23 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte Vulnerabilities: canSkipField(preloads, "Repos.NewestImage.Vulnerabilities"), } - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), ), } - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.SearchRepos(ctx, query, localFilter, pageInput) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, query) if err != nil { return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err } - for _, repoMeta := range reposMeta { - repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap, - skip, cveInfo) - - repos = append(repos, repoSummary) + repos, pageInfo, err := convert.PaginatedRepoMeta2Summaries(ctx, reposMeta, manifestMetaMap, indexDataMap, + skip, cveInfo, localFilter, pageInput) + if err != nil { + return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err } paginatedRepos.Page = &gql_generated.PageInfo{ @@ -740,25 +748,27 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte Vulnerabilities: canSkipField(preloads, "Images.Vulnerabilities"), } - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), ), } - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.SearchTags(ctx, query, localFilter, pageInput) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, query) if err != nil { return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err } - for _, repoMeta := range reposMeta { - imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo) - - images = append(images, imageSummaries...) + imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, + indexDataMap, skip, cveInfo, localFilter, pageInput) + if err != nil { + return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err } + images = imageSummaries + paginatedRepos.Page = &gql_generated.PageInfo{ TotalCount: pageInfo.TotalCount, ItemCount: pageInfo.ItemCount, @@ -778,16 +788,14 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB requestedPage *gql_generated.PageInput, cveInfo cveinfo.CveInfo, log log.Logger, ) (*gql_generated.PaginatedImagesResult, error) { - derivedList := make([]*gql_generated.ImageSummary, 0) - if requestedPage == nil { requestedPage = &gql_generated.PageInput{} } - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } @@ -811,26 +819,15 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB } // we need all available tags - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx, - filterDerivedImages(searchedImage), - mTypes.Filter{}, - pageInput) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, filterDerivedImages(searchedImage)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - for _, repoMeta := range reposMeta { - summary := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo) - derivedList = append(derivedList, summary...) - } - - if len(derivedList) == 0 { - log.Info().Msg("no images found") - - return &gql_generated.PaginatedImagesResult{ - Page: &gql_generated.PageInfo{}, - Results: derivedList, - }, nil + derivedList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap, + skip, cveInfo, mTypes.Filter{}, pageInput) + if err != nil { + return &gql_generated.PaginatedImagesResult{}, err } return &gql_generated.PaginatedImagesResult{ @@ -892,16 +889,14 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy requestedPage *gql_generated.PageInput, cveInfo cveinfo.CveInfo, log log.Logger, ) (*gql_generated.PaginatedImagesResult, error) { - imageSummaries := make([]*gql_generated.ImageSummary, 0) - if requestedPage == nil { requestedPage = &gql_generated.PageInput{} } - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } @@ -926,26 +921,15 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy } // we need all available tags - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx, - filterBaseImages(searchedImage), - mTypes.Filter{}, - pageInput) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, filterBaseImages(searchedImage)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - for _, repoMeta := range reposMeta { - summary := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo) - imageSummaries = append(imageSummaries, summary...) - } - - if len(imageSummaries) == 0 { - log.Info().Msg("no images found") - - return &gql_generated.PaginatedImagesResult{ - Results: imageSummaries, - Page: &gql_generated.PageInfo{}, - }, nil + baseList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap, + skip, cveInfo, mTypes.Filter{}, pageInput) + if err != nil { + return &gql_generated.PaginatedImagesResult{}, err } return &gql_generated.PaginatedImagesResult{ @@ -953,7 +937,7 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy TotalCount: pageInfo.TotalCount, ItemCount: pageInfo.ItemCount, }, - Results: imageSummaries, + Results: baseList, }, nil } @@ -1265,8 +1249,6 @@ func searchingForRepos(query string) bool { func getImageList(ctx context.Context, repo string, metaDB mTypes.MetaDB, cveInfo cveinfo.CveInfo, requestedPage *gql_generated.PageInput, log log.Logger, //nolint:unparam ) (*gql_generated.PaginatedImagesResult, error) { - imageList := make([]*gql_generated.ImageSummary, 0) - if requestedPage == nil { requestedPage = &gql_generated.PageInput{} } @@ -1275,32 +1257,27 @@ func getImageList(ctx context.Context, repo string, metaDB mTypes.MetaDB, cveInf Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"), } - pageInput := mTypes.PageInput{ + pageInput := pagination.PageInput{ Limit: safeDereferencing(requestedPage.Limit, 0), Offset: safeDereferencing(requestedPage.Offset, 0), - SortBy: mTypes.SortCriteria( + SortBy: pagination.SortCriteria( safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), ), } - // reposMeta, manifestMetaMap, err := metaDB.SearchRepos(ctx, repo, mTypes.Filter{}, pageInput) - reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx, + // reposMeta, manifestMetaMap, err := metaDB.SearchRepos(ctx, repo) + reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }, - mTypes.Filter{}, - pageInput) + return repoMeta.Name == repo || repo == "" + }) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - for _, repoMeta := range reposMeta { - if repoMeta.Name != repo && repo != "" { - continue - } - imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo) - - imageList = append(imageList, imageSummaries...) + imageList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, + indexDataMap, skip, cveInfo, mTypes.Filter{}, pageInput) + if err != nil { + return &gql_generated.PaginatedImagesResult{}, err } return &gql_generated.PaginatedImagesResult{ diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 14ec42a7a4..f2e2a4b456 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -21,7 +21,6 @@ import ( "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/boltdb" - "zotregistry.io/zot/pkg/meta/pagination" mTypes "zotregistry.io/zot/pkg/meta/types" localCtx "zotregistry.io/zot/pkg/requestcontext" "zotregistry.io/zot/pkg/storage" @@ -35,13 +34,12 @@ func TestGlobalSearch(t *testing.T) { const query = "repo1" Convey("MetaDB SearchRepos error", func() { mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchReposFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata), - map[string]mTypes.IndexData{}, common.PageInfo{}, ErrTestError + map[string]mTypes.IndexData{}, ErrTestError }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, @@ -55,12 +53,28 @@ func TestGlobalSearch(t *testing.T) { So(repos.Results, ShouldBeEmpty) }) + Convey("paginated fail", func() { + pageInput := &gql_generated.PageInput{ + Limit: ref(-1), + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, _, _, err := globalSearch(responseContext, "repo", mocks.MetaDBMock{}, &gql_generated.Filter{}, + pageInput, mocks.CveInfoMock{}, log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + + _, _, _, err = globalSearch(responseContext, "repo:tag", mocks.MetaDBMock{}, &gql_generated.Filter{}, + pageInput, mocks.CveInfoMock{}, log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + }) + Convey("MetaDB SearchRepo is successful", func() { mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchReposFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -120,7 +134,7 @@ func TestGlobalSearch(t *testing.T) { }, } - return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } @@ -148,10 +162,9 @@ func TestGlobalSearch(t *testing.T) { Convey("MetaDB SearchRepo Bad manifest referenced", func() { mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchReposFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -183,7 +196,7 @@ func TestGlobalSearch(t *testing.T) { }, } - return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } @@ -222,10 +235,9 @@ func TestGlobalSearch(t *testing.T) { Convey("MetaDB SearchRepo good manifest referenced and bad config blob", func() { mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchReposFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -257,7 +269,7 @@ func TestGlobalSearch(t *testing.T) { }, } - return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } @@ -295,13 +307,12 @@ func TestGlobalSearch(t *testing.T) { Convey("MetaDB SearchTags gives error", func() { mockMetaDB := mocks.MetaDBMock{ - SearchTagsFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, + SearchTagsFn: func(ctx context.Context, searchText string, + ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata), - map[string]mTypes.IndexData{}, common.PageInfo{}, ErrTestError + map[string]mTypes.IndexData{}, ErrTestError }, } const query = "repo1:1.0.1" @@ -319,10 +330,9 @@ func TestGlobalSearch(t *testing.T) { Convey("MetaDB SearchTags is successful", func() { mockMetaDB := mocks.MetaDBMock{ - SearchTagsFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchTagsFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -376,7 +386,7 @@ func TestGlobalSearch(t *testing.T) { }, } - return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } @@ -408,13 +418,11 @@ func TestRepoListWithNewestImage(t *testing.T) { Convey("RepoListWithNewestImage", t, func() { Convey("MetaDB SearchRepos error", func() { mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, - error, + SearchReposFn: func(ctx context.Context, searchText string, + ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata), - map[string]mTypes.IndexData{}, common.PageInfo{}, ErrTestError + map[string]mTypes.IndexData{}, ErrTestError }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, @@ -434,12 +442,24 @@ func TestRepoListWithNewestImage(t *testing.T) { So(repos.Results, ShouldBeEmpty) }) + Convey("paginated fail", func() { + pageInput := &gql_generated.PageInput{ + Limit: ref(-1), + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err := repoListWithNewestImage(responseContext, mocks.CveInfoMock{}, log.NewLogger("debug", ""), + pageInput, mocks.MetaDBMock{}) + So(err, ShouldNotBeNil) + }) + Convey("MetaDB SearchRepo bad manifest referenced", func() { mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchReposFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -496,7 +516,7 @@ func TestRepoListWithNewestImage(t *testing.T) { }, } - return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } @@ -521,15 +541,10 @@ func TestRepoListWithNewestImage(t *testing.T) { createTime := time.Now() createTime2 := createTime.Add(time.Second) mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchReposFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { - pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, - requestedPage.SortBy) - So(err, ShouldBeNil) - repos := []mTypes.RepoMetadata{ { Name: "repo1", @@ -567,16 +582,6 @@ func TestRepoListWithNewestImage(t *testing.T) { }, } - for _, repoMeta := range repos { - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - UpdateTime: createTime, - }) - createTime = createTime.Add(time.Second) - } - - repos, _ = pageFinder.Page() - configBlob1, err := json.Marshal(ispec.Image{ Config: ispec.ImageConfig{ Labels: map[string]string{}, @@ -607,7 +612,7 @@ func TestRepoListWithNewestImage(t *testing.T) { }, } - return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } Convey("MetaDB missing requestedPage", func() { @@ -694,30 +699,42 @@ func TestGetFilteredPaginatedRepos(t *testing.T) { log.NewLogger("debug", ""), nil, mocks.MetaDBMock{ - FilterReposFn: func(ctx context.Context, filter mTypes.FilterRepoFunc, requestedPage mTypes.PageInput, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, + FilterReposFn: func(ctx context.Context, filter mTypes.FilterRepoFunc, + ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, ErrTestError + ErrTestError }, }, ) So(err, ShouldNotBeNil) }) + + Convey("Paginated convert fails", t, func() { + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + _, err := getFilteredPaginatedRepos(responseContext, + mocks.CveInfoMock{}, + func(repoMeta mTypes.RepoMetadata) bool { return true }, + log.NewLogger("debug", ""), + &gql_generated.PageInput{Limit: ref(-1)}, + mocks.MetaDBMock{}, + ) + So(err, ShouldNotBeNil) + }) } func TestImageListForDigest(t *testing.T) { Convey("getImageList", t, func() { Convey("no page requested, FilterTagsFn returns error", func() { mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, + ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, ErrTestError + ErrTestError }, } @@ -727,12 +744,19 @@ func TestImageListForDigest(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("Paginated convert fails", func() { + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + _, err := getImageListForDigest(responseContext, "invalid", mocks.MetaDBMock{}, mocks.CveInfoMock{}, + &gql_generated.PageInput{Limit: ref(-1)}) + So(err, ShouldNotBeNil) + }) + Convey("invalid manifest blob", func() { mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -760,7 +784,7 @@ func TestImageListForDigest(t *testing.T) { }, } - return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil }, } @@ -779,10 +803,9 @@ func TestImageListForDigest(t *testing.T) { manifestDigest := godigest.FromBytes(manifestBlob).String() mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -816,7 +839,7 @@ func TestImageListForDigest(t *testing.T) { repos[0].Tags = matchedTags - return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil }, } @@ -854,10 +877,9 @@ func TestImageListForDigest(t *testing.T) { configDigest := godigest.FromBytes(configBlob) mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -896,7 +918,7 @@ func TestImageListForDigest(t *testing.T) { repos[0].Tags = matchedTags - return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil }, } @@ -930,10 +952,9 @@ func TestImageListForDigest(t *testing.T) { layerDigest := godigest.Digest("validDigest") mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -974,7 +995,7 @@ func TestImageListForDigest(t *testing.T) { repos[0].Tags = matchedTags - return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil }, } @@ -1006,10 +1027,9 @@ func TestImageListForDigest(t *testing.T) { So(err, ShouldBeNil) mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -1045,7 +1065,7 @@ func TestImageListForDigest(t *testing.T) { repos[i].Tags = matchedTags } - return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil }, } @@ -1064,7 +1084,7 @@ func TestImageListForDigest(t *testing.T) { imageSummaries, err := getImageListForDigest(responseContext, manifestDigest, mockSearchDB, mocks.CveInfoMock{}, &pageInput) So(err, ShouldBeNil) - So(len(imageSummaries.Results), ShouldEqual, 2) + So(len(imageSummaries.Results), ShouldEqual, 1) }) Convey("valid imageListForDigest, multiple matching tags limited by pageInput", func() { @@ -1077,18 +1097,9 @@ func TestImageListForDigest(t *testing.T) { So(err, ShouldBeNil) mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, - error, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, + ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { - pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, - requestedPage.SortBy) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, err - } - repos := []mTypes.RepoMetadata{ { Name: "test", @@ -1122,14 +1133,10 @@ func TestImageListForDigest(t *testing.T) { repos[i].Tags = matchedTags - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repo, - }) + repos = append(repos, repo) } - repos, _ = pageFinder.Page() - - return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil }, } @@ -1360,13 +1367,12 @@ func TestImageList(t *testing.T) { testLogger := log.NewLogger("debug", "") Convey("no page requested, SearchRepoFn returns error", func() { mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, + ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, common.PageInfo{}, ErrTestError + map[string]mTypes.IndexData{}, ErrTestError }, } @@ -1377,12 +1383,21 @@ func TestImageList(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("Paginated convert fails", func() { + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err := getImageList(responseContext, "test", mocks.MetaDBMock{}, mocks.CveInfoMock{}, + &gql_generated.PageInput{Limit: ref(-1)}, log.NewLogger("debug", "")) + + So(err, ShouldNotBeNil) + }) + Convey("valid repoList returned", func() { mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, + FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { repos := []mTypes.RepoMetadata{ { @@ -1427,7 +1442,12 @@ func TestImageList(t *testing.T) { }, } - return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + if !filterFunc(repos[0], manifestMetaDatas["digestTag1.0.1"]) { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, + map[string]mTypes.IndexData{}, nil + } + + return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil }, } @@ -1567,7 +1587,6 @@ func TestQueryResolverErrors(t *testing.T) { }, mocks.MetaDBMock{ GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, - requestedPage mTypes.PageInput, ) ([]mTypes.RepoMetadata, error) { return []mTypes.RepoMetadata{}, ErrTestError }, @@ -1591,12 +1610,12 @@ func TestQueryResolverErrors(t *testing.T) { }, mocks.MetaDBMock{ FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, ErrTestError + ErrTestError }, }, mocks.CveInfoMock{}, @@ -1618,12 +1637,12 @@ func TestQueryResolverErrors(t *testing.T) { }, mocks.MetaDBMock{ FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, ErrTestError + ErrTestError }, }, mocks.CveInfoMock{}, @@ -1644,12 +1663,11 @@ func TestQueryResolverErrors(t *testing.T) { DefaultStore: mocks.MockedImageStore{}, }, mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchReposFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { - return nil, nil, nil, common.PageInfo{}, ErrTestError + return nil, nil, nil, ErrTestError }, }, mocks.CveInfoMock{}, @@ -1668,12 +1686,11 @@ func TestQueryResolverErrors(t *testing.T) { log, storage.StoreController{}, mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, + SearchReposFn: func(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { - return nil, nil, nil, common.PageInfo{}, ErrTestError + return nil, nil, nil, ErrTestError }, }, mocks.CveInfoMock{}, @@ -1693,12 +1710,12 @@ func TestQueryResolverErrors(t *testing.T) { storage.StoreController{}, mocks.MetaDBMock{ FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, ErrTestError + ErrTestError }, }, mocks.CveInfoMock{}, @@ -1790,12 +1807,12 @@ func TestQueryResolverErrors(t *testing.T) { storage.StoreController{}, mocks.MetaDBMock{ FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, ErrTestError + ErrTestError }, GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { return mTypes.RepoMetadata{ @@ -2211,6 +2228,18 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo So(expectedCves, ShouldContain, *cve.ID) } }) + + Convey("paginated fail", func() { + pageInput := &gql_generated.PageInput{ + Limit: ref(-1), + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err = getCVEListForImage(responseContext, "repo1:1.1.0", cveInfo, pageInput, "", log) + So(err, ShouldNotBeNil) + }) }) Convey("Get a list of images affected by a particular CVE ", t, func() { @@ -2260,6 +2289,19 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo } }) + Convey("paginated fail", func() { + pageInput := &gql_generated.PageInput{ + Limit: ref(-1), + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err = getImageListForCVE(responseContext, "repo1:1.1.0", cveInfo, &gql_generated.Filter{}, + pageInput, mocks.MetaDBMock{}, log) + So(err, ShouldNotBeNil) + }) + Convey("Paginated requests", func() { responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover, @@ -2503,6 +2545,19 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo So(len(images.Results), ShouldEqual, 0) }) + Convey("paginated fail", func() { + pageInput := &gql_generated.PageInput{ + Limit: ref(-1), + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err = getImageListWithCVEFixed(responseContext, "cve", "repo1:1.1.0", cveInfo, &gql_generated.Filter{}, + pageInput, mocks.MetaDBMock{}, log) + So(err, ShouldNotBeNil) + }) + Convey("Paginated requests", func() { responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover, @@ -2690,7 +2745,6 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo nil, mocks.MetaDBMock{ GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, - requestedPage mTypes.PageInput, ) ([]mTypes.RepoMetadata, error) { return []mTypes.RepoMetadata{{}}, nil }, @@ -2715,12 +2769,12 @@ func TestDerivedImageList(t *testing.T) { Convey("MetaDB FilterTags error", t, func() { mockSearchDB := mocks.MetaDBMock{ FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata), - make(map[string]mTypes.IndexData), common.PageInfo{}, ErrTestError + make(map[string]mTypes.IndexData), ErrTestError }, GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { return mTypes.RepoMetadata{}, ErrTestError @@ -2739,6 +2793,19 @@ func TestDerivedImageList(t *testing.T) { So(images.Results, ShouldBeEmpty) }) + Convey("paginated fail", t, func() { + pageInput := &gql_generated.PageInput{ + Limit: ref(-1), + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mocks.MetaDBMock{}, pageInput, + mocks.CveInfoMock{}, log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + }) + //nolint: dupl Convey("MetaDB FilterTags no repo available", t, func() { configBlob, err := json.Marshal(ispec.Image{ @@ -2757,12 +2824,12 @@ func TestDerivedImageList(t *testing.T) { mockSearchDB := mocks.MetaDBMock{ FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, nil + nil }, GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { return mTypes.RepoMetadata{ @@ -2908,14 +2975,10 @@ func TestDerivedImageList(t *testing.T) { }, nil }, FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { - pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, - requestedPage.SortBy) - So(err, ShouldBeNil) - repos := []mTypes.RepoMetadata{ { Name: "repo1", @@ -2941,14 +3004,9 @@ func TestDerivedImageList(t *testing.T) { } repos[i].Tags = matchedTags - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repo, - }) } - repos, pageInfo := pageFinder.Page() - return repos, manifestMetas, map[string]mTypes.IndexData{}, pageInfo, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } @@ -2988,12 +3046,12 @@ func TestBaseImageList(t *testing.T) { Convey("MetaDB FilterTags error", t, func() { mockSearchDB := mocks.MetaDBMock{ FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, ErrTestError + ErrTestError }, GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { return mTypes.RepoMetadata{}, ErrTestError @@ -3012,6 +3070,18 @@ func TestBaseImageList(t *testing.T) { So(images.Results, ShouldBeEmpty) }) + Convey("paginated fail", t, func() { + pageInput := &gql_generated.PageInput{ + Limit: ref(-1), + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + _, err := baseImageList(responseContext, "repo1:1.0.2", nil, mocks.MetaDBMock{}, + pageInput, mocks.CveInfoMock{}, log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + }) + //nolint: dupl Convey("MetaDB FilterTags no repo available", t, func() { configBlob, err := json.Marshal(ispec.Image{ @@ -3030,12 +3100,12 @@ func TestBaseImageList(t *testing.T) { mockSearchDB := mocks.MetaDBMock{ FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - common.PageInfo{}, nil + nil }, GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { return mTypes.RepoMetadata{ @@ -3175,14 +3245,10 @@ func TestBaseImageList(t *testing.T) { }, nil }, FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { - pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, - requestedPage.SortBy) - So(err, ShouldBeNil) - repos := []mTypes.RepoMetadata{ { Name: "repo1", @@ -3208,15 +3274,9 @@ func TestBaseImageList(t *testing.T) { } repos[i].Tags = matchedTags - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repo, - }) } - repos, pageInfo := pageFinder.Page() - - return repos, manifestMetas, map[string]mTypes.IndexData{}, pageInfo, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, @@ -3352,14 +3412,10 @@ func TestBaseImageList(t *testing.T) { }, nil }, FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput, + filterFunc mTypes.FilterFunc, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - common.PageInfo, error, + error, ) { - pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, - requestedPage.SortBy) - So(err, ShouldBeNil) - repos := []mTypes.RepoMetadata{ { Name: "repo1", @@ -3384,13 +3440,9 @@ func TestBaseImageList(t *testing.T) { } repos[i].Tags = matchedTags - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repo, - }) } - return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil + return repos, manifestMetas, map[string]mTypes.IndexData{}, nil }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, @@ -3537,5 +3589,31 @@ func TestFilterFunctions(t *testing.T) { ) So(ok, ShouldBeFalse) }) + + Convey("FilterByTagInfo", func() { + fFunc := FilterByTagInfo([]cvemodel.TagInfo{ + { + Descriptor: cvemodel.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + }, + Manifests: []cvemodel.DescriptorInfo{ + { + Descriptor: cvemodel.Descriptor{ + Digest: godigest.FromString("{}"), + }, + }, + }, + }, + }) + + ok := fFunc(mTypes.RepoMetadata{}, mTypes.ManifestMetadata{ManifestBlob: []byte("{}")}) + So(ok, ShouldBeTrue) + }) }) } + +func ref[T any](val T) *T { + ref := val + + return &ref +} diff --git a/pkg/extensions/search/schema.graphql b/pkg/extensions/search/schema.graphql index 236096b73e..a2d24af6bb 100644 --- a/pkg/extensions/search/schema.graphql +++ b/pkg/extensions/search/schema.graphql @@ -315,6 +315,10 @@ type RepoSummary { True if the repository is stared by the current user, fale otherwise """ IsStarred: Boolean + """ + Rank represents how good the match was between the querried repo name and this repo summary. + """ + Rank: Int } """ diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index 6350ed6815..b0732cfbf2 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -16,7 +16,6 @@ import ( zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/common" - "zotregistry.io/zot/pkg/meta/pagination" "zotregistry.io/zot/pkg/meta/signatures" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/meta/version" @@ -645,19 +644,10 @@ func (bdw *BoltDB) GetRepoStars(repo string) (int, error) { } func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, - requestedPage mTypes.PageInput, ) ([]mTypes.RepoMetadata, error) { - var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - pageFinder pagination.PageFinder - ) + foundRepos := []mTypes.RepoMetadata{} - pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return nil, err - } - - err = bdw.DB.View(func(tx *bbolt.Tx) error { + err := bdw.DB.View(func(tx *bbolt.Tx) error { buck := tx.Bucket([]byte(RepoMetadataBucket)) cursor := buck.Cursor() @@ -675,14 +665,10 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta } if filter(repoMeta) { - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - }) + foundRepos = append(foundRepos, repoMeta) } } - foundRepos, _ = pageFinder.Page() - return nil }) @@ -965,34 +951,21 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di return err } -func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, - error, -) { +func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - foundindexDataMap = make(map[string]mTypes.IndexData) - pageFinder pagination.PageFinder - pageInfo zcommon.PageInfo + foundRepos = make([]mTypes.RepoMetadata, 0) + manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) + indexDataMap = make(map[string]mTypes.IndexData) ) - pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - zcommon.PageInfo{}, err - } - - err = bdw.DB.View(func(transaction *bbolt.Tx) error { + err := bdw.DB.View(func(transaction *bbolt.Tx) error { var ( - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - repoBuck = transaction.Bucket([]byte(RepoMetadataBucket)) - indexBuck = transaction.Bucket([]byte(IndexDataBucket)) - manifestBuck = transaction.Bucket([]byte(ManifestDataBucket)) - userBookmarks = getUserBookmarks(ctx, transaction) - userStars = getUserStars(ctx, transaction) + repoBuck = transaction.Bucket([]byte(RepoMetadataBucket)) + indexBuck = transaction.Bucket([]byte(IndexDataBucket)) + manifestBuck = transaction.Bucket([]byte(ManifestDataBucket)) + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) ) cursor := repoBuck.Cursor() @@ -1009,22 +982,14 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT return err } - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - rank := common.RankRepoName(searchText, repoMeta.Name) if rank == -1 { continue } - var ( - repoDownloads = 0 - repoLastUpdated = time.Time{} - osSet = map[string]bool{} - archSet = map[string]bool{} - noImageChecked = true - isSigned = false - ) + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + repoMeta.Rank = rank for tag, descriptor := range repoMeta.Tags { switch descriptor.MediaType { @@ -1038,24 +1003,6 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT manifestDigest, err) } - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", - manifestDigest, err) - } - - repoDownloads += manifestFilterData.DownloadCount - - for _, os := range manifestFilterData.OsList { - osSet[os] = true - } - for _, arch := range manifestFilterData.ArchList { - archSet[arch] = true - } - - repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, - noImageChecked, manifestFilterData) - manifestMetadataMap[descriptor.Digest] = manifestMeta case ispec.MediaTypeImageIndex: indexDigest := descriptor.Digest @@ -1074,27 +1021,18 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT repoName, tag, err) } - // this also updates manifestMetadataMap - indexFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap, - manifestBuck) - if err != nil { - return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w", - indexDigest, err) - } + for _, manifest := range indexContent.Manifests { + manifestDigest := manifest.Digest - for _, arch := range indexFilterData.ArchList { - archSet[arch] = true - } + manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(), + manifestMetadataMap, manifestBuck) + if err != nil { + return err + } - for _, os := range indexFilterData.OsList { - osSet[os] = true + manifestMetadataMap[manifest.Digest.String()] = manifestMeta } - repoDownloads += indexFilterData.DownloadCount - - repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, - noImageChecked, indexFilterData) - indexDataMap[indexDigest] = indexData default: bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") @@ -1103,37 +1041,13 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT } } - repoFilterData := mTypes.FilterData{ - OsList: common.GetMapKeys(osSet), - ArchList: common.GetMapKeys(archSet), - LastUpdated: repoLastUpdated, - DownloadCount: repoDownloads, - IsSigned: isSigned, - IsBookmarked: repoMeta.IsBookmarked, - IsStarred: repoMeta.IsStarred, - } - - if !common.AcceptedByFilter(filter, repoFilterData) { - continue - } - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - Rank: rank, - Downloads: repoDownloads, - UpdateTime: repoLastUpdated, - }) + foundRepos = append(foundRepos, repoMeta) } - foundRepos, pageInfo = pageFinder.Page() - - foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap, - indexDataMap) - - return err + return nil }) - return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err + return foundRepos, manifestMetadataMap, indexDataMap, err } func fetchManifestMetaWithCheck(repoMeta mTypes.RepoMetadata, manifestDigest string, @@ -1187,95 +1101,6 @@ func fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]mTypes. return indexData, err } -func collectImageManifestFilterData(digest string, repoMeta mTypes.RepoMetadata, - manifestMeta mTypes.ManifestMetadata, -) (mTypes.FilterData, error) { - // get fields related to filtering - var ( - configContent ispec.Image - osList []string - archList []string - ) - - err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent) - if err != nil { - return mTypes.FilterData{}, - fmt.Errorf("metadb: error while unmarshaling config content %w", err) - } - - if configContent.OS != "" { - osList = append(osList, configContent.OS) - } - - if configContent.Architecture != "" { - archList = append(archList, configContent.Architecture) - } - - return mTypes.FilterData{ - DownloadCount: repoMeta.Statistics[digest].DownloadCount, - OsList: osList, - ArchList: archList, - LastUpdated: common.GetImageLastUpdatedTimestamp(configContent), - IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]), - }, nil -} - -func collectImageIndexFilterInfo(indexDigest string, repoMeta mTypes.RepoMetadata, - indexData mTypes.IndexData, manifestMetadataMap map[string]mTypes.ManifestMetadata, - manifestBuck *bbolt.Bucket, -) (mTypes.FilterData, error) { - var indexContent ispec.Index - - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return mTypes.FilterData{}, - fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", indexDigest, err) - } - - var ( - indexLastUpdated time.Time - firstManifestChecked = false - indexOsList = []string{} - indexArchList = []string{} - ) - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(), - manifestMetadataMap, manifestBuck) - if err != nil { - return mTypes.FilterData{}, - fmt.Errorf("%w", err) - } - - manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta, - manifestMeta) - if err != nil { - return mTypes.FilterData{}, - fmt.Errorf("%w", err) - } - - indexOsList = append(indexOsList, manifestFilterData.OsList...) - indexArchList = append(indexArchList, manifestFilterData.ArchList...) - - if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) { - indexLastUpdated = manifestFilterData.LastUpdated - firstManifestChecked = true - } - - manifestMetadataMap[manifest.Digest.String()] = manifestMeta - } - - return mTypes.FilterData{ - DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount, - LastUpdated: indexLastUpdated, - OsList: indexOsList, - ArchList: indexArchList, - IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]), - }, nil -} - func NewManifestMetadata(manifestDigest string, repoMeta mTypes.RepoMetadata, manifestData mTypes.ManifestData, ) mTypes.ManifestMetadata { @@ -1294,28 +1119,16 @@ func NewManifestMetadata(manifestDigest string, repoMeta mTypes.RepoMetadata, return manifestMeta } -func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - zcommon.PageInfo, error, +func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - foundindexDataMap = make(map[string]mTypes.IndexData) - pageFinder pagination.PageFinder - pageInfo zcommon.PageInfo + foundRepos = make([]mTypes.RepoMetadata, 0) + manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) + indexDataMap = make(map[string]mTypes.IndexData) ) - pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - zcommon.PageInfo{}, err - } - - err = bdw.DB.View(func(transaction *bbolt.Tx) error { + err := bdw.DB.View(func(transaction *bbolt.Tx) error { var ( repoBuck = transaction.Bucket([]byte(RepoMetadataBucket)) indexBuck = transaction.Bucket([]byte(IndexDataBucket)) @@ -1354,16 +1167,6 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, return fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err) } - imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", - manifestDigest, err) - } - - if !common.AcceptedByFilter(filter, imageFilterData) { - continue - } - if filterFunc(repoMeta, manifestMeta) { matchedTags[tag] = descriptor manifestMetadataMap[manifestDigest] = manifestMeta @@ -1393,16 +1196,6 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, return fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err) } - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", - manifestDigest, err) - } - - if !common.AcceptedByFilter(filter, manifestFilterData) { - continue - } - if filterFunc(repoMeta, manifestMeta) { matchedManifests = append(matchedManifests, manifest) manifestMetadataMap[manifestDigest] = manifestMeta @@ -1435,44 +1228,21 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, repoMeta.Tags = matchedTags - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - }) + foundRepos = append(foundRepos, repoMeta) } - foundRepos, pageInfo = pageFinder.Page() - - foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap, - indexDataMap) - - return err + return nil }) - return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err + return foundRepos, manifestMetadataMap, indexDataMap, err } -func (bdw *BoltDB) FilterRepos(ctx context.Context, - filter mTypes.FilterRepoFunc, - requestedPage mTypes.PageInput, -) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error, +func (bdw *BoltDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc) ( + []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { - var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - pageFinder pagination.PageFinder - pageInfo zcommon.PageInfo - ) - - pageFinder, err := pagination.NewBaseRepoPageFinder( - requestedPage.Limit, - requestedPage.Offset, - requestedPage.SortBy, - ) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, pageInfo, err - } + foundRepos := make([]mTypes.RepoMetadata, 0) - err = bdw.DB.View(func(tx *bbolt.Tx) error { + err := bdw.DB.View(func(tx *bbolt.Tx) error { var ( buck = tx.Bucket([]byte(RepoMetadataBucket)) cursor = buck.Cursor() @@ -1496,49 +1266,32 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) if filter(repoMeta) { - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - }) + foundRepos = append(foundRepos, repoMeta) } } - foundRepos, pageInfo = pageFinder.Page() - return nil }) if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, pageInfo, err + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, err } foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(bdw, foundRepos) - return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err + return foundRepos, foundManifestMetadataMap, foundIndexDataMap, err } -func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error) { +func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - foundindexDataMap = make(map[string]mTypes.IndexData) - pageInfo zcommon.PageInfo - - pageFinder pagination.PageFinder + foundRepos = make([]mTypes.RepoMetadata, 0) + manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) + indexDataMap = make(map[string]mTypes.IndexData) ) - pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - zcommon.PageInfo{}, err - } - searchedRepo, searchedTag, err := common.GetRepoTag(searchText) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - zcommon.PageInfo{}, fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) } @@ -1547,143 +1300,99 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, filter mTy repoBuck = transaction.Bucket([]byte(RepoMetadataBucket)) indexBuck = transaction.Bucket([]byte(IndexDataBucket)) manifestBuck = transaction.Bucket([]byte(ManifestDataBucket)) - cursor = repoBuck.Cursor() userBookmarks = getUserBookmarks(ctx, transaction) userStars = getUserStars(ctx, transaction) ) - repoName, repoMetaBlob := cursor.Seek([]byte(searchedRepo)) - - for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } + repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo)) - repoMeta := mTypes.RepoMetadata{} + if string(repoName) != searchedRepo { + return nil + } - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } + if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + return err + } - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + repoMeta := mTypes.RepoMetadata{} - if string(repoName) != searchedRepo { - continue - } + err := json.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } - matchedTags := make(map[string]mTypes.Descriptor) + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - for tag, descriptor := range repoMeta.Tags { - if !strings.HasPrefix(tag, searchedTag) { - continue - } + matchedTags := make(map[string]mTypes.Descriptor) - matchedTags[tag] = descriptor + for tag, descriptor := range repoMeta.Tags { + if !strings.HasPrefix(tag, searchedTag) { + continue + } - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest + matchedTags[tag] = descriptor - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) - if err != nil { - return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", - manifestDigest, err) - } + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest - imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", - manifestDigest, err) - } + manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) + if err != nil { + return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", + manifestDigest, err) + } - if !common.AcceptedByFilter(filter, imageFilterData) { - delete(matchedTags, tag) + manifestMetadataMap[descriptor.Digest] = manifestMeta + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest - continue - } + indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) + if err != nil { + return fmt.Errorf("metadb: error fetching index data for index with digest %s %w", + indexDigest, err) + } - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest + var indexContent ispec.Index - indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) - if err != nil { - return fmt.Errorf("metadb: error fetching index data for index with digest %s %w", - indexDigest, err) - } + err = json.Unmarshal(indexData.IndexBlob, &indexContent) + if err != nil { + return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w", + indexDigest, err) + } - var indexContent ispec.Index + for _, manifest := range indexContent.Manifests { + manifestDigest := manifest.Digest.String() - err = json.Unmarshal(indexData.IndexBlob, &indexContent) + manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) if err != nil { - return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w", - indexDigest, err) - } - - manifestHasBeenMatched := false - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest.String() - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) - if err != nil { - return fmt.Errorf("metadb: error fetching from db manifest meta for manifest with digest %s %w", - manifestDigest, err) - } - - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", - manifestDigest, err) - } - - manifestMetadataMap[manifestDigest] = manifestMeta - - if common.AcceptedByFilter(filter, manifestFilterData) { - manifestHasBeenMatched = true - } - } - - if !manifestHasBeenMatched { - delete(matchedTags, tag) - - for _, manifest := range indexContent.Manifests { - delete(manifestMetadataMap, manifest.Digest.String()) - } - - continue + return fmt.Errorf("metadb: error fetching from db manifest meta for manifest with digest %s %w", + manifestDigest, err) } - indexDataMap[indexDigest] = indexData - default: - bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue + manifestMetadataMap[manifestDigest] = manifestMeta } - } - if len(matchedTags) == 0 { + indexDataMap[indexDigest] = indexData + default: + bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") + continue } + } - repoMeta.Tags = matchedTags - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - }) + if len(matchedTags) == 0 { + return nil } - foundRepos, pageInfo = pageFinder.Page() + repoMeta.Tags = matchedTags - foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap, - indexDataMap) + foundRepos = append(foundRepos, repoMeta) return nil }) - return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err + return foundRepos, manifestMetadataMap, indexDataMap, err } func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.ToggleState, error) { diff --git a/pkg/meta/boltdb/boltdb_test.go b/pkg/meta/boltdb/boltdb_test.go index 79fbc84614..c529a30ab8 100644 --- a/pkg/meta/boltdb/boltdb_test.go +++ b/pkg/meta/boltdb/boltdb_test.go @@ -314,8 +314,8 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.FilterRepos(context.Background(), - func(repoMeta mTypes.RepoMetadata) bool { return true }, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.FilterRepos(context.Background(), + func(repoMeta mTypes.RepoMetadata) bool { return true }) So(err, ShouldNotBeNil) }) @@ -453,7 +453,7 @@ func TestWrapperErrors(t *testing.T) { _, err = boltdbWrapper.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool { return true - }, mTypes.PageInput{}) + }) So(err, ShouldNotBeNil) }) @@ -619,7 +619,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "") So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { @@ -660,47 +660,10 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1") So(err, ShouldNotBeNil) - _, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2", mTypes.Filter{}, mTypes.PageInput{}) - So(err, ShouldNotBeNil) - - err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - dataBuck := tx.Bucket([]byte(boltdb.ManifestDataBucket)) - - manifestMeta := mTypes.ManifestMetadata{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("wrong json"), - Signatures: mTypes.ManifestSignatures{}, - } - - manifestMetaBlob, err := json.Marshal(manifestMeta) - if err != nil { - return err - } - - err = dataBuck.Put([]byte("dig1"), manifestMetaBlob) - if err != nil { - return err - } - - repoMeta = mTypes.RepoMetadata{ - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, - }, - Signatures: map[string]mTypes.ManifestSignatures{}, - } - repoMetaBlob, err = json.Marshal(repoMeta) - So(err, ShouldBeNil) - - return repoBuck.Put([]byte("repo1"), repoMetaBlob) - }) - So(err, ShouldBeNil) - - _, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2") So(err, ShouldNotBeNil) }) @@ -714,10 +677,10 @@ func TestWrapperErrors(t *testing.T) { err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchRepos(ctx, "") So(err, ShouldNotBeNil) - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:") So(err, ShouldNotBeNil) }) @@ -732,49 +695,10 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) - So(err, ShouldNotBeNil) - - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) - So(err, ShouldNotBeNil) - }) - - Convey("Good index data, bad manifest inside index", func() { - var ( - indexDigest = digest.FromString("indexDigest") - manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1") - manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") - ) - - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ - manifestDigestFromIndex1, manifestDigestFromIndex2, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) - So(err, ShouldBeNil) - - _, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchRepos(ctx, "") So(err, ShouldNotBeNil) - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:") So(err, ShouldNotBeNil) }) }) @@ -789,10 +713,10 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchTags(ctx, "") So(err, ShouldNotBeNil) - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:") So(err, ShouldNotBeNil) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { @@ -866,13 +790,10 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:") So(err, ShouldNotBeNil) - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:", mTypes.Filter{}, mTypes.PageInput{}) - So(err, ShouldNotBeNil) - - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo3:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:") So(err, ShouldNotBeNil) }) @@ -886,11 +807,8 @@ func TestWrapperErrors(t *testing.T) { err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + _, _, _, err = boltdbWrapper.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) So(err, ShouldNotBeNil) }) @@ -905,11 +823,8 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + _, _, _, err = boltdbWrapper.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) So(err, ShouldNotBeNil) }) @@ -945,76 +860,9 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) - So(err, ShouldBeNil) - }) - - Convey("FilterTags bad config blob in image with a single manifest", func() { - manifestDigest := digest.FromString("manifestDigestBadConfig") - - err := boltdbWrapper.SetRepoReference("repo", "tag1", //nolint:contextcheck - manifestDigest, ispec.MediaTypeImageManifest, - ) + _, _, _, err = boltdbWrapper.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false }) So(err, ShouldBeNil) - - err = boltdbWrapper.SetManifestData(manifestDigest, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("bad blob"), - }) - So(err, ShouldBeNil) - - _, _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags bad config blob in index", func() { - var ( - indexDigest = digest.FromString("indexDigest") - manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndexGoodConfig") - manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndexBadConfig") - ) - - err := boltdbWrapper.SetRepoReference("repo", "tag1", //nolint:contextcheck - indexDigest, ispec.MediaTypeImageIndex, - ) - So(err, ShouldBeNil) - - indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ - manifestDigestFromIndex1, manifestDigestFromIndex2, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("bad blob"), - }) - So(err, ShouldBeNil) - - _, _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) - So(err, ShouldNotBeNil) }) }) @@ -1161,18 +1009,14 @@ func TestWrapperErrors(t *testing.T) { err := boltdbWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchRepos(ctx, "") So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:") So(err, ShouldBeNil) - _, _, _, _, err = boltdbWrapper.FilterTags( - ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + _, _, _, err = boltdbWrapper.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) So(err, ShouldBeNil) }) diff --git a/pkg/meta/common/common.go b/pkg/meta/common/common.go index 763cdbfcd9..155d942927 100644 --- a/pkg/meta/common/common.go +++ b/pkg/meta/common/common.go @@ -142,34 +142,6 @@ func RankRepoName(searchText string, repoName string) int { return -1 } -func GetImageLastUpdatedTimestamp(configContent ispec.Image) time.Time { - var timeStamp *time.Time - - if configContent.Created != nil && !configContent.Created.IsZero() { - return *configContent.Created - } - - if len(configContent.History) != 0 { - timeStamp = configContent.History[len(configContent.History)-1].Created - } - - if timeStamp == nil { - timeStamp = &time.Time{} - } - - return *timeStamp -} - -func CheckIsSigned(signatures mTypes.ManifestSignatures) bool { - for _, signatures := range signatures { - if len(signatures) > 0 { - return true - } - } - - return false -} - func GetRepoTag(searchText string) (string, string, error) { const repoTagCount = 2 @@ -185,66 +157,6 @@ func GetRepoTag(searchText string) (string, string, error) { return repo, tag, nil } -func GetMapKeys[K comparable, V any](genericMap map[K]V) []K { - keys := make([]K, 0, len(genericMap)) - - for k := range genericMap { - keys = append(keys, k) - } - - return keys -} - -// acceptedByFilter checks that data contains at least 1 element of each filter -// criteria(os, arch) present in filter. -func AcceptedByFilter(filter mTypes.Filter, data mTypes.FilterData) bool { - if filter.Arch != nil { - foundArch := false - for _, arch := range filter.Arch { - foundArch = foundArch || containsString(data.ArchList, *arch) - } - - if !foundArch { - return false - } - } - - if filter.Os != nil { - foundOs := false - for _, os := range filter.Os { - foundOs = foundOs || containsString(data.OsList, *os) - } - - if !foundOs { - return false - } - } - - if filter.HasToBeSigned != nil && *filter.HasToBeSigned != data.IsSigned { - return false - } - - if filter.IsBookmarked != nil && *filter.IsBookmarked != data.IsBookmarked { - return false - } - - if filter.IsStarred != nil && *filter.IsStarred != data.IsStarred { - return false - } - - return true -} - -func containsString(strSlice []string, str string) bool { - for _, val := range strSlice { - if strings.EqualFold(val, str) { - return true - } - } - - return false -} - func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) { var manifest ispec.Manifest diff --git a/pkg/meta/common/common_test.go b/pkg/meta/common/common_test.go index 26a7524a08..6c7c9efe94 100644 --- a/pkg/meta/common/common_test.go +++ b/pkg/meta/common/common_test.go @@ -118,6 +118,39 @@ func TestUtils(t *testing.T) { }) Convey("FilterDataByRepo", t, func() { + Convey("Functionality", func() { + _, _, err := common.FilterDataByRepo( + []mTypes.RepoMetadata{{ + Tags: map[string]mTypes.Descriptor{ + "manifest": { + Digest: "manifestDigest", + MediaType: ispec.MediaTypeImageManifest, + }, + "index": { + Digest: "indexDigest", + MediaType: ispec.MediaTypeImageIndex, + }, + "rand": { + Digest: "randDigest", + MediaType: "rand", + }, + }, + }}, + map[string]mTypes.ManifestMetadata{}, + map[string]mTypes.IndexData{ + "indexDigest": { + IndexBlob: []byte(`{ + "manifests": [ + { + "digest": "manifestDigest" + } + ] + }`), + }, + }, + ) + So(err, ShouldBeNil) + }) Convey("Errors", func() { // Unmarshal index data error _, _, err := common.FilterDataByRepo( diff --git a/pkg/meta/dynamodb/dynamo.go b/pkg/meta/dynamodb/dynamo.go index 67c78f48a0..698ef61241 100644 --- a/pkg/meta/dynamodb/dynamo.go +++ b/pkg/meta/dynamodb/dynamo.go @@ -19,7 +19,6 @@ import ( zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/common" - "zotregistry.io/zot/pkg/meta/pagination" "zotregistry.io/zot/pkg/meta/signatures" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/meta/version" @@ -787,27 +786,21 @@ func (dwr *DynamoDB) DeleteSignature(repo string, signedManifestDigest godigest. } func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context, - filter func(repoMeta mTypes.RepoMetadata) bool, requestedPage mTypes.PageInput, + filter func(repoMeta mTypes.RepoMetadata) bool, ) ([]mTypes.RepoMetadata, error) { var ( + foundRepos = []mTypes.RepoMetadata{} repoMetaAttributeIterator AttributesIterator - pageFinder pagination.PageFinder ) repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) - pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return nil, err - } - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { if err != nil { - // log return []mTypes.RepoMetadata{}, err } @@ -823,26 +816,20 @@ func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context, } if filter(repoMeta) { - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - }) + foundRepos = append(foundRepos, repoMeta) } } - foundRepos, _ := pageFinder.Page() - return foundRepos, err } -func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error) { +func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { var ( + repos = []mTypes.RepoMetadata{} manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) indexDataMap = make(map[string]mTypes.IndexData) repoMetaAttributeIterator AttributesIterator - pageFinder pagination.PageFinder - pageInfo zcommon.PageInfo userBookmarks = getUserBookmarks(ctx, dwr) userStars = getUserStars(ctx, dwr) @@ -852,18 +839,12 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) - pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err - } - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err + err } var repoMeta mTypes.RepoMetadata @@ -871,7 +852,7 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err + err } if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { @@ -885,15 +866,7 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - - var ( - repoDownloads = 0 - repoLastUpdated = time.Time{} - osSet = map[string]bool{} - archSet = map[string]bool{} - noImageChecked = true - isSigned = false - ) + repoMeta.Rank = rank for _, descriptor := range repoMeta.Tags { switch descriptor.MediaType { @@ -904,64 +877,37 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter manifestMetadataMap) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) + err } - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } - - repoDownloads += manifestFilterData.DownloadCount - - for _, os := range manifestFilterData.OsList { - osSet[os] = true - } - - for _, arch := range manifestFilterData.ArchList { - archSet[arch] = true - } - - repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, - noImageChecked, manifestFilterData) - manifestMetadataMap[descriptor.Digest] = manifestMeta case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck + indexData, err := dwr.fetchIndexDataWithCheck(descriptor.Digest, indexDataMap) //nolint:contextcheck if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) + err } - // this also updates manifestMetadataMap - indexFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck - manifestMetadataMap) + var indexContent ispec.Index + + err = json.Unmarshal(indexData.IndexBlob, &indexContent) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) + fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", descriptor.Digest, err) } - for _, arch := range indexFilterData.ArchList { - archSet[arch] = true - } + for _, manifest := range indexContent.Manifests { + manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifest.Digest.String(), //nolint: contextcheck + manifestMetadataMap) + if err != nil { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, + err + } - for _, os := range indexFilterData.OsList { - osSet[os] = true + manifestMetadataMap[manifest.Digest.String()] = manifestMeta } - repoDownloads += indexFilterData.DownloadCount - - repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, - noImageChecked, indexFilterData) - - indexDataMap[indexDigest] = indexData + indexDataMap[descriptor.Digest] = indexData default: dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") @@ -969,32 +915,10 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter } } - repoFilterData := mTypes.FilterData{ - OsList: common.GetMapKeys(osSet), - ArchList: common.GetMapKeys(archSet), - LastUpdated: repoLastUpdated, - DownloadCount: repoDownloads, - IsSigned: isSigned, - } - - if !common.AcceptedByFilter(filter, repoFilterData) { - continue - } - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - Rank: rank, - Downloads: repoDownloads, - UpdateTime: repoLastUpdated, - }) + repos = append(repos, repoMeta) } - foundRepos, pageInfo := pageFinder.Page() - - foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap, - indexDataMap) - - return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err + return repos, manifestMetadataMap, indexDataMap, nil } func getUserStars(ctx context.Context, dwr *DynamoDB) []string { @@ -1035,38 +959,6 @@ func (dwr *DynamoDB) fetchManifestMetaWithCheck(repoName string, manifestDigest return manifestMeta, nil } -func collectImageManifestFilterData(digest string, repoMeta mTypes.RepoMetadata, - manifestMeta mTypes.ManifestMetadata, -) (mTypes.FilterData, error) { - // get fields related to filtering - var ( - configContent ispec.Image - osList []string - archList []string - ) - - err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent) - if err != nil { - return mTypes.FilterData{}, fmt.Errorf("metadb: error while unmarshaling config content %w", err) - } - - if configContent.OS != "" { - osList = append(osList, configContent.OS) - } - - if configContent.Architecture != "" { - archList = append(archList, configContent.Architecture) - } - - return mTypes.FilterData{ - DownloadCount: repoMeta.Statistics[digest].DownloadCount, - OsList: osList, - ArchList: archList, - LastUpdated: common.GetImageLastUpdatedTimestamp(configContent), - IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]), - }, nil -} - func (dwr *DynamoDB) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]mTypes.IndexData, ) (mTypes.IndexData, error) { var ( @@ -1087,72 +979,14 @@ func (dwr *DynamoDB) fetchIndexDataWithCheck(indexDigest string, indexDataMap ma return indexData, err } -func (dwr *DynamoDB) collectImageIndexFilterInfo(indexDigest string, repoMeta mTypes.RepoMetadata, - indexData mTypes.IndexData, manifestMetadataMap map[string]mTypes.ManifestMetadata, -) (mTypes.FilterData, error) { - var indexContent ispec.Index - - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return mTypes.FilterData{}, - fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", indexDigest, err) - } - - var ( - indexLastUpdated time.Time - firstManifestChecked = false - indexOsList = []string{} - indexArchList = []string{} - ) - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest - - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest.String(), - manifestMetadataMap) - if err != nil { - return mTypes.FilterData{}, - fmt.Errorf("%w", err) - } - - manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta, - manifestMeta) - if err != nil { - return mTypes.FilterData{}, - fmt.Errorf("%w", err) - } - - indexOsList = append(indexOsList, manifestFilterData.OsList...) - indexArchList = append(indexArchList, manifestFilterData.ArchList...) - - if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) { - indexLastUpdated = manifestFilterData.LastUpdated - firstManifestChecked = true - } - - manifestMetadataMap[manifest.Digest.String()] = manifestMeta - } - - return mTypes.FilterData{ - DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount, - LastUpdated: indexLastUpdated, - OsList: indexOsList, - ArchList: indexArchList, - IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]), - }, nil -} - -func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - zcommon.PageInfo, error, +func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, ) { var ( + foundRepos = make([]mTypes.RepoMetadata, 0) manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) indexDataMap = make(map[string]mTypes.IndexData) repoMetaAttributeIterator AttributesIterator - pageFinder pagination.PageFinder - pageInfo zcommon.PageInfo userBookmarks = getUserBookmarks(ctx, dwr) userStars = getUserStars(ctx, dwr) ) @@ -1161,18 +995,12 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) - pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err - } - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err + err } var repoMeta mTypes.RepoMetadata @@ -1180,7 +1008,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err + err } if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { @@ -1201,23 +1029,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun manifestMetadataMap) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s \n%w", manifestDigest, err) } - imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", manifestDigest, err) - } - - if !common.AcceptedByFilter(filter, imageFilterData) { - delete(matchedTags, tag) - - continue - } - if filterFunc(repoMeta, manifestMeta) { matchedTags[tag] = descriptor manifestMetadataMap[manifestDigest] = manifestMeta @@ -1228,7 +1042,6 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err) } @@ -1237,7 +1050,6 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun err = json.Unmarshal(indexData.IndexBlob, &indexContent) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) } @@ -1250,21 +1062,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun manifestMetadataMap) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, fmt.Errorf("%w metadb: error while getting manifest data for digest %s", err, manifestDigest) } - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", manifestDigest, err) - } - - if !common.AcceptedByFilter(filter, manifestFilterData) { - continue - } - if filterFunc(repoMeta, manifestMeta) { matchedManifests = append(matchedManifests, manifest) manifestMetadataMap[manifestDigest] = manifestMeta @@ -1277,7 +1077,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun indexBlob, err := json.Marshal(indexContent) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err + err } indexData.IndexBlob = indexBlob @@ -1298,29 +1098,17 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun repoMeta.Tags = matchedTags - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - }) + foundRepos = append(foundRepos, repoMeta) } - foundRepos, pageInfo := pageFinder.Page() - - foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap, - indexDataMap) - - return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err + return foundRepos, manifestMetadataMap, indexDataMap, err } -func (dwr *DynamoDB) FilterRepos(ctx context.Context, - filter mTypes.FilterRepoFunc, - requestedPage mTypes.PageInput, -) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - zcommon.PageInfo, error, -) { +func (dwr *DynamoDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { var ( + foundRepos = []mTypes.RepoMetadata{} repoMetaAttributeIterator AttributesIterator - pageInfo zcommon.PageInfo userBookmarks = getUserBookmarks(ctx, dwr) userStars = getUserStars(ctx, dwr) ) @@ -1329,22 +1117,16 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context, dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) - pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err - } - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err + err } for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err + err } var repoMeta mTypes.RepoMetadata @@ -1352,7 +1134,7 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context, err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err + err } if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { @@ -1363,40 +1145,28 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context, repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) if filter(repoMeta) { - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - }) + foundRepos = append(foundRepos, repoMeta) } } - foundRepos, pageInfo := pageFinder.Page() - foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(dwr, foundRepos) - return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err + return foundRepos, foundManifestMetadataMap, foundIndexDataMap, err } -func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, +func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string, ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - zcommon.PageInfo, error, + error, ) { var ( + foundRepos = make([]mTypes.RepoMetadata, 0, 1) manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) indexDataMap = make(map[string]mTypes.IndexData) repoMetaAttributeIterator AttributesIterator - pageFinder pagination.PageFinder - pageInfo zcommon.PageInfo userBookmarks = getUserBookmarks(ctx, dwr) userStars = getUserStars(ctx, dwr) ) - pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err - } - repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) @@ -1404,154 +1174,109 @@ func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string, filter m searchedRepo, searchedTag, err := common.GetRepoTag(searchText) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) } repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) + if err != nil { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, + err + } - for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { - if err != nil { - // log - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err - } + var repoMeta mTypes.RepoMetadata - var repoMeta mTypes.RepoMetadata + err = attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) + if err != nil { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, + err + } - err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, err - } + if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, + err + } - if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { - continue - } + if repoMeta.Name != searchedRepo { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, + err + } + + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + + matchedTags := make(map[string]mTypes.Descriptor) - if repoMeta.Name != searchedRepo { + for tag, descriptor := range repoMeta.Tags { + if !strings.HasPrefix(tag, searchedTag) { continue } - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + matchedTags[tag] = descriptor - matchedTags := make(map[string]mTypes.Descriptor) + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest - for tag, descriptor := range repoMeta.Tags { - if !strings.HasPrefix(tag, searchedTag) { - continue - } - - matchedTags[tag] = descriptor + manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck + manifestMetadataMap) + if err != nil { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest + fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err) + } - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err) - } + manifestMetadataMap[descriptor.Digest] = manifestMeta + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest - imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } + indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck + if err != nil { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - if !common.AcceptedByFilter(filter, imageFilterData) { - delete(matchedTags, tag) + fmt.Errorf("%w", err) + } - continue - } + var indexContent ispec.Index - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest + err = json.Unmarshal(indexData.IndexBlob, &indexContent) + if err != nil { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } + fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) + } - var indexContent ispec.Index + for _, manifest := range indexContent.Manifests { + manifestDigest := manifest.Digest.String() - err = json.Unmarshal(indexData.IndexBlob, &indexContent) + manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck + manifestMetadataMap) if err != nil { return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) - } - manifestHasBeenMatched := false - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest.String() - - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } - - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } - - manifestMetadataMap[manifestDigest] = manifestMeta - - if common.AcceptedByFilter(filter, manifestFilterData) { - manifestHasBeenMatched = true - } - } - - if !manifestHasBeenMatched { - delete(matchedTags, tag) - - for _, manifest := range indexContent.Manifests { - delete(manifestMetadataMap, manifest.Digest.String()) - } - - continue + fmt.Errorf("%w", err) } - indexDataMap[indexDigest] = indexData - default: - dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue + manifestMetadataMap[manifestDigest] = manifestMeta } - } - if len(matchedTags) == 0 { + indexDataMap[indexDigest] = indexData + default: + dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") + continue } + } - repoMeta.Tags = matchedTags - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: repoMeta, - }) + if len(matchedTags) == 0 { + return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, + err } - foundRepos, pageInfo := pageFinder.Page() + repoMeta.Tags = matchedTags - foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap, - indexDataMap) + foundRepos = append(foundRepos, repoMeta) - return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err + return foundRepos, manifestMetadataMap, indexDataMap, err } func (dwr *DynamoDB) PatchDB() error { diff --git a/pkg/meta/dynamodb/dynamo_test.go b/pkg/meta/dynamodb/dynamo_test.go index 847c547262..f9adf596d0 100644 --- a/pkg/meta/dynamodb/dynamo_test.go +++ b/pkg/meta/dynamodb/dynamo_test.go @@ -823,8 +823,7 @@ func TestWrapperErrors(t *testing.T) { err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck So(err, ShouldBeNil) - _, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }, - mTypes.PageInput{}) + _, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }) So(err, ShouldNotBeNil) }) @@ -833,7 +832,7 @@ func TestWrapperErrors(t *testing.T) { err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") So(err, ShouldNotBeNil) }) @@ -843,22 +842,7 @@ func TestWrapperErrors(t *testing.T) { ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) - - So(err, ShouldNotBeNil) - }) - - Convey("SearchRepos config unmarshal error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData("dig1", mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") So(err, ShouldNotBeNil) }) @@ -869,18 +853,14 @@ func TestWrapperErrors(t *testing.T) { err := dynamoWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.FilterTags( - ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + _, _, _, err = dynamoWrapper.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) So(err, ShouldBeNil) }) @@ -893,7 +873,7 @@ func TestWrapperErrors(t *testing.T) { err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") So(err, ShouldNotBeNil) }) @@ -908,43 +888,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) - So(err, ShouldNotBeNil) - }) - - Convey("SearchRepos good index data, bad manifest inside index", func() { - var ( - indexDigest = digest.FromString("indexDigest") - manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1") - manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") - ) - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ - manifestDigestFromIndex1, manifestDigestFromIndex2, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) - So(err, ShouldBeNil) - - _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") So(err, ShouldNotBeNil) }) @@ -952,7 +896,7 @@ func TestWrapperErrors(t *testing.T) { err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") So(err, ShouldNotBeNil) }) @@ -962,25 +906,7 @@ func TestWrapperErrors(t *testing.T) { ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) - - So(err, ShouldNotBeNil) - }) - - Convey("SearchTags config unmarshal error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData( //nolint:contextcheck - "dig1", - mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("bad json"), - }, - ) - So(err, ShouldBeNil) - - _, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") So(err, ShouldNotBeNil) }) @@ -994,7 +920,7 @@ func TestWrapperErrors(t *testing.T) { err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") So(err, ShouldNotBeNil) }) @@ -1009,49 +935,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) - So(err, ShouldNotBeNil) - }) - - Convey("SearchTags good index data, bad manifest inside index", func() { - var ( - indexDigest = digest.FromString("indexDigest") - manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1") - manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") - ) - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ - manifestDigestFromIndex1, manifestDigestFromIndex2, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) - So(err, ShouldBeNil) - - _, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{}) - So(err, ShouldNotBeNil) - }) - - Convey("FilterRepos NewBaseRepoPageFinder errors", func() { - _, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "text", mTypes.Filter{}, - mTypes.PageInput{Offset: -2, Limit: -2}) + _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") So(err, ShouldNotBeNil) }) @@ -1059,7 +943,7 @@ func TestWrapperErrors(t *testing.T) { err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{}) + _, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo") So(err, ShouldNotBeNil) }) @@ -1067,14 +951,10 @@ func TestWrapperErrors(t *testing.T) { err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.FilterTags( - ctx, + _, _, _, err = dynamoWrapper.FilterTags(ctx, func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true - }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + }) So(err, ShouldNotBeNil) }) @@ -1084,14 +964,10 @@ func TestWrapperErrors(t *testing.T) { ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.FilterTags( - ctx, + _, _, _, err = dynamoWrapper.FilterTags(ctx, func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true - }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + }) So(err, ShouldNotBeNil) }) @@ -1103,14 +979,11 @@ func TestWrapperErrors(t *testing.T) { err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.FilterTags( + _, _, _, err = dynamoWrapper.FilterTags( ctx, func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true - }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + }) So(err, ShouldNotBeNil) }) @@ -1124,11 +997,8 @@ func TestWrapperErrors(t *testing.T) { err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + _, _, _, err = dynamoWrapper.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) So(err, ShouldNotBeNil) }) @@ -1143,11 +1013,8 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + _, _, _, err = dynamoWrapper.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) So(err, ShouldNotBeNil) }) @@ -1183,76 +1050,9 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) - So(err, ShouldBeNil) - }) - - Convey("FilterTags bad config blob in image with a single manifest", func() { - manifestDigest := digest.FromString("manifestDigestBadConfig") - - err := dynamoWrapper.SetRepoReference( //nolint:contextcheck - "repo", "tag1", manifestDigest, ispec.MediaTypeImageManifest, - ) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigest, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("bad blob"), - }) - So(err, ShouldBeNil) - - _, _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags bad config blob in index", func() { - var ( - indexDigest = digest.FromString("indexDigest") - manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndexGoodConfig") - manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndexBadConfig") - ) - - err := dynamoWrapper.SetRepoReference( //nolint:contextcheck - "repo", "tag1", indexDigest, ispec.MediaTypeImageIndex, - ) - So(err, ShouldBeNil) - - indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ - manifestDigestFromIndex1, manifestDigestFromIndex2, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("bad blob"), - }) + _, _, _, err = dynamoWrapper.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false }) So(err, ShouldBeNil) - - _, _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) - So(err, ShouldNotBeNil) }) Convey("PatchDB dwr.getDBVersion errors", func() { diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index c06f19fb2d..59f70d474d 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -3,12 +3,9 @@ package meta_test import ( "context" "encoding/json" - "fmt" "math/rand" "os" "path" - "strconv" - "strings" "testing" "time" @@ -520,7 +517,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func Convey("Get all Repometa", func() { repoMetaSlice, err := metaDB.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool { return true - }, mTypes.PageInput{}) + }) So(err, ShouldBeNil) So(len(repoMetaSlice), ShouldEqual, 2) }) @@ -534,26 +531,11 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func } return false - }, mTypes.PageInput{}) + }) So(err, ShouldBeNil) So(len(repoMetaSlice), ShouldEqual, 1) So(repoMetaSlice[0].Tags[tag1].Digest == manifestDigest1.String(), ShouldBeTrue) }) - - Convey("Wrong page input", func() { - repoMetaSlice, err := metaDB.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool { - for tag := range repoMeta.Tags { - if tag == tag1 { - return true - } - } - - return false - }, mTypes.PageInput{Limit: -1, Offset: -1}) - - So(err, ShouldNotBeNil) - So(len(repoMetaSlice), ShouldEqual, 0) - }) }) Convey("Test IncrementRepoStars", func() { @@ -1388,7 +1370,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetManifestMeta(repo1, manifestDigest3, emptyRepoMeta) So(err, ShouldBeNil) - repos, manifestMetaMap, _, _, err := metaDB.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 2) So(len(manifestMetaMap), ShouldEqual, 3) @@ -1404,7 +1386,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) So(err, ShouldBeNil) - repos, manifestMetaMap, _, _, err := metaDB.SearchRepos(ctx, repo1, mTypes.Filter{}, mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, repo1) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) So(len(manifestMetaMap), ShouldEqual, 1) @@ -1418,8 +1400,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - repos, manifestMetaMap, _, _, err := metaDB.SearchRepos(ctx, "RepoThatDoesntExist", mTypes.Filter{}, - mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "RepoThatDoesntExist") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) So(len(manifestMetaMap), ShouldEqual, 0) @@ -1440,7 +1421,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetManifestMeta("golang", manifestDigest3, emptyRepoMeta) So(err, ShouldBeNil) - repos, manifestMetaMap, _, _, err := metaDB.SearchRepos(ctx, "pine", mTypes.Filter{}, mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "pine") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 2) So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) @@ -1463,7 +1444,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetManifestMeta(repo3, manifestDigest1, emptyRepoMeta) So(err, ShouldBeNil) - repos, manifestMetaMap, _, _, err := metaDB.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 3) So(len(manifestMetaMap), ShouldEqual, 1) @@ -1494,7 +1475,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func authzCtxKey := localCtx.GetContextKey() ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) - repos, _, _, _, err := metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{}) + repos, _, _, err := metaDB.SearchRepos(ctx, "repo") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 2) for _, k := range repos { @@ -1502,159 +1483,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func } }) - Convey("Search paginated repos", func() { - reposCount := 50 - repoNameBuilder := strings.Builder{} - - for _, i := range rand.Perm(reposCount) { - manifestDigest := godigest.FromString("fakeManifest" + strconv.Itoa(i)) - timeString := fmt.Sprintf("1%02d0-01-01 04:35", i) - createdTime, err := time.Parse("2006-01-02 15:04", timeString) - So(err, ShouldBeNil) - - configContent := ispec.Image{ - History: []ispec.History{ - { - Created: &createdTime, - }, - }, - } - - configBlob, err := json.Marshal(configContent) - So(err, ShouldBeNil) - - manifestMeta := mTypes.ManifestMetadata{ - ManifestBlob: emptyManifestBlob, - ConfigBlob: configBlob, - DownloadCount: i, - } - repoName := "repo" + strconv.Itoa(i) - - err = metaDB.SetRepoReference(repoName, tag1, manifestDigest, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repoName, manifestDigest, manifestMeta) - So(err, ShouldBeNil) - - repoNameBuilder.Reset() - } - - repos, _, _, _, err := metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, reposCount) - - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 20, - SortBy: mTypes.AlphabeticAsc, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 20) - - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 0, - SortBy: mTypes.AlphabeticAsc, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo0") - - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 1, - SortBy: mTypes.AlphabeticAsc, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo1") - - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 49, - SortBy: mTypes.AlphabeticAsc, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo9") - - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 49, - SortBy: mTypes.AlphabeticDsc, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo0") - - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 0, - SortBy: mTypes.AlphabeticDsc, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo9") - - // sort by downloads - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 0, - SortBy: mTypes.Downloads, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo49") - - // sort by last update - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 0, - SortBy: mTypes.UpdateTime, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo49") - - repos, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 100, - SortBy: mTypes.UpdateTime, - }) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) - So(repos, ShouldBeEmpty) - }) - - Convey("Search with wrong pagination input", func() { - _, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 100, - SortBy: mTypes.UpdateTime, - }) - So(err, ShouldBeNil) - - _, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: -1, - Offset: 100, - SortBy: mTypes.UpdateTime, - }) - So(err, ShouldNotBeNil) - - _, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: -1, - SortBy: mTypes.UpdateTime, - }) - So(err, ShouldNotBeNil) - - _, _, _, _, err = metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 1, - SortBy: mTypes.SortCriteria("InvalidSortingCriteria"), - }) - So(err, ShouldNotBeNil) - }) - Convey("Search Repos with Indexes", func() { var ( tag4 = "0.0.4" @@ -1719,7 +1547,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - repos, manifestMetaMap, indexDataMap, _, err := metaDB.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{}) + repos, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, "repo") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -1779,8 +1607,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) Convey("With exact match", func() { - repos, manifestMetaMap, _, _, err := metaDB.SearchTags(ctx, "repo1:0.0.1", mTypes.Filter{}, - mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchTags(ctx, "repo1:0.0.1") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) So(len(repos[0].Tags), ShouldEqual, 1) @@ -1789,15 +1616,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("With partial repo path", func() { - repos, manifestMetaMap, _, _, err := metaDB.SearchTags(ctx, "repo:0.0.1", mTypes.Filter{}, - mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchTags(ctx, "repo:0.0.1") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) So(len(manifestMetaMap), ShouldEqual, 0) }) Convey("With partial tag", func() { - repos, manifestMetaMap, _, _, err := metaDB.SearchTags(ctx, "repo1:0.0", mTypes.Filter{}, mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchTags(ctx, "repo1:0.0") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) So(len(repos[0].Tags), ShouldEqual, 2) @@ -1806,7 +1632,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) - repos, manifestMetaMap, _, _, err = metaDB.SearchTags(ctx, "repo1:0.", mTypes.Filter{}, mTypes.PageInput{}) + repos, manifestMetaMap, _, err = metaDB.SearchTags(ctx, "repo1:0.") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) So(len(repos[0].Tags), ShouldEqual, 3) @@ -1819,7 +1645,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("With bad query", func() { - repos, manifestMetaMap, _, _, err := metaDB.SearchTags(ctx, "repo:0.0.1:test", mTypes.Filter{}, mTypes.PageInput{}) + repos, manifestMetaMap, _, err := metaDB.SearchTags(ctx, "repo:0.0.1:test") So(err, ShouldNotBeNil) So(len(repos), ShouldEqual, 0) So(len(manifestMetaMap), ShouldEqual, 0) @@ -1864,24 +1690,16 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func authzCtxKey := localCtx.GetContextKey() ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) - repos, _, _, _, err := metaDB.SearchTags(ctx, "repo1:", mTypes.Filter{}, mTypes.PageInput{}) + repos, _, _, err := metaDB.SearchTags(ctx, "repo1:") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) So(repos[0].Name, ShouldResemble, repo1) - repos, _, _, _, err = metaDB.SearchTags(ctx, "repo2:", mTypes.Filter{}, mTypes.PageInput{}) + repos, _, _, err = metaDB.SearchTags(ctx, "repo2:") So(err, ShouldBeNil) So(repos, ShouldBeEmpty) }) - Convey("With wrong pagination input", func() { - repos, _, _, _, err := metaDB.SearchTags(ctx, "repo2:", mTypes.Filter{}, mTypes.PageInput{ - Limit: -1, - }) - So(err, ShouldNotBeNil) - So(repos, ShouldBeEmpty) - }) - Convey("Search Tags with Indexes", func() { var ( tag4 = "0.0.4" @@ -1951,8 +1769,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference("repo", tag6, manifestDigest4, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - repos, manifestMetaMap, indexDataMap, _, err := metaDB.SearchTags(ctx, "repo:0.0", mTypes.Filter{}, - mTypes.PageInput{}) + repos, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, "repo:0.0") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -1997,11 +1814,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob}) So(err, ShouldBeNil) - repos, _, _, _, err := metaDB.SearchTags(context.TODO(), "repo1:", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 0, - SortBy: mTypes.AlphabeticAsc, - }) + repos, _, _, err := metaDB.SearchTags(context.TODO(), "repo1:") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -2010,11 +1823,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func keys = append(keys, k) } - repos, _, _, _, err = metaDB.SearchTags(context.TODO(), "repo1:", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 1, - SortBy: mTypes.AlphabeticAsc, - }) + repos, _, _, err = metaDB.SearchTags(context.TODO(), "repo1:") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -2022,11 +1831,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func keys = append(keys, k) } - repos, _, _, _, err = metaDB.SearchTags(context.TODO(), "repo1:", mTypes.Filter{}, mTypes.PageInput{ - Limit: 1, - Offset: 2, - SortBy: mTypes.AlphabeticAsc, - }) + repos, _, _, err = metaDB.SearchTags(context.TODO(), "repo1:") So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -2039,238 +1844,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(keys, ShouldContain, tag3) }) - Convey("Test repo search with filtering", func() { - var ( - repo1 = "repo1" - repo2 = "repo2" - repo3 = "repo3" - repo4 = "repo4" - tag1 = "0.0.1" - tag2 = "0.0.2" - manifestDigest1 = godigest.FromString("fake-manifest1") - manifestDigest2 = godigest.FromString("fake-manifest2") - manifestDigest3 = godigest.FromString("fake-manifest3") - ) - - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - config1 := ispec.Image{ - Platform: ispec.Platform{ - Architecture: AMD, - OS: LINUX, - }, - } - configBlob1, err := json.Marshal(config1) - So(err, ShouldBeNil) - - config2 := ispec.Image{ - Platform: ispec.Platform{ - Architecture: "arch", - OS: WINDOWS, - }, - } - configBlob2, err := json.Marshal(config2) - So(err, ShouldBeNil) - - config3 := ispec.Image{} - configBlob3, err := json.Marshal(config3) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob1}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest2, mTypes.ManifestMetadata{ConfigBlob: configBlob2}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo2, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob1}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo3, manifestDigest2, mTypes.ManifestMetadata{ConfigBlob: configBlob2}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo4, manifestDigest3, mTypes.ManifestMetadata{ConfigBlob: configBlob3}) - So(err, ShouldBeNil) - - opSys := LINUX - arch := "" - filter := mTypes.Filter{ - Os: []*string{&opSys}, - } - - repos, _, _, _, err := metaDB.SearchRepos(context.TODO(), "", filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - So(repos[0].Name, ShouldResemble, "repo1") - So(repos[1].Name, ShouldResemble, "repo2") - - opSys = WINDOWS - filter = mTypes.Filter{ - Os: []*string{&opSys}, - } - repos, _, _, _, err = metaDB.SearchRepos(context.TODO(), "repo", filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - So(repos[0].Name, ShouldResemble, "repo1") - So(repos[1].Name, ShouldResemble, "repo3") - - opSys = "wrong" - filter = mTypes.Filter{ - Os: []*string{&opSys}, - } - repos, _, _, _, err = metaDB.SearchRepos(context.TODO(), "repo", filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) - - opSys = LINUX - arch = AMD - filter = mTypes.Filter{ - Os: []*string{&opSys}, - Arch: []*string{&arch}, - } - repos, _, _, _, err = metaDB.SearchRepos(context.TODO(), "repo", filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - So(repos[0].Name, ShouldResemble, "repo1") - So(repos[1].Name, ShouldResemble, "repo2") - - opSys = WINDOWS - arch = AMD - filter = mTypes.Filter{ - Os: []*string{&opSys}, - Arch: []*string{&arch}, - } - repos, _, _, _, err = metaDB.SearchRepos(context.TODO(), "repo", filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - }) - - Convey("Test tags search with filtering", func() { - var ( - repo1 = "repo1" - repo2 = "repo2" - repo3 = "repo3" - repo4 = "repo4" - tag1 = "0.0.1" - tag2 = "0.0.2" - tag3 = "0.0.3" - manifestDigest1 = godigest.FromString("fake-manifest1") - manifestDigest2 = godigest.FromString("fake-manifest2") - manifestDigest3 = godigest.FromString("fake-manifest3") - - indexDigest = godigest.FromString("index-digest") - manifestFromIndexDigest1 = godigest.FromString("fake-manifestFromIndexDigest1") - manifestFromIndexDigest2 = godigest.FromString("fake-manifestFromIndexDigest2") - ) - - err := metaDB.SetRepoReference(repo1, tag3, indexDigest, ispec.MediaTypeImageIndex) - So(err, ShouldBeNil) - - indexBlob, err := test.GetIndexBlobWithManifests( - []godigest.Digest{ - manifestFromIndexDigest1, - manifestFromIndexDigest2, - }, - ) - So(err, ShouldBeNil) - - err = metaDB.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - config1 := ispec.Image{ - Platform: ispec.Platform{ - Architecture: AMD, - OS: LINUX, - }, - } - configBlob1, err := json.Marshal(config1) - So(err, ShouldBeNil) - - config2 := ispec.Image{ - Platform: ispec.Platform{ - Architecture: "arch", - OS: WINDOWS, - }, - } - configBlob2, err := json.Marshal(config2) - So(err, ShouldBeNil) - - config3 := ispec.Image{} - configBlob3, err := json.Marshal(config3) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob1}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest2, mTypes.ManifestMetadata{ConfigBlob: configBlob2}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo2, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob1}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo3, manifestDigest2, mTypes.ManifestMetadata{ConfigBlob: configBlob2}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo4, manifestDigest3, mTypes.ManifestMetadata{ConfigBlob: configBlob3}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestFromIndexDigest1, - mTypes.ManifestMetadata{ConfigBlob: []byte("{}")}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestFromIndexDigest2, - mTypes.ManifestMetadata{ConfigBlob: []byte("{}")}) - So(err, ShouldBeNil) - - opSys := LINUX - arch := AMD - filter := mTypes.Filter{ - Os: []*string{&opSys}, - Arch: []*string{&arch}, - } - repos, _, _, _, err := metaDB.SearchTags(context.TODO(), "repo1:", filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Tags, ShouldContainKey, tag1) - - opSys = LINUX - arch = "badArch" - filter = mTypes.Filter{ - Os: []*string{&opSys}, - Arch: []*string{&arch}, - } - repos, _, _, _, err = metaDB.SearchTags(context.TODO(), "repo1:", filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) - }) + // MyTODO: add filtering tests for PaginatedRepoMeta2RepoSum or RepoImg Convey("Test FilterTags", func() { var ( @@ -2346,14 +1920,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) Convey("Return all tags", func() { - repos, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags( - ctx, + repos, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true - }, - mTypes.Filter{}, - mTypes.PageInput{Limit: 10, Offset: 0, SortBy: mTypes.AlphabeticAsc}, - ) + }) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 2) @@ -2368,25 +1938,22 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(repos[0].Tags, ShouldContainKey, "1.0.1") So(repos[0].Tags, ShouldContainKey, "2.0.0") So(repos[1].Tags, ShouldContainKey, "0.0.1") + So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) So(manifestMetaMap, ShouldContainKey, manifestDigest2.String()) So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) + So(indexDataMap, ShouldContainKey, indexDigest.String()) So(manifestMetaMap, ShouldContainKey, manifestFromIndexDigest1.String()) So(manifestMetaMap, ShouldContainKey, manifestFromIndexDigest2.String()) - So(pageInfo.ItemCount, ShouldEqual, 7) - So(pageInfo.TotalCount, ShouldEqual, 7) }) Convey("Return all tags in a specific repo", func() { - repos, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags( + repos, manifestMetaMap, indexDataMap, err := metaDB.FilterTags( ctx, func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return repoMeta.Name == repo1 - }, - mTypes.Filter{}, - mTypes.PageInput{Limit: 10, Offset: 0, SortBy: mTypes.AlphabeticAsc}, - ) + }) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -2404,25 +1971,18 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(indexDataMap, ShouldContainKey, indexDigest.String()) So(manifestMetaMap, ShouldContainKey, manifestFromIndexDigest1.String()) So(manifestMetaMap, ShouldContainKey, manifestFromIndexDigest2.String()) - So(pageInfo.ItemCount, ShouldEqual, 6) - So(pageInfo.TotalCount, ShouldEqual, 6) }) Convey("Filter everything out", func() { - repos, manifestMetaMap, _, pageInfo, err := metaDB.FilterTags( + repos, manifestMetaMap, _, err := metaDB.FilterTags( ctx, func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false - }, - mTypes.Filter{}, - mTypes.PageInput{Limit: 10, Offset: 0, SortBy: mTypes.AlphabeticAsc}, - ) + }) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) So(len(manifestMetaMap), ShouldEqual, 0) - So(pageInfo.ItemCount, ShouldEqual, 0) - So(pageInfo.TotalCount, ShouldEqual, 0) }) Convey("Search with access control", func() { @@ -2437,14 +1997,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func authzCtxKey := localCtx.GetContextKey() ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) - repos, manifestMetaMap, _, pageInfo, err := metaDB.FilterTags( - ctx, + repos, manifestMetaMap, _, err := metaDB.FilterTags(ctx, func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true - }, - mTypes.Filter{}, - mTypes.PageInput{Limit: 10, Offset: 0, SortBy: mTypes.AlphabeticAsc}, - ) + }) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -2452,236 +2008,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(len(repos[0].Tags), ShouldEqual, 1) So(repos[0].Tags, ShouldContainKey, "0.0.1") So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) - So(pageInfo.ItemCount, ShouldEqual, 1) - So(pageInfo.TotalCount, ShouldEqual, 1) }) - - Convey("With wrong pagination input", func() { - repos, _, _, _, err := metaDB.FilterTags( - ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }, - mTypes.Filter{}, - mTypes.PageInput{Limit: -1}, - ) - So(err, ShouldNotBeNil) - So(repos, ShouldBeEmpty) - }) - }) - - Convey("Test tags filtering by filter function and OS/Arch Filter", func() { - var ( - repo1 = "repo1" - repo2 = "repo2" - repo3 = "repo3" - repo4 = "repo4" - tag1 = "0.0.1" - tag2 = "0.0.2" - tag3 = "0.0.3" - manifestDigest1 = godigest.FromString("fake-manifest1") - manifestDigest2 = godigest.FromString("fake-manifest2") - manifestDigest3 = godigest.FromString("fake-manifest3") - - indexDigest = godigest.FromString("index-digest") - manifestFromIndexDigest1 = godigest.FromString("fake-manifestFromIndexDigest1") - manifestFromIndexDigest2 = godigest.FromString("fake-manifestFromIndexDigest2") - manifestFromIndexDigest3 = godigest.FromString("fake-manifestFromIndexDigest3") - ) - - err := metaDB.SetRepoReference(repo1, tag3, indexDigest, ispec.MediaTypeImageIndex) - So(err, ShouldBeNil) - - indexBlob, err := test.GetIndexBlobWithManifests( - []godigest.Digest{ - manifestFromIndexDigest1, - manifestFromIndexDigest2, - manifestFromIndexDigest3, - }, - ) - So(err, ShouldBeNil) - - err = metaDB.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - config1 := ispec.Image{ - Platform: ispec.Platform{ - Architecture: AMD, - OS: LINUX, - }, - } - configBlob1, err := json.Marshal(config1) - So(err, ShouldBeNil) - - config2 := ispec.Image{ - Platform: ispec.Platform{ - Architecture: ARM, - OS: LINUX, - }, - } - configBlob2, err := json.Marshal(config2) - So(err, ShouldBeNil) - - config3 := ispec.Image{ - Platform: ispec.Platform{ - Architecture: AMD, - OS: WINDOWS, - }, - } - configBlob3, err := json.Marshal(config3) - So(err, ShouldBeNil) - - config4 := ispec.Image{} - configBlob4, err := json.Marshal(config4) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob1}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest2, mTypes.ManifestMetadata{ConfigBlob: configBlob2}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo2, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob1}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo3, manifestDigest2, mTypes.ManifestMetadata{ConfigBlob: configBlob2}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo4, manifestDigest3, mTypes.ManifestMetadata{ConfigBlob: configBlob4}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestFromIndexDigest1, - mTypes.ManifestMetadata{ConfigBlob: configBlob1}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestFromIndexDigest2, - mTypes.ManifestMetadata{ConfigBlob: configBlob2}) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestFromIndexDigest3, - mTypes.ManifestMetadata{ConfigBlob: configBlob3}) - So(err, ShouldBeNil) - - opSys := LINUX - arch := AMD - filter := mTypes.Filter{ - Os: []*string{&opSys}, - Arch: []*string{&arch}, - } - repos, _, _, _, err := metaDB.FilterTags(context.TODO(), - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }, - filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - So(len(repos[0].Tags), ShouldEqual, 2) - So(repos[0].Tags, ShouldContainKey, tag1) - So(repos[0].Tags, ShouldContainKey, tag3) - So(len(repos[1].Tags), ShouldEqual, 1) - So(repos[1].Tags, ShouldContainKey, tag1) - - opSys = LINUX - filter = mTypes.Filter{ - Os: []*string{&opSys}, - } - repos, _, _, _, err = metaDB.FilterTags(context.TODO(), - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }, - filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 3) - So(len(repos[0].Tags), ShouldEqual, 3) - So(repos[0].Tags, ShouldContainKey, tag1) - So(repos[0].Tags, ShouldContainKey, tag2) - So(repos[0].Tags, ShouldContainKey, tag3) - So(len(repos[1].Tags), ShouldEqual, 1) - So(repos[1].Tags, ShouldContainKey, tag1) - So(len(repos[2].Tags), ShouldEqual, 1) - So(repos[1].Tags, ShouldContainKey, tag1) - - opSys = WINDOWS - filter = mTypes.Filter{ - Os: []*string{&opSys}, - } - repos, _, _, _, err = metaDB.FilterTags(context.TODO(), - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }, - filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(len(repos[0].Tags), ShouldEqual, 1) - So(repos[0].Tags, ShouldContainKey, tag3) - - arch = AMD - filter = mTypes.Filter{ - Arch: []*string{&arch}, - } - repos, _, _, _, err = metaDB.FilterTags(context.TODO(), - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }, - filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - So(len(repos[0].Tags), ShouldEqual, 2) - So(repos[0].Tags, ShouldContainKey, tag1) - So(repos[0].Tags, ShouldContainKey, tag3) - So(len(repos[1].Tags), ShouldEqual, 1) - So(repos[1].Tags, ShouldContainKey, tag1) - - repos, _, _, _, err = metaDB.FilterTags(context.TODO(), - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }, - mTypes.Filter{}, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 4) - So(len(repos[0].Tags), ShouldEqual, 3) - So(repos[0].Tags, ShouldContainKey, tag1) - So(repos[0].Tags, ShouldContainKey, tag2) - So(repos[0].Tags, ShouldContainKey, tag3) - So(len(repos[1].Tags), ShouldEqual, 1) - So(repos[1].Tags, ShouldContainKey, tag1) - So(len(repos[2].Tags), ShouldEqual, 1) - So(repos[2].Tags, ShouldContainKey, tag1) - So(len(repos[3].Tags), ShouldEqual, 1) - So(repos[3].Tags, ShouldContainKey, tag1) - - opSys = LINUX - arch = "badArch" - filter = mTypes.Filter{ - Os: []*string{&opSys}, - Arch: []*string{&arch}, - } - repos, _, _, _, err = metaDB.FilterTags(context.TODO(), - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }, - filter, - mTypes.PageInput{SortBy: mTypes.AlphabeticAsc}) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) }) Convey("Test index logic", func() { @@ -2903,17 +2230,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference("repo", multiarch.Reference, multiarchDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) - repoMetas, _, _, _, err := metaDB.FilterRepos(context.Background(), - func(repoMeta mTypes.RepoMetadata) bool { return true }, mTypes.PageInput{}) + repoMetas, _, _, err := metaDB.FilterRepos(context.Background(), + func(repoMeta mTypes.RepoMetadata) bool { return true }) So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) - - _, _, _, _, err = metaDB.FilterRepos(context.Background(), - func(repoMeta mTypes.RepoMetadata) bool { return true }, mTypes.PageInput{ - Limit: -1, - Offset: -1, - }) - So(err, ShouldNotBeNil) }) Convey("Test bookmarked/starred field present in returned RepoMeta", func() { @@ -2938,31 +2258,28 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference(repo99, "tag", manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - repoMetas, _, _, _, err := metaDB.SearchRepos(ctx, repo99, mTypes.Filter{}, mTypes.PageInput{}) + repoMetas, _, _, err := metaDB.SearchRepos(ctx, repo99) So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) So(repoMetas[0].IsBookmarked, ShouldBeFalse) So(repoMetas[0].IsStarred, ShouldBeFalse) - repoMetas, _, _, _, err = metaDB.SearchTags(ctx, repo99+":", mTypes.Filter{}, mTypes.PageInput{}) + repoMetas, _, _, err = metaDB.SearchTags(ctx, repo99+":") So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) So(repoMetas[0].IsBookmarked, ShouldBeFalse) So(repoMetas[0].IsStarred, ShouldBeFalse) - repoMetas, _, _, _, err = metaDB.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { + repoMetas, _, _, err = metaDB.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true - }, mTypes.PageInput{}) + }) So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) So(repoMetas[0].IsBookmarked, ShouldBeFalse) So(repoMetas[0].IsStarred, ShouldBeFalse) - repoMetas, _, _, _, err = metaDB.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + repoMetas, _, _, err = metaDB.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) So(repoMetas[0].IsBookmarked, ShouldBeFalse) @@ -2974,31 +2291,28 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func _, err = metaDB.ToggleStarRepo(ctx, repo99) So(err, ShouldBeNil) - repoMetas, _, _, _, err = metaDB.SearchRepos(ctx, repo99, mTypes.Filter{}, mTypes.PageInput{}) + repoMetas, _, _, err = metaDB.SearchRepos(ctx, repo99) So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) So(repoMetas[0].IsBookmarked, ShouldBeTrue) So(repoMetas[0].IsStarred, ShouldBeTrue) - repoMetas, _, _, _, err = metaDB.SearchTags(ctx, repo99+":", mTypes.Filter{}, mTypes.PageInput{}) + repoMetas, _, _, err = metaDB.SearchTags(ctx, repo99+":") So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) So(repoMetas[0].IsBookmarked, ShouldBeTrue) So(repoMetas[0].IsStarred, ShouldBeTrue) - repoMetas, _, _, _, err = metaDB.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { + repoMetas, _, _, err = metaDB.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true - }, mTypes.PageInput{}) + }) So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) So(repoMetas[0].IsBookmarked, ShouldBeTrue) So(repoMetas[0].IsStarred, ShouldBeTrue) - repoMetas, _, _, _, err = metaDB.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }, - mTypes.Filter{}, - mTypes.PageInput{}, - ) + repoMetas, _, _, err = metaDB.FilterTags(ctx, + func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) So(err, ShouldBeNil) So(len(repoMetas), ShouldEqual, 1) So(repoMetas[0].IsBookmarked, ShouldBeTrue) @@ -3077,84 +2391,19 @@ func TestRelevanceSorting(t *testing.T) { So(common.RankRepoName("debian/base-amd64", "c3/aux/debian/base-amd64"), ShouldEqual, 800) So(common.RankRepoName("aux/debian", "c3/aux/debian/base-amd64"), ShouldEqual, 400) - Convey("Integration", func() { - filePath := path.Join(t.TempDir(), "repo.db") - boltDBParams := boltdb.DBParameters{ - RootDir: t.TempDir(), - } - boltDriver, err := boltdb.GetBoltDriver(boltDBParams) - So(err, ShouldBeNil) - - log := log.NewLogger("debug", "") - - metaDB, err := boltdb.New(boltDriver, log) - So(metaDB, ShouldNotBeNil) - So(err, ShouldBeNil) - - defer os.Remove(filePath) - - var ( - repo1 = "alpine" - repo2 = "alpine/test" - repo3 = "notalpine" - repo4 = "unmached/repo" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - tag2 = "0.0.2" - manifestDigest2 = godigest.FromString("fake-manifest2") - tag3 = "0.0.3" - manifestDigest3 = godigest.FromString("fake-manifest3") - ctx = context.Background() - emptyManifest ispec.Manifest - emptyConfig ispec.Manifest - ) - emptyManifestBlob, err := json.Marshal(emptyManifest) - So(err, ShouldBeNil) - - emptyConfigBlob, err := json.Marshal(emptyConfig) - So(err, ShouldBeNil) - - emptyRepoMeta := mTypes.ManifestMetadata{ - ManifestBlob: emptyManifestBlob, - ConfigBlob: emptyConfigBlob, - } - - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest2, emptyRepoMeta) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo2, manifestDigest1, emptyRepoMeta) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo3, manifestDigest2, emptyRepoMeta) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo4, manifestDigest3, emptyRepoMeta) - So(err, ShouldBeNil) - - repos, _, _, _, err := metaDB.SearchRepos(ctx, "pine", mTypes.Filter{}, - mTypes.PageInput{SortBy: mTypes.Relevance}, - ) - - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 3) - So(repos[0].Name, ShouldEqual, repo1) - So(repos[1].Name, ShouldEqual, repo3) - So(repos[2].Name, ShouldEqual, repo2) - }) + // MyTODO: add these tests for convert functions + // repo1 = "alpine" + // repo2 = "alpine/test" + // repo3 = "notalpine" + // repo4 = "unmached/repo" + // + // repos, _, _, err := metaDB.SearchRepos(ctx, "pine") + // + // So(err, ShouldBeNil) + // So(len(repos), ShouldEqual, 3) + // So(repos[0].Name, ShouldEqual, repo1) + // So(repos[1].Name, ShouldEqual, repo3) + // So(repos[2].Name, ShouldEqual, repo2) }) } diff --git a/pkg/meta/pagination/pagination.go b/pkg/meta/pagination/pagination.go deleted file mode 100644 index 7a8b012704..0000000000 --- a/pkg/meta/pagination/pagination.go +++ /dev/null @@ -1,273 +0,0 @@ -package pagination - -import ( - "fmt" - "sort" - - zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/common" - mTypes "zotregistry.io/zot/pkg/meta/types" -) - -// PageFinder permits keeping a pool of objects using Add -// and returning a specific page. -type PageFinder interface { - Add(detailedRepoMeta mTypes.DetailedRepoMeta) - Page() ([]mTypes.RepoMetadata, common.PageInfo) - Reset() -} - -// RepoPageFinder implements PageFinder. It manages RepoMeta objects and calculates the page -// using the given limit, offset and sortBy option. -type RepoPageFinder struct { - limit int - offset int - sortBy mTypes.SortCriteria - pageBuffer []mTypes.DetailedRepoMeta -} - -func NewBaseRepoPageFinder(limit, offset int, sortBy mTypes.SortCriteria) (*RepoPageFinder, error) { - if sortBy == "" { - sortBy = mTypes.AlphabeticAsc - } - - if limit < 0 { - return nil, zerr.ErrLimitIsNegative - } - - if offset < 0 { - return nil, zerr.ErrOffsetIsNegative - } - - if _, found := mTypes.SortFunctions()[sortBy]; !found { - return nil, fmt.Errorf("sorting repos by '%s' is not supported %w", - sortBy, zerr.ErrSortCriteriaNotSupported) - } - - return &RepoPageFinder{ - limit: limit, - offset: offset, - sortBy: sortBy, - pageBuffer: make([]mTypes.DetailedRepoMeta, 0, limit), - }, nil -} - -func (bpt *RepoPageFinder) Reset() { - bpt.pageBuffer = []mTypes.DetailedRepoMeta{} -} - -func (bpt *RepoPageFinder) Add(namedRepoMeta mTypes.DetailedRepoMeta) { - bpt.pageBuffer = append(bpt.pageBuffer, namedRepoMeta) -} - -func (bpt *RepoPageFinder) Page() ([]mTypes.RepoMetadata, common.PageInfo) { - if len(bpt.pageBuffer) == 0 { - return []mTypes.RepoMetadata{}, common.PageInfo{} - } - - pageInfo := &common.PageInfo{} - - sort.Slice(bpt.pageBuffer, mTypes.SortFunctions()[bpt.sortBy](bpt.pageBuffer)) - - // the offset and limit are calculatd in terms of repos counted - start := bpt.offset - end := bpt.offset + bpt.limit - - // we'll return an empty array when the offset is greater than the number of elements - if start >= len(bpt.pageBuffer) { - start = len(bpt.pageBuffer) - end = start - } - - if end >= len(bpt.pageBuffer) { - end = len(bpt.pageBuffer) - } - - detailedReposPage := bpt.pageBuffer[start:end] - - pageInfo.ItemCount = len(detailedReposPage) - - if start == 0 && end == 0 { - detailedReposPage = bpt.pageBuffer - pageInfo.ItemCount = len(detailedReposPage) - } - - repos := make([]mTypes.RepoMetadata, 0, len(detailedReposPage)) - - for _, drm := range detailedReposPage { - repos = append(repos, drm.RepoMetadata) - } - - pageInfo.TotalCount = len(bpt.pageBuffer) - - return repos, *pageInfo -} - -type ImagePageFinder struct { - limit int - offset int - sortBy mTypes.SortCriteria - pageBuffer []mTypes.DetailedRepoMeta -} - -func NewBaseImagePageFinder(limit, offset int, sortBy mTypes.SortCriteria) (*ImagePageFinder, error) { - if sortBy == "" { - sortBy = mTypes.AlphabeticAsc - } - - if limit < 0 { - return nil, zerr.ErrLimitIsNegative - } - - if offset < 0 { - return nil, zerr.ErrOffsetIsNegative - } - - if _, found := mTypes.SortFunctions()[sortBy]; !found { - return nil, fmt.Errorf("sorting repos by '%s' is not supported %w", - sortBy, zerr.ErrSortCriteriaNotSupported) - } - - return &ImagePageFinder{ - limit: limit, - offset: offset, - sortBy: sortBy, - pageBuffer: make([]mTypes.DetailedRepoMeta, 0, limit), - }, nil -} - -func (bpt *ImagePageFinder) Reset() { - bpt.pageBuffer = []mTypes.DetailedRepoMeta{} -} - -func (bpt *ImagePageFinder) Add(namedRepoMeta mTypes.DetailedRepoMeta) { - bpt.pageBuffer = append(bpt.pageBuffer, namedRepoMeta) -} - -func (bpt *ImagePageFinder) Page() ([]mTypes.RepoMetadata, common.PageInfo) { - if len(bpt.pageBuffer) == 0 { - return []mTypes.RepoMetadata{}, common.PageInfo{} - } - - pageInfo := common.PageInfo{} - - for _, drm := range bpt.pageBuffer { - repo := drm.RepoMetadata - pageInfo.TotalCount += len(repo.Tags) - } - - sort.Slice(bpt.pageBuffer, mTypes.SortFunctions()[bpt.sortBy](bpt.pageBuffer)) - - repoStartIndex := 0 - tagStartIndex := 0 - - // the offset and limit are calculatd in terms of tags counted - remainingOffset := bpt.offset - remainingLimit := bpt.limit - - repos := make([]mTypes.RepoMetadata, 0) - - if remainingOffset == 0 && remainingLimit == 0 { - for _, drm := range bpt.pageBuffer { - repo := drm.RepoMetadata - repos = append(repos, repo) - - pageInfo.ItemCount += len(repo.Tags) - } - - return repos, pageInfo - } - - // bring cursor to position in RepoMeta array - for _, drm := range bpt.pageBuffer { - if remainingOffset < len(drm.Tags) { - tagStartIndex = remainingOffset - - break - } - - remainingOffset -= len(drm.Tags) - repoStartIndex++ - } - - // offset is larger than the number of tags - if repoStartIndex >= len(bpt.pageBuffer) { - return []mTypes.RepoMetadata{}, common.PageInfo{} - } - - // finish counting remaining tags inside the first repo meta - partialTags := map[string]mTypes.Descriptor{} - firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMetadata - - tags := make([]string, 0, len(firstRepoMeta.Tags)) - for k := range firstRepoMeta.Tags { - tags = append(tags, k) - } - - sort.Strings(tags) - - for i := tagStartIndex; i < len(tags); i++ { - tag := tags[i] - - partialTags[tag] = firstRepoMeta.Tags[tag] - remainingLimit-- - - if remainingLimit == 0 { - firstRepoMeta.Tags = partialTags - repos = append(repos, firstRepoMeta) - pageInfo.ItemCount = len(partialTags) - - return repos, pageInfo - } - } - - firstRepoMeta.Tags = partialTags - pageInfo.ItemCount += len(firstRepoMeta.Tags) - repos = append(repos, firstRepoMeta) - repoStartIndex++ - - // continue with the remaining repos - for i := repoStartIndex; i < len(bpt.pageBuffer); i++ { - repoMeta := bpt.pageBuffer[i].RepoMetadata - - if len(repoMeta.Tags) > remainingLimit { - partialTags := map[string]mTypes.Descriptor{} - - tags := make([]string, 0, len(repoMeta.Tags)) - for k := range repoMeta.Tags { - tags = append(tags, k) - } - - sort.Strings(tags) - - for _, tag := range tags { - partialTags[tag] = repoMeta.Tags[tag] - remainingLimit-- - - if remainingLimit == 0 { - repoMeta.Tags = partialTags - repos = append(repos, repoMeta) - - pageInfo.ItemCount += len(partialTags) - - break - } - } - - return repos, pageInfo - } - - // add the whole repo - repos = append(repos, repoMeta) - pageInfo.ItemCount += len(repoMeta.Tags) - remainingLimit -= len(repoMeta.Tags) - - if remainingLimit == 0 { - return repos, pageInfo - } - } - - // we arrive here when the limit is bigger than the number of tags - - return repos, pageInfo -} diff --git a/pkg/meta/pagination/pagination_test.go b/pkg/meta/pagination/pagination_test.go deleted file mode 100644 index 5747a3f631..0000000000 --- a/pkg/meta/pagination/pagination_test.go +++ /dev/null @@ -1,241 +0,0 @@ -package pagination_test - -import ( - "testing" - - ispec "github.com/opencontainers/image-spec/specs-go/v1" - . "github.com/smartystreets/goconvey/convey" - - "zotregistry.io/zot/pkg/meta/pagination" - mTypes "zotregistry.io/zot/pkg/meta/types" -) - -func TestPagination(t *testing.T) { - Convey("Repo Pagination", t, func() { - Convey("reset", func() { - pageFinder, err := pagination.NewBaseRepoPageFinder(1, 0, mTypes.AlphabeticAsc) - So(err, ShouldBeNil) - So(pageFinder, ShouldNotBeNil) - - pageFinder.Add(mTypes.DetailedRepoMeta{}) - pageFinder.Add(mTypes.DetailedRepoMeta{}) - pageFinder.Add(mTypes.DetailedRepoMeta{}) - - pageFinder.Reset() - - result, _ := pageFinder.Page() - So(result, ShouldBeEmpty) - }) - }) - - Convey("Image Pagination", t, func() { - Convey("create new pageFinder errors", func() { - pageFinder, err := pagination.NewBaseImagePageFinder(-1, 10, mTypes.AlphabeticAsc) - So(pageFinder, ShouldBeNil) - So(err, ShouldNotBeNil) - - pageFinder, err = pagination.NewBaseImagePageFinder(2, -1, mTypes.AlphabeticAsc) - So(pageFinder, ShouldBeNil) - So(err, ShouldNotBeNil) - - pageFinder, err = pagination.NewBaseImagePageFinder(2, 1, "wrong sorting criteria") - So(pageFinder, ShouldBeNil) - So(err, ShouldNotBeNil) - }) - - Convey("Reset", func() { - pageFinder, err := pagination.NewBaseImagePageFinder(1, 0, mTypes.AlphabeticAsc) - So(err, ShouldBeNil) - So(pageFinder, ShouldNotBeNil) - - pageFinder.Add(mTypes.DetailedRepoMeta{}) - pageFinder.Add(mTypes.DetailedRepoMeta{}) - pageFinder.Add(mTypes.DetailedRepoMeta{}) - - pageFinder.Reset() - - result, _ := pageFinder.Page() - So(result, ShouldBeEmpty) - }) - - Convey("Page", func() { - Convey("no limit or offset", func() { - pageFinder, err := pagination.NewBaseImagePageFinder(0, 0, mTypes.AlphabeticAsc) - So(err, ShouldBeNil) - So(pageFinder, ShouldNotBeNil) - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, - }, - }, - }) - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo2", - Tags: map[string]mTypes.Descriptor{ - "Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, - "Tag2": {Digest: "dig2", MediaType: ispec.MediaTypeImageManifest}, - "Tag3": {Digest: "dig3", MediaType: ispec.MediaTypeImageManifest}, - "Tag4": {Digest: "dig4", MediaType: ispec.MediaTypeImageManifest}, - }, - }, - }) - _, pageInfo := pageFinder.Page() - So(pageInfo.ItemCount, ShouldEqual, 5) - So(pageInfo.TotalCount, ShouldEqual, 5) - }) - Convey("Test 1 limit < len(tags)", func() { - pageFinder, err := pagination.NewBaseImagePageFinder(5, 2, mTypes.AlphabeticAsc) - So(err, ShouldBeNil) - So(pageFinder, ShouldNotBeNil) - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, - }, - }, - }) - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo2", - Tags: map[string]mTypes.Descriptor{ - "Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, - "Tag2": {Digest: "dig2", MediaType: ispec.MediaTypeImageManifest}, - "Tag3": {Digest: "dig3", MediaType: ispec.MediaTypeImageManifest}, - "Tag4": {Digest: "dig4", MediaType: ispec.MediaTypeImageManifest}, - }, - }, - }) - _, pageInfo := pageFinder.Page() - So(pageInfo.ItemCount, ShouldEqual, 3) - So(pageInfo.TotalCount, ShouldEqual, 5) - }) - Convey("Test 2 limit < len(tags)", func() { - pageFinder, err := pagination.NewBaseImagePageFinder(5, 2, mTypes.AlphabeticAsc) - So(err, ShouldBeNil) - So(pageFinder, ShouldNotBeNil) - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "tag1": { - Digest: "dig1", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }, - }) - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo2", - Tags: map[string]mTypes.Descriptor{ - "Tag1": { - Digest: "dig1", - MediaType: ispec.MediaTypeImageManifest, - }, - "Tag2": { - Digest: "dig2", - MediaType: ispec.MediaTypeImageManifest, - }, - "Tag3": { - Digest: "dig3", - MediaType: ispec.MediaTypeImageManifest, - }, - "Tag4": { - Digest: "dig4", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }, - }) - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo3", - Tags: map[string]mTypes.Descriptor{ - "Tag11": { - Digest: "dig11", - MediaType: ispec.MediaTypeImageManifest, - }, - "Tag12": { - Digest: "dig12", - MediaType: ispec.MediaTypeImageManifest, - }, - "Tag13": { - Digest: "dig13", - MediaType: ispec.MediaTypeImageManifest, - }, - "Tag14": { - Digest: "dig14", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }, - }) - - result, pageInfo := pageFinder.Page() - So(result[0].Tags, ShouldContainKey, "Tag2") - So(result[0].Tags, ShouldContainKey, "Tag3") - So(result[0].Tags, ShouldContainKey, "Tag4") - So(result[1].Tags, ShouldContainKey, "Tag11") - So(result[1].Tags, ShouldContainKey, "Tag12") - So(pageInfo.ItemCount, ShouldEqual, 5) - So(pageInfo.TotalCount, ShouldEqual, 9) - }) - - Convey("Test 2 limit > len(tags)", func() { - pageFinder, err := pagination.NewBaseImagePageFinder(3, 0, mTypes.AlphabeticAsc) - So(err, ShouldBeNil) - So(pageFinder, ShouldNotBeNil) - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "tag1": { - Digest: "dig1", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }, - }) - - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo2", - Tags: map[string]mTypes.Descriptor{ - "Tag1": { - Digest: "dig1", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }, - }) - pageFinder.Add(mTypes.DetailedRepoMeta{ - RepoMetadata: mTypes.RepoMetadata{ - Name: "repo3", - Tags: map[string]mTypes.Descriptor{ - "Tag11": { - Digest: "dig11", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }, - }) - - result, _ := pageFinder.Page() - So(result[0].Tags, ShouldContainKey, "tag1") - So(result[1].Tags, ShouldContainKey, "Tag1") - So(result[2].Tags, ShouldContainKey, "Tag11") - }) - }) - }) -} diff --git a/pkg/meta/storage_parsing_test.go b/pkg/meta/storage_parsing_test.go index faed808841..415bd5c259 100644 --- a/pkg/meta/storage_parsing_test.go +++ b/pkg/meta/storage_parsing_test.go @@ -480,11 +480,8 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) - repos, err := metaDB.GetMultipleRepoMeta( - context.Background(), - func(repoMeta mTypes.RepoMetadata) bool { return true }, - mTypes.PageInput{}, - ) + repos, err := metaDB.GetMultipleRepoMeta(context.Background(), + func(repoMeta mTypes.RepoMetadata) bool { return true }) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -552,7 +549,6 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { repos, err := metaDB.GetMultipleRepoMeta( context.Background(), func(repoMeta mTypes.RepoMetadata) bool { return true }, - mTypes.PageInput{}, ) So(err, ShouldBeNil) diff --git a/pkg/meta/types/sort_criteria.go b/pkg/meta/types/sort_criteria.go deleted file mode 100644 index 9d62f73703..0000000000 --- a/pkg/meta/types/sort_criteria.go +++ /dev/null @@ -1,54 +0,0 @@ -package types - -type SortCriteria string - -const ( - Relevance = SortCriteria("RELEVANCE") - UpdateTime = SortCriteria("UPDATE_TIME") - AlphabeticAsc = SortCriteria("ALPHABETIC_ASC") - AlphabeticDsc = SortCriteria("ALPHABETIC_DSC") - Stars = SortCriteria("STARS") - Downloads = SortCriteria("DOWNLOADS") -) - -func SortFunctions() map[SortCriteria]func(pageBuffer []DetailedRepoMeta) func(i, j int) bool { - return map[SortCriteria]func(pageBuffer []DetailedRepoMeta) func(i, j int) bool{ - AlphabeticAsc: SortByAlphabeticAsc, - AlphabeticDsc: SortByAlphabeticDsc, - Relevance: SortByRelevance, - UpdateTime: SortByUpdateTime, - Downloads: SortByDownloads, - } -} - -func SortByAlphabeticAsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool { - return func(i, j int) bool { - return pageBuffer[i].Name < pageBuffer[j].Name - } -} - -func SortByAlphabeticDsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool { - return func(i, j int) bool { - return pageBuffer[i].Name > pageBuffer[j].Name - } -} - -func SortByRelevance(pageBuffer []DetailedRepoMeta) func(i, j int) bool { - return func(i, j int) bool { - return pageBuffer[i].Rank < pageBuffer[j].Rank - } -} - -// SortByUpdateTime sorting descending by time. -func SortByUpdateTime(pageBuffer []DetailedRepoMeta) func(i, j int) bool { - return func(i, j int) bool { - return pageBuffer[i].UpdateTime.After(pageBuffer[j].UpdateTime) - } -} - -// SortByDownloads returns a comparison function for descendant sorting by downloads. -func SortByDownloads(pageBuffer []DetailedRepoMeta) func(i, j int) bool { - return func(i, j int) bool { - return pageBuffer[i].Downloads > pageBuffer[j].Downloads - } -} diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index 68097eb3b0..c170c001b0 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -5,8 +5,6 @@ import ( "time" godigest "github.com/opencontainers/go-digest" - - "zotregistry.io/zot/pkg/common" ) // DetailedRepoMeta is a auxiliary structure used for sorting RepoMeta arrays by information @@ -62,7 +60,7 @@ type MetaDB interface { //nolint:interfacebloat // GetMultipleRepoMeta returns information about all repositories as map[string]RepoMetadata filtered by the filter // function - GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMetadata) bool, requestedPage PageInput) ( + GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMetadata) bool) ( []RepoMetadata, error) // SetManifestData sets ManifestData for a given manifest in the database @@ -107,20 +105,20 @@ type MetaDB interface { //nolint:interfacebloat UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error // SearchRepos searches for repos given a search string - SearchRepos(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) ( - []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error) + SearchRepos(ctx context.Context, searchText string) ( + []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error) // SearchTags searches for images(repo:tag) given a search string - SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) ( - []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error) + SearchTags(ctx context.Context, searchText string) ( + []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error) // FilterRepos filters for repos given a filter function - FilterRepos(ctx context.Context, filter FilterRepoFunc, requestedPage PageInput) ( - []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error) + FilterRepos(ctx context.Context, filter FilterRepoFunc) ( + []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error) // FilterTags filters for images given a filter function - FilterTags(ctx context.Context, filterFunc FilterFunc, filter Filter, - requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error) + FilterTags(ctx context.Context, filterFunc FilterFunc) ( + []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error) PatchDB() error } @@ -204,6 +202,7 @@ type RepoMetadata struct { IsStarred bool IsBookmarked bool + Rank int Stars int } @@ -234,12 +233,6 @@ type UserData struct { APIKeys map[string]APIKeyDetails } -type PageInput struct { - Limit int - Offset int - SortBy SortCriteria -} - type Filter struct { Os []*string Arch []*string diff --git a/pkg/test/images.go b/pkg/test/images.go index 61af54b2a0..acde61e586 100644 --- a/pkg/test/images.go +++ b/pkg/test/images.go @@ -14,6 +14,10 @@ import ( storageConstants "zotregistry.io/zot/pkg/storage/constants" ) +const ( + TestFakeSignatureArtType = "application/test.fake.signature" +) + // LayerBuilder abstracts the first step in creating an OCI image, specifying the layers of the image. type LayerBuilder interface { // LayerBlobs sets the image layers from the gives blobs array, adding a default zipped layer media type. @@ -163,6 +167,13 @@ func CreateRandomVulnerableImageWith() ManifestBuilder { return CreateImageWith().VulnerableLayers().RandomVulnConfig() } +// CreateFakeTestSignature returns a test signature that is used to mark a image as signed +// when creating a test Repo. It won't be recognized as a signature if uploaded to the repository directly. +func CreateFakeTestSignature(subject *ispec.Descriptor) Image { + return CreateImageWith().RandomLayers(1, 10).DefaultConfig(). + ArtifactType(TestFakeSignatureArtType).Subject(subject).Build() +} + type BaseImageBuilder struct { layers []Layer diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index 85e7daafe5..1ebdced8cc 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -5,7 +5,6 @@ import ( godigest "github.com/opencontainers/go-digest" - "zotregistry.io/zot/pkg/common" mTypes "zotregistry.io/zot/pkg/meta/types" ) @@ -30,8 +29,8 @@ type MetaDBMock struct { SetRepoMetaFn func(repo string, repoMeta mTypes.RepoMetadata) error - GetMultipleRepoMetaFn func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, - requestedPage mTypes.PageInput) ([]mTypes.RepoMetadata, error) + GetMultipleRepoMetaFn func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool) ( + []mTypes.RepoMetadata, error) GetManifestDataFn func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) @@ -62,34 +61,19 @@ type MetaDBMock struct { DeleteSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm mTypes.SignatureMetadata) error - SearchReposFn func(ctx context.Context, txt string, filter mTypes.Filter, requestedPage mTypes.PageInput) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, + SearchReposFn func(ctx context.Context, txt string) ( + []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) - SearchTagsFn func(ctx context.Context, txt string, filter mTypes.Filter, requestedPage mTypes.PageInput) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, + SearchTagsFn func(ctx context.Context, txt string) ( + []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) - FilterReposFn func(ctx context.Context, filter mTypes.FilterRepoFunc, requestedPage mTypes.PageInput) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, - error) - - FilterTagsFn func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, - error) - - SearchDigestsFn func(ctx context.Context, searchText string, requestedPage mTypes.PageInput) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) - - SearchLayersFn func(ctx context.Context, searchText string, requestedPage mTypes.PageInput) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) - - SearchForAscendantImagesFn func(ctx context.Context, searchText string, requestedPage mTypes.PageInput) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) + FilterReposFn func(ctx context.Context, filter mTypes.FilterRepoFunc) ( + []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) - SearchForDescendantImagesFn func(ctx context.Context, searchText string, requestedPage mTypes.PageInput) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) + FilterTagsFn func(ctx context.Context, filterFunc mTypes.FilterFunc) ( + []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) GetStarredReposFn func(ctx context.Context) ([]string, error) @@ -195,10 +179,9 @@ func (sdm MetaDBMock) SetRepoMeta(repo string, repoMeta mTypes.RepoMetadata) err } func (sdm MetaDBMock) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, - requestedPage mTypes.PageInput, ) ([]mTypes.RepoMetadata, error) { if sdm.GetMultipleRepoMetaFn != nil { - return sdm.GetMultipleRepoMetaFn(ctx, filter, requestedPage) + return sdm.GetMultipleRepoMetaFn(ctx, filter) } return []mTypes.RepoMetadata{}, nil @@ -272,86 +255,44 @@ func (sdm MetaDBMock) DeleteSignature(repo string, signedManifestDigest godigest return nil } -func (sdm MetaDBMock) SearchRepos(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, error) { +func (sdm MetaDBMock) SearchRepos(ctx context.Context, searchText string, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { if sdm.SearchReposFn != nil { - return sdm.SearchReposFn(ctx, searchText, filter, requestedPage) + return sdm.SearchReposFn(ctx, searchText) } return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, common.PageInfo{}, nil + map[string]mTypes.IndexData{}, nil } -func (sdm MetaDBMock) SearchTags(ctx context.Context, searchText string, filter mTypes.Filter, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, error) { +func (sdm MetaDBMock) SearchTags(ctx context.Context, searchText string, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { if sdm.SearchTagsFn != nil { - return sdm.SearchTagsFn(ctx, searchText, filter, requestedPage) + return sdm.SearchTagsFn(ctx, searchText) } return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, common.PageInfo{}, nil + map[string]mTypes.IndexData{}, nil } func (sdm MetaDBMock) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, error) { +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { if sdm.FilterReposFn != nil { - return sdm.FilterReposFn(ctx, filter, requestedPage) + return sdm.FilterReposFn(ctx, filter) } return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, common.PageInfo{}, nil + map[string]mTypes.IndexData{}, nil } -func (sdm MetaDBMock) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, error) { +func (sdm MetaDBMock) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, +) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { if sdm.FilterTagsFn != nil { - return sdm.FilterTagsFn(ctx, filterFunc, filter, requestedPage) + return sdm.FilterTagsFn(ctx, filterFunc) } return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, common.PageInfo{}, nil -} - -func (sdm MetaDBMock) SearchDigests(ctx context.Context, searchText string, requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) { - if sdm.SearchDigestsFn != nil { - return sdm.SearchDigestsFn(ctx, searchText, requestedPage) - } - - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, nil -} - -func (sdm MetaDBMock) SearchLayers(ctx context.Context, searchText string, requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) { - if sdm.SearchLayersFn != nil { - return sdm.SearchLayersFn(ctx, searchText, requestedPage) - } - - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, nil -} - -func (sdm MetaDBMock) SearchForAscendantImages(ctx context.Context, searchText string, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) { - if sdm.SearchForAscendantImagesFn != nil { - return sdm.SearchForAscendantImagesFn(ctx, searchText, requestedPage) - } - - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, nil -} - -func (sdm MetaDBMock) SearchForDescendantImages(ctx context.Context, searchText string, - requestedPage mTypes.PageInput, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) { - if sdm.SearchForDescendantImagesFn != nil { - return sdm.SearchForDescendantImagesFn(ctx, searchText, requestedPage) - } - - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, nil + map[string]mTypes.IndexData{}, nil } func (sdm MetaDBMock) SetIndexData(digest godigest.Digest, indexData mTypes.IndexData) error { diff --git a/pkg/test/repo.go b/pkg/test/repo.go new file mode 100644 index 0000000000..1a10bd6e27 --- /dev/null +++ b/pkg/test/repo.go @@ -0,0 +1,91 @@ +package test + +import ( + ispec "github.com/opencontainers/image-spec/specs-go/v1" + + mTypes "zotregistry.io/zot/pkg/meta/types" +) + +type RepoImage struct { + Image + Tag string +} + +type RepoMultiArchImage struct { + MultiarchImage + Tag string +} + +type Repo struct { + Name string + Images []RepoImage + MultiArchImages []RepoMultiArchImage + IsBookmarked bool + IsStarred bool +} + +func GetMetadataForRepos(repos ...Repo) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, + map[string]mTypes.IndexData, +) { + var ( + reposMetadata = []mTypes.RepoMetadata{} + manifestMetadataMap = map[string]mTypes.ManifestMetadata{} + indexDataMap = map[string]mTypes.IndexData{} + ) + + for _, repo := range repos { + repoMeta := mTypes.RepoMetadata{ + Name: repo.Name, + Tags: map[string]mTypes.Descriptor{}, + Signatures: map[string]mTypes.ManifestSignatures{}, + IsStarred: repo.IsStarred, + IsBookmarked: repo.IsBookmarked, + } + + for _, image := range repo.Images { + if image.Tag != "" { + repoMeta.Tags[image.Tag] = mTypes.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: image.DigestStr(), + } + } + // here we can do many more checks about the images like check for referrers, signatures but it's not needed yet + // I need just the tags for now. + + // This is done just to mark a manifest as signed in the resulted RepoMeta + if image.Manifest.ArtifactType == TestFakeSignatureArtType && image.Manifest.Subject != nil { + signedManifestDig := image.Manifest.Subject.Digest.String() + repoMeta.Signatures[signedManifestDig] = mTypes.ManifestSignatures{ + "fakeSignature": []mTypes.SignatureInfo{{SignatureManifestDigest: image.ManifestDescriptor.Digest.String()}}, + } + } + + manifestMetadataMap[image.ManifestDescriptor.Digest.String()] = mTypes.ManifestMetadata{ + ManifestBlob: image.ManifestDescriptor.Data, + ConfigBlob: image.ConfigDescriptor.Data, + } + } + + for _, multiArch := range repo.MultiArchImages { + repoMeta.Tags[multiArch.Tag] = mTypes.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: multiArch.DigestStr(), + } + + for _, image := range multiArch.Images { + manifestMetadataMap[image.ManifestDescriptor.Digest.String()] = mTypes.ManifestMetadata{ + ManifestBlob: image.ManifestDescriptor.Data, + ConfigBlob: image.ConfigDescriptor.Data, + } + } + + indexDataMap[multiArch.indexDescriptor.Digest.String()] = mTypes.IndexData{ + IndexBlob: multiArch.indexDescriptor.Data, + } + } + + reposMetadata = append(reposMetadata, repoMeta) + } + + return reposMetadata, manifestMetadataMap, indexDataMap +}