diff --git a/Makefile b/Makefile index 794f009d95..0019079720 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ SWAGGER_VERSION := v1.8.12 STACKER := $(TOOLSDIR)/bin/stacker BATS := $(TOOLSDIR)/bin/bats PROTOC_VERSION := 3.15.8 +GO_PROTOC_VERSION := 1.31.0 PROTOC := $(TOOLSDIR)/bin/protoc TESTDATA := $(TOP_LEVEL)/test/data OS ?= $(shell go env GOOS) @@ -237,9 +238,9 @@ $(CRICTL): $(PROTOC): mkdir -p $(TOOLSDIR)/bin curl -Lo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-linux-x86_64.zip - unzip -d $(TOOLSDIR) protoc.zip bin/protoc + unzip -o -d $(TOOLSDIR) protoc.zip bin/protoc chmod +x $(PROTOC) - go install google.golang.org/protobuf/cmd/protoc-gen-go@latest + go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(GO_PROTOC_VERSION) $(ACTION_VALIDATOR): mkdir -p $(TOOLSDIR)/bin diff --git a/codecov.yml b/codecov.yml index 913bdf3a65..734d4be28e 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,3 +6,4 @@ ignore: - "./pkg/test/mocks/*.go" - "./swagger/*.go" - "./pkg/test/test_http_server.go" + - "./pkg/meta/proto_go/*.go" diff --git a/errors/errors.go b/errors/errors.go index 5e4811cc84..a03b136c6d 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -108,7 +108,7 @@ var ( ErrManifestConflict = errors.New("manifest: multiple manifests found") ErrManifestMetaNotFound = errors.New("metadb: image metadata not found for given manifest reference") ErrManifestDataNotFound = errors.New("metadb: image data not found for given manifest digest") - ErrImageDataNotFound = errors.New("metadb: image data not found for") + ErrImageDataNotFound = errors.New("metadb: image data not found") ErrWrongMediaType = errors.New("metadb: got unexpected media type") ErrIndexDataNotFount = errors.New("metadb: index data not found for given digest") ErrRepoMetaNotFound = errors.New("metadb: repo metadata not found for given repo name") diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 25d41d87a3..de14c64c75 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -254,7 +254,6 @@ func TestCreateMetaDBDriver(t *testing.T) { "repometatablename": "RepoMetadataTable", "imagedatatablename": "ZotImageDataTable", "repoblobsinfotablename": "ZotRepoBlobsInfoTable", - "indexdatatablename": "IndexDataTable", "userdatatablename": "ZotUserDataTable", "apikeytablename": "APIKeyTable", "versiontablename": "1", @@ -437,7 +436,6 @@ func TestObjectStorageController(t *testing.T) { "repometatablename": "RepoMetadataTable", "imagedatatablename": "ZotImageDataTable", "repoblobsinfotablename": "ZotRepoBlobsInfoTable", - "indexdatatablename": "IndexDataTable", "userdatatablename": "ZotUserDataTable", "apikeytablename": "APIKeyTable1", "versiontablename": "Version", diff --git a/pkg/common/common.go b/pkg/common/common.go index 6081ab7dd2..e374e21fcc 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -29,6 +29,12 @@ const ( ArtifactTypeNotation = "application/vnd.cncf.notary.signature" ) +var cosignTagRule = regexp.MustCompile(`sha256\-.+\.sig`) + +func IsCosignTag(tag string) bool { + return cosignTagRule.MatchString(tag) +} + func Contains[T comparable](elems []T, v T) bool { for _, s := range elems { if v == s { diff --git a/pkg/extensions/imagetrust/image_trust.go b/pkg/extensions/imagetrust/image_trust.go index c239988c8a..1d5dd49c24 100644 --- a/pkg/extensions/imagetrust/image_trust.go +++ b/pkg/extensions/imagetrust/image_trust.go @@ -140,7 +140,7 @@ func GetSecretsManagerRetrieval(region, endpoint string) *secretcache.Cache { return cache } -func (imgTrustStore *ImageTrustStore) ProtoVerifySignature( +func (imgTrustStore *ImageTrustStore) VerifySignature( signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData, repo string, ) (string, time.Time, bool, error) { @@ -166,7 +166,7 @@ func (imgTrustStore *ImageTrustStore) ProtoVerifySignature( } } -func (imgTrustStore *ImageTrustStore) VerifySignature( +func (imgTrustStore *ImageTrustStore) DepVerifySignature( signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, blob []byte, repo string, ) (string, time.Time, bool, error) { diff --git a/pkg/extensions/imagetrust/image_trust_disabled.go b/pkg/extensions/imagetrust/image_trust_disabled.go index e3959f0d39..5b1bf40eb6 100644 --- a/pkg/extensions/imagetrust/image_trust_disabled.go +++ b/pkg/extensions/imagetrust/image_trust_disabled.go @@ -21,14 +21,14 @@ func NewAWSImageTrustStore(region, endpoint string) (*imageTrustDisabled, error) type imageTrustDisabled struct{} -func (imgTrustStore *imageTrustDisabled) VerifySignature( +func (imgTrustStore *imageTrustDisabled) DepVerifySignature( signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, manifestContent []byte, repo string, ) (string, time.Time, bool, error) { return "", time.Time{}, false, nil } -func (imgTrustStore *imageTrustDisabled) ProtoVerifySignature( +func (imgTrustStore *imageTrustDisabled) VerifySignature( signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData mTypes.ImageData, repo string, ) (string, time.Time, bool, error) { diff --git a/pkg/extensions/imagetrust/image_trust_disabled_test.go b/pkg/extensions/imagetrust/image_trust_disabled_test.go index 1140f7cc70..a55373bc56 100644 --- a/pkg/extensions/imagetrust/image_trust_disabled_test.go +++ b/pkg/extensions/imagetrust/image_trust_disabled_test.go @@ -34,7 +34,7 @@ func TestImageTrust(t *testing.T) { localImgTrustStore, err := imagetrust.NewLocalImageTrustStore(rootDir) So(err, ShouldBeNil) - author, expTime, ok, err := localImgTrustStore.VerifySignature("cosign", + author, expTime, ok, err := localImgTrustStore.DepVerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo, ) So(author, ShouldBeEmpty) @@ -53,7 +53,7 @@ func TestImageTrust(t *testing.T) { ) So(err, ShouldBeNil) - author, expTime, ok, err = cloudImgTrustStore.VerifySignature("cosign", + author, expTime, ok, err = cloudImgTrustStore.DepVerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo, ) So(author, ShouldBeEmpty) diff --git a/pkg/extensions/imagetrust/image_trust_test.go b/pkg/extensions/imagetrust/image_trust_test.go index b83c1e42a9..c84cf61d2f 100644 --- a/pkg/extensions/imagetrust/image_trust_test.go +++ b/pkg/extensions/imagetrust/image_trust_test.go @@ -146,13 +146,20 @@ func TestInitCosignAndNotationDirs(t *testing.T) { }) } +func SkipTest(t *testing.T) { + t.Helper() + t.SkipNow() +} + func TestVerifySignatures(t *testing.T) { + SkipTest(t) + Convey("empty manifest digest", t, func() { image := CreateRandomImage() manifestContent := image.ManifestDescriptor.Data imgTrustStore := &imagetrust.ImageTrustStore{} - _, _, _, err := imgTrustStore.VerifySignature("", []byte(""), "", "", manifestContent, "repo") + _, _, _, err := imgTrustStore.DepVerifySignature("", []byte(""), "", "", manifestContent, "repo") So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrBadManifestDigest) }) @@ -163,7 +170,7 @@ func TestVerifySignatures(t *testing.T) { manifestDigest := image.ManifestDescriptor.Digest imgTrustStore := &imagetrust.ImageTrustStore{} - _, _, _, err := imgTrustStore.VerifySignature("wrongType", []byte(""), "", manifestDigest, manifestContent, "repo") + _, _, _, err := imgTrustStore.DepVerifySignature("wrongType", []byte(""), "", manifestDigest, manifestContent, "repo") So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrInvalidSignatureType) }) @@ -181,7 +188,7 @@ func TestVerifySignatures(t *testing.T) { CosignStorage: &imagetrust.PublicKeyLocalStorage{}, } - _, _, _, err := imgTrustStore.VerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo) + _, _, _, err := imgTrustStore.DepVerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrSignConfigDirNotSet) }) @@ -201,7 +208,7 @@ func TestVerifySignatures(t *testing.T) { CosignStorage: pubKeyStorage, } - _, _, _, err = imgTrustStore.VerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo) + _, _, _, err = imgTrustStore.DepVerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) }) @@ -221,7 +228,7 @@ func TestVerifySignatures(t *testing.T) { CosignStorage: pubKeyStorage, } - _, _, isTrusted, err := imgTrustStore.VerifySignature("cosign", []byte(""), "", manifestDigest, + _, _, isTrusted, err := imgTrustStore.DepVerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeFalse) @@ -309,7 +316,7 @@ func TestVerifySignatures(t *testing.T) { } // signature is trusted - author, _, isTrusted, err := imgTrustStore.VerifySignature("cosign", rawSignature, sigKey, manifestDigest, + author, _, isTrusted, err := imgTrustStore.DepVerifySignature("cosign", rawSignature, sigKey, manifestDigest, manifestContent, repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeTrue) @@ -329,7 +336,7 @@ func TestVerifySignatures(t *testing.T) { NotationStorage: &imagetrust.CertificateLocalStorage{}, } - _, _, _, err := imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, + _, _, _, err := imgTrustStore.DepVerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrSignConfigDirNotSet) @@ -345,7 +352,7 @@ func TestVerifySignatures(t *testing.T) { NotationStorage: certStorage, } - _, _, isTrusted, err := imgTrustStore.VerifySignature("notation", []byte(""), "", manifestDigest, + _, _, isTrusted, err := imgTrustStore.DepVerifySignature("notation", []byte(""), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) So(isTrusted, ShouldBeFalse) @@ -366,7 +373,7 @@ func TestVerifySignatures(t *testing.T) { NotationStorage: certStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, + _, _, _, err = imgTrustStore.DepVerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) }) @@ -388,7 +395,7 @@ func TestVerifySignatures(t *testing.T) { NotationStorage: certStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, + _, _, _, err = imgTrustStore.DepVerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) }) @@ -491,7 +498,7 @@ func TestVerifySignatures(t *testing.T) { } // signature is trusted - author, _, isTrusted, err := imgTrustStore.VerifySignature("notation", rawSignature, sigKey, manifestDigest, + author, _, isTrusted, err := imgTrustStore.DepVerifySignature("notation", rawSignature, sigKey, manifestDigest, manifestContent, repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeTrue) @@ -501,7 +508,7 @@ func TestVerifySignatures(t *testing.T) { So(err, ShouldBeNil) // signature is not trusted - author, _, isTrusted, err = imgTrustStore.VerifySignature("notation", rawSignature, sigKey, manifestDigest, + author, _, isTrusted, err = imgTrustStore.DepVerifySignature("notation", rawSignature, sigKey, manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) So(isTrusted, ShouldBeFalse) @@ -923,7 +930,7 @@ func TestAWSTrustStore(t *testing.T) { NotationStorage: notationStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, + _, _, _, err = imgTrustStore.DepVerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) }) @@ -956,7 +963,7 @@ func TestAWSTrustStore(t *testing.T) { NotationStorage: notationStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, + _, _, _, err = imgTrustStore.DepVerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) @@ -973,7 +980,7 @@ func TestAWSTrustStore(t *testing.T) { NotationStorage: notationStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, + _, _, _, err = imgTrustStore.DepVerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) @@ -990,7 +997,7 @@ func TestAWSTrustStore(t *testing.T) { NotationStorage: notationStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, + _, _, _, err = imgTrustStore.DepVerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, repo) So(err, ShouldNotBeNil) }) @@ -1230,7 +1237,7 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { imageTrustStore := ctlr.MetaDB.ImageTrustStore() // signature is trusted - author, _, isTrusted, err := imageTrustStore.VerifySignature("cosign", rawSignature, sigKey, manifestDigest, + author, _, isTrusted, err := imageTrustStore.DepVerifySignature("cosign", rawSignature, sigKey, manifestDigest, manifestContent, repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeTrue) @@ -1322,7 +1329,7 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { imageTrustStore := ctlr.MetaDB.ImageTrustStore() // signature is trusted - author, _, isTrusted, err := imageTrustStore.VerifySignature("notation", rawSignature, sigKey, manifestDigest, + author, _, isTrusted, err := imageTrustStore.DepVerifySignature("notation", rawSignature, sigKey, manifestDigest, manifestContent, repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeTrue) diff --git a/pkg/extensions/search/convert/convert_internal_test.go b/pkg/extensions/search/convert/convert_internal_test.go index 47bb5c974f..77afe6ae4f 100644 --- a/pkg/extensions/search/convert/convert_internal_test.go +++ b/pkg/extensions/search/convert/convert_internal_test.go @@ -71,7 +71,7 @@ func TestCVEConvert(t *testing.T) { So(imageSummary, ShouldBeNil) So(graphql.GetErrors(ctx), ShouldBeNil) - imageSummary, _, err = ProtoImageManifest2ImageSummary(ctx, "repo1", "0.1.0", repoMetaList[0], + imageSummary, _, err = ImageManifest2ImageSummary(ctx, "repo1", "0.1.0", repoMetaList[0], imageData[image.DigestStr()]) So(err, ShouldBeNil) @@ -162,7 +162,7 @@ func TestCVEConvert(t *testing.T) { So(repoSummary, ShouldBeNil) So(graphql.GetErrors(ctx), ShouldBeNil) - imageSummary, _, err := ProtoImageManifest2ImageSummary(ctx, "repo1", "0.1.0", repoMetaList[0], + imageSummary, _, err := ImageManifest2ImageSummary(ctx, "repo1", "0.1.0", repoMetaList[0], imageData[image.DigestStr()]) So(err, ShouldBeNil) @@ -220,7 +220,7 @@ func TestCVEConvert(t *testing.T) { So(manifestSummary, ShouldBeNil) So(graphql.GetErrors(ctx), ShouldBeNil) - imageSummary, _, err := ProtoImageManifest2ImageSummary(ctx, "repo1", "0.1.0", repoMetaList[0], + imageSummary, _, err := ImageManifest2ImageSummary(ctx, "repo1", "0.1.0", repoMetaList[0], imageData[image.DigestStr()]) So(err, ShouldBeNil) manifestSummary = imageSummary.Manifests[0] diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index f13bb3f9b1..871255db83 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -32,7 +32,7 @@ func TestConvertErrors(t *testing.T) { ctx := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - _, _, err := convert.ImageIndex2ImageSummary( + _, _, err := convert.DepImageIndex2ImageSummary( ctx, "repo", "tag", @@ -50,7 +50,7 @@ func TestConvertErrors(t *testing.T) { ctx := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - _, _, err := convert.ImageIndex2ImageSummary( + _, _, err := convert.DepImageIndex2ImageSummary( ctx, "repo", "tag", @@ -76,7 +76,7 @@ func TestConvertErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, err = convert.ImageManifest2ImageSummary( + _, _, err = convert.DepImageManifest2ImageSummary( ctx, "repo", "tag", @@ -95,7 +95,7 @@ func TestConvertErrors(t *testing.T) { graphql.DefaultErrorPresenter, graphql.DefaultRecover) // with bad config json, shouldn't error when unmarshaling - _, _, _, err := convert.ImageManifest2ManifestSummary( + _, _, _, err := convert.DepImageManifest2ManifestSummary( ctx, "repo", "tag", @@ -127,7 +127,7 @@ func TestConvertErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, _, _, err = convert.ImageManifest2ManifestSummary( + _, _, _, err = convert.DepImageManifest2ManifestSummary( ctx, "repo", "tag", @@ -155,7 +155,7 @@ func TestConvertErrors(t *testing.T) { graphql.DefaultErrorPresenter, graphql.DefaultRecover) // with bad config json, error while unmarshaling - _, imageSummaries := convert.RepoMeta2ExpandedRepoInfo( + _, imageSummaries := convert.DepRepoMeta2ExpandedRepoInfo( ctx, mTypes.DepRepoMetadata{ Tags: map[string]mTypes.Descriptor{ @@ -181,7 +181,7 @@ func TestConvertErrors(t *testing.T) { So(len(imageSummaries), ShouldEqual, 1) // cveInfo present no error - _, imageSummaries = convert.RepoMeta2ExpandedRepoInfo( + _, imageSummaries = convert.DepRepoMeta2ExpandedRepoInfo( ctx, mTypes.DepRepoMetadata{ Tags: map[string]mTypes.Descriptor{ @@ -307,7 +307,7 @@ func TestGetSignaturesInfo(t *testing.T) { }}}}, } - signaturesSummary := convert.GetSignaturesInfo(true, repoMeta, indexDigest) + signaturesSummary := convert.DepGetSignaturesInfo(true, repoMeta, indexDigest) So(signaturesSummary, ShouldNotBeEmpty) So(*signaturesSummary[0].Author, ShouldEqual, "author") So(*signaturesSummary[0].IsTrusted, ShouldEqual, true) @@ -330,7 +330,7 @@ func TestGetSignaturesInfo(t *testing.T) { }}}}, } - signaturesSummary := convert.GetSignaturesInfo(true, repoMeta, indexDigest) + signaturesSummary := convert.DepGetSignaturesInfo(true, repoMeta, indexDigest) So(signaturesSummary, ShouldNotBeEmpty) So(*signaturesSummary[0].Author, ShouldEqual, "author") So(*signaturesSummary[0].IsTrusted, ShouldEqual, false) @@ -495,7 +495,7 @@ func TestPaginatedConvert(t *testing.T) { Convey("PaginatedRepoMeta2RepoSummaries filtering and sorting", t, func() { // Test different combinations of the filter - reposSum, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries( + reposSum, pageInfo, err := convert.PaginatedDepRepoMeta2RepoSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -511,7 +511,7 @@ func TestPaginatedConvert(t *testing.T) { So(*reposSum[1].Name, ShouldResemble, "repo3-only-multiarch") So(pageInfo.ItemCount, ShouldEqual, 2) - reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( + reposSum, pageInfo, err = convert.PaginatedDepRepoMeta2RepoSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -526,7 +526,7 @@ func TestPaginatedConvert(t *testing.T) { So(len(reposSum), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) - reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( + reposSum, pageInfo, err = convert.PaginatedDepRepoMeta2RepoSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ HasToBeSigned: ref(true), @@ -539,7 +539,7 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.ItemCount, ShouldEqual, 1) // no filter - reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( + reposSum, pageInfo, err = convert.PaginatedDepRepoMeta2RepoSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticAsc}, ) @@ -553,7 +553,7 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.ItemCount, ShouldEqual, 5) // no filter opposite sorting - reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( + reposSum, pageInfo, err = convert.PaginatedDepRepoMeta2RepoSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticDsc}, ) @@ -567,7 +567,7 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.ItemCount, ShouldEqual, 5) // add pagination - reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( + reposSum, pageInfo, err = convert.PaginatedDepRepoMeta2RepoSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -583,7 +583,7 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.ItemCount, ShouldEqual, 1) So(pageInfo.TotalCount, ShouldEqual, 2) - reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( + reposSum, pageInfo, err = convert.PaginatedDepRepoMeta2RepoSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -601,7 +601,7 @@ func TestPaginatedConvert(t *testing.T) { }) Convey("PaginatedRepoMeta2ImageSummaries filtering and sorting", t, func() { - imgSum, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries( + imgSum, pageInfo, err := convert.PaginatedDepRepoMeta2ImageSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -624,7 +624,7 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.ItemCount, ShouldEqual, 5) // add page of size 2 - imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries( + imgSum, pageInfo, err = convert.PaginatedDepRepoMeta2ImageSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -642,7 +642,7 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.TotalCount, ShouldEqual, 5) // next page - imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries( + imgSum, pageInfo, err = convert.PaginatedDepRepoMeta2ImageSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -660,7 +660,7 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.TotalCount, ShouldEqual, 5) // last page - imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries( + imgSum, pageInfo, err = convert.PaginatedDepRepoMeta2ImageSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -676,7 +676,7 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.TotalCount, ShouldEqual, 5) // has to be signed - imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries( + imgSum, pageInfo, err = convert.PaginatedDepRepoMeta2ImageSummaries( ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, @@ -755,7 +755,7 @@ func TestIndexAnnotations(t *testing.T) { digest := indexWithAnnotations.Digest() - imageSummary, _, err := convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, repoMeta[0], + imageSummary, _, err := convert.DepImageIndex2ImageSummary(ctx, "repo", "tag", digest, repoMeta[0], indexData[digest.String()], manifestMetadata) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldResemble, "IndexDescription") @@ -777,7 +777,7 @@ func TestIndexAnnotations(t *testing.T) { }) digest = indexWithManifestAndConfigAnnotations.Digest() - imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, + imageSummary, _, err = convert.DepImageIndex2ImageSummary(ctx, "repo", "tag", digest, repoMeta[0], indexData[digest.String()], manifestMetadata) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldResemble, "ManifestDescription") @@ -798,7 +798,7 @@ func TestIndexAnnotations(t *testing.T) { }) digest = indexWithConfigAnnotations.Digest() - imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, + imageSummary, _, err = convert.DepImageIndex2ImageSummary(ctx, "repo", "tag", digest, repoMeta[0], indexData[digest.String()], manifestMetadata) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldResemble, "ConfigDescription") @@ -840,7 +840,7 @@ func TestIndexAnnotations(t *testing.T) { }) digest = indexWithMixAnnotations.Digest() - imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, + imageSummary, _, err = convert.DepImageIndex2ImageSummary(ctx, "repo", "tag", digest, repoMeta[0], indexData[digest.String()], manifestMetadata) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldResemble, "ConfigDescription") @@ -860,7 +860,7 @@ func TestIndexAnnotations(t *testing.T) { }) digest = indexWithNoAnnotations.Digest() - imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, + imageSummary, _, err = convert.DepImageIndex2ImageSummary(ctx, "repo", "tag", digest, repoMeta[0], indexData[digest.String()], manifestMetadata) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldBeBlank) @@ -872,92 +872,3 @@ func TestIndexAnnotations(t *testing.T) { So(*imageSummary.Source, ShouldBeBlank) }) } - -func TestDownloadCount(t *testing.T) { - Convey("manifest", t, func() { - repoMeta, manifestMetaMap, indexDataMap := ociutils.GetMetadataForRepos( - ociutils.Repo{ - Name: "repo", - Images: []ociutils.RepoImage{ - { - Image: CreateRandomImage(), - Tag: "10-downloads", - Statistics: mTypes.DescriptorStatistics{ - DownloadCount: 10, - }, - }, - }, - }, - ) - - repoSummary := convert.RepoMeta2RepoSummary(context.Background(), repoMeta[0], manifestMetaMap, indexDataMap) - So(*repoSummary.DownloadCount, ShouldEqual, 10) - So(*repoSummary.NewestImage.DownloadCount, ShouldEqual, 10) - }) - - Convey("index", t, func() { - img1, img2, img3 := CreateRandomImage(), CreateRandomImage(), CreateRandomImage() - multiArch := CreateMultiarchWith().Images([]Image{img1, img2, img3}).Build() - - repoMeta, manifestMetaMap, indexDataMap := ociutils.GetMetadataForRepos( - ociutils.Repo{ - Name: "repo", - MultiArchImages: []ociutils.RepoMultiArchImage{ - { - MultiarchImage: multiArch, - Tag: "160-multiarch", - ImageStatistics: map[string]mTypes.DescriptorStatistics{ - img1.DigestStr(): {DownloadCount: 10}, - img2.DigestStr(): {DownloadCount: 20}, - img3.DigestStr(): {DownloadCount: 30}, - multiArch.DigestStr(): {DownloadCount: 100}, - }, - }, - }, - }, - ) - - repoSummary := convert.RepoMeta2RepoSummary(context.Background(), repoMeta[0], manifestMetaMap, indexDataMap) - So(*repoSummary.DownloadCount, ShouldEqual, 100) - So(*repoSummary.NewestImage.DownloadCount, ShouldEqual, 100) - }) - - Convey("index + manifest mixed", t, func() { - img1 := CreateRandomImage() - img2 := CreateRandomImage() - img3 := CreateImageWith().DefaultLayers().ImageConfig( - ispec.Image{Created: DateRef(2020, 1, 1, 1, 1, 1, 0, time.UTC)}, - ).Build() - - multiArch := CreateMultiarchWith().Images([]Image{img1, img2, img3}).Build() - - repoMeta, manifestMetaMap, indexDataMap := ociutils.GetMetadataForRepos( - ociutils.Repo{ - Name: "repo", - Images: []ociutils.RepoImage{ - { - Image: CreateRandomImage(), - Tag: "5-downloads", - Statistics: mTypes.DescriptorStatistics{DownloadCount: 5}, - }, - }, - MultiArchImages: []ociutils.RepoMultiArchImage{ - { - MultiarchImage: multiArch, - Tag: "160-multiarch", - ImageStatistics: map[string]mTypes.DescriptorStatistics{ - img1.DigestStr(): {DownloadCount: 10}, - img2.DigestStr(): {DownloadCount: 20}, - img3.DigestStr(): {DownloadCount: 30}, - multiArch.DigestStr(): {DownloadCount: 100}, - }, - }, - }, - }, - ) - - repoSummary := convert.RepoMeta2RepoSummary(context.Background(), repoMeta[0], manifestMetaMap, indexDataMap) - So(*repoSummary.DownloadCount, ShouldEqual, 105) - So(*repoSummary.NewestImage.DownloadCount, ShouldEqual, 100) - }) -} diff --git a/pkg/extensions/search/convert/metadb.go b/pkg/extensions/search/convert/metadb.go index 070cc541d2..cb15859ee5 100644 --- a/pkg/extensions/search/convert/metadb.go +++ b/pkg/extensions/search/convert/metadb.go @@ -2,11 +2,8 @@ package convert import ( "context" - "encoding/json" - "fmt" "sort" "strconv" - "strings" "time" "github.com/99designs/gqlgen/graphql" @@ -20,7 +17,6 @@ import ( "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/extensions/search/pagination" "zotregistry.io/zot/pkg/log" - mcommon "zotregistry.io/zot/pkg/meta/common" mTypes "zotregistry.io/zot/pkg/meta/types" ) @@ -28,120 +24,6 @@ type SkipQGLField struct { Vulnerabilities bool } -func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.DepRepoMetadata, - manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, -) *gql_generated.RepoSummary { - var ( - repoName = repoMeta.Name - repoLastUpdatedTimestamp = time.Time{} - repoPlatformsSet = map[string]*gql_generated.Platform{} - repoVendorsSet = map[string]bool{} - lastUpdatedImageSummary *gql_generated.ImageSummary - repoDownloadCount = 0 - repoStarCount = repoMeta.Stars // total number of stars - repoIsUserStarred = repoMeta.IsStarred // value specific to the current user - repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user - - // map used to keep track of all blobs of a repo without duplicates as - // some images may have the same layers - repoBlob2Size = make(map[string]int64, 10) - - // made up of all manifests, configs and image layers - size = int64(0) - ) - - for tag, descriptor := range repoMeta.Tags { - imageSummary, imageBlobsMap, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, - tag, repoMeta, manifestMetaMap, indexDataMap, - ) - if err != nil { - continue - } - - for blobDigest, blobSize := range imageBlobsMap { - repoBlob2Size[blobDigest] = blobSize - } - - for _, manifestSummary := range imageSummary.Manifests { - if *manifestSummary.Platform.Os != "" || *manifestSummary.Platform.Arch != "" { - opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch - - platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch)) - repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch} - } - } - - repoDownloadCount += *imageSummary.DownloadCount - - if *imageSummary.Vendor != "" { - repoVendorsSet[*imageSummary.Vendor] = true - } - - lastUpdatedImageSummary = UpdateLastUpdatedTimestamp(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary) - } - - // calculate repo size = sum all manifest, config and layer blobs sizes - for _, blobSize := range repoBlob2Size { - size += blobSize - } - - repoSize := strconv.FormatInt(size, 10) - - repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) - - for _, platform := range repoPlatformsSet { - repoPlatforms = append(repoPlatforms, platform) - } - - repoVendors := make([]*string, 0, len(repoVendorsSet)) - - for vendor := range repoVendorsSet { - vendor := vendor - repoVendors = append(repoVendors, &vendor) - } - - return &gql_generated.RepoSummary{ - Name: &repoName, - LastUpdated: &repoLastUpdatedTimestamp, - Size: &repoSize, - Platforms: repoPlatforms, - Vendors: repoVendors, - NewestImage: lastUpdatedImageSummary, - DownloadCount: &repoDownloadCount, - StarCount: &repoStarCount, - IsBookmarked: &repoIsUserBookMarked, - IsStarred: &repoIsUserStarred, - Rank: &repoMeta.Rank, - } -} - -func PaginatedRepoMeta2RepoSummaries(ctx context.Context, reposMeta []mTypes.DepRepoMetadata, - manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, - skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, -) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) { - reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) - if err != nil { - return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err - } - - for _, repoMeta := range reposMeta { - repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap) - - if RepoSumAcceptedByFilter(repoSummary, filter) { - reposPageFinder.Add(repoSummary) - } - } - - page, pageInfo := reposPageFinder.Page() - - // CVE scanning is expensive, only scan for the current page - for _, repoSummary := range page { - updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo) - } - - return page, pageInfo, nil -} - func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time, lastUpdatedImageSummary *gql_generated.ImageSummary, imageSummary *gql_generated.ImageSummary, ) *gql_generated.ImageSummary { @@ -159,224 +41,6 @@ func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time, return newLastUpdatedImageSummary } -func Descriptor2ImageSummary(ctx context.Context, descriptor mTypes.Descriptor, repo, tag string, - repoMeta mTypes.DepRepoMetadata, manifestMetaMap map[string]mTypes.DepManifestMetadata, - indexDataMap map[string]mTypes.DepIndexData, -) (*gql_generated.ImageSummary, map[string]int64, error) { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - return ImageManifest2ImageSummary(ctx, repo, tag, godigest.Digest(descriptor.Digest), - repoMeta, manifestMetaMap[descriptor.Digest]) - case ispec.MediaTypeImageIndex: - return ImageIndex2ImageSummary(ctx, repo, tag, godigest.Digest(descriptor.Digest), - repoMeta, indexDataMap[descriptor.Digest], manifestMetaMap) - default: - return &gql_generated.ImageSummary{}, map[string]int64{}, zerr.ErrMediaTypeNotSupported - } -} - -// Deprecated: ImageIndex2ImageSummary. -func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest godigest.Digest, - repoMeta mTypes.DepRepoMetadata, indexData mTypes.DepIndexData, manifestMetaMap map[string]mTypes.DepManifestMetadata, -) (*gql_generated.ImageSummary, map[string]int64, error) { - var indexContent ispec.Index - - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return &gql_generated.ImageSummary{}, map[string]int64{}, err - } - - var ( - indexLastUpdated time.Time - isSigned bool - totalIndexSize int64 - indexSize string - totalDownloadCount int - manifestAnnotations *ImageAnnotations - manifestSummaries = make([]*gql_generated.ManifestSummary, 0, len(indexContent.Manifests)) - indexBlobs = make(map[string]int64, 0) - - indexDigestStr = indexDigest.String() - indexMediaType = ispec.MediaTypeImageIndex - ) - - for _, descriptor := range indexContent.Manifests { - manifestSummary, manifestBlobs, annotations, err := ImageManifest2ManifestSummary(ctx, repo, tag, descriptor, - repoMeta, manifestMetaMap[descriptor.Digest.String()], repoMeta.Referrers[descriptor.Digest.String()]) - if err != nil { - return &gql_generated.ImageSummary{}, map[string]int64{}, err - } - - manifestSize := int64(0) - - for digest, size := range manifestBlobs { - indexBlobs[digest] = size - manifestSize += size - } - - if indexLastUpdated.Before(*manifestSummary.LastUpdated) { - indexLastUpdated = *manifestSummary.LastUpdated - } - - if manifestAnnotations == nil { - manifestAnnotations = annotations - } - - totalIndexSize += manifestSize - - manifestSummaries = append(manifestSummaries, manifestSummary) - } - - totalDownloadCount += repoMeta.Statistics[indexDigestStr].DownloadCount - - for _, signatures := range repoMeta.Signatures[indexDigest.String()] { - if len(signatures) > 0 { - isSigned = true - } - } - - indexSize = strconv.FormatInt(totalIndexSize, 10) - - signaturesInfo := GetSignaturesInfo(isSigned, repoMeta, indexDigest) - - if manifestAnnotations == nil { - // The index doesn't have manifests - manifestAnnotations = &ImageAnnotations{} - } - - annotations := GetIndexAnnotations(indexContent.Annotations, manifestAnnotations) - - indexSummary := gql_generated.ImageSummary{ - RepoName: &repo, - Tag: &tag, - Digest: &indexDigestStr, - MediaType: &indexMediaType, - Manifests: manifestSummaries, - LastUpdated: &indexLastUpdated, - IsSigned: &isSigned, - SignatureInfo: signaturesInfo, - Size: &indexSize, - DownloadCount: &totalDownloadCount, - Description: &annotations.Description, - Title: &annotations.Title, - Documentation: &annotations.Documentation, - Licenses: &annotations.Licenses, - Labels: &annotations.Labels, - Source: &annotations.Source, - Vendor: &annotations.Vendor, - Authors: &annotations.Authors, - Referrers: getReferrers(repoMeta.Referrers[indexDigest.String()]), - } - - return &indexSummary, indexBlobs, nil -} - -func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest godigest.Digest, - repoMeta mTypes.DepRepoMetadata, manifestMeta mTypes.DepManifestMetadata, -) (*gql_generated.ImageSummary, map[string]int64, error) { - var ( - manifestContent ispec.Manifest - manifestDigest = digest.String() - mediaType = ispec.MediaTypeImageManifest - ) - - err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent) - if err != nil { - graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+ - "error: %s", repo, tag, manifestDigest, err.Error())) - - return &gql_generated.ImageSummary{}, map[string]int64{}, err - } - - configContent := mcommon.InitializeImageConfig(manifestMeta.ConfigBlob) - - var ( - repoName = repo - configDigest = manifestContent.Config.Digest.String() - configSize = manifestContent.Config.Size - artifactType = zcommon.GetManifestArtifactType(manifestContent) - imageLastUpdated = zcommon.GetImageLastUpdated(configContent) - downloadCount = repoMeta.Statistics[digest.String()].DownloadCount - isSigned = false - ) - - opSys := configContent.OS - arch := configContent.Architecture - variant := configContent.Variant - - if variant != "" { - arch = arch + "/" + variant - } - - platform := gql_generated.Platform{Os: &opSys, Arch: &arch} - - for _, signatures := range repoMeta.Signatures[digest.String()] { - if len(signatures) > 0 { - isSigned = true - } - } - - size, imageBlobsMap := getImageBlobsInfo( - manifestDigest, int64(len(manifestMeta.ManifestBlob)), - configDigest, configSize, - manifestContent.Layers) - imageSize := strconv.FormatInt(size, 10) - - annotations := GetAnnotations(manifestContent.Annotations, configContent.Config.Labels) - - authors := annotations.Authors - if authors == "" { - authors = configContent.Author - } - - historyEntries, err := getAllHistory(manifestContent, configContent) - if err != nil { - graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ - "manifest digest: %s, error: %s", tag, repo, manifestDigest, err.Error())) - } - - signaturesInfo := GetSignaturesInfo(isSigned, repoMeta, digest) - - manifestSummary := gql_generated.ManifestSummary{ - Digest: &manifestDigest, - ConfigDigest: &configDigest, - LastUpdated: &imageLastUpdated, - Size: &imageSize, - IsSigned: &isSigned, - SignatureInfo: signaturesInfo, - Platform: &platform, - DownloadCount: &downloadCount, - Layers: getLayersSummaries(manifestContent), - History: historyEntries, - Referrers: getReferrers(repoMeta.Referrers[manifestDigest]), - ArtifactType: &artifactType, - } - - imageSummary := gql_generated.ImageSummary{ - RepoName: &repoName, - Tag: &tag, - Digest: &manifestDigest, - MediaType: &mediaType, - Manifests: []*gql_generated.ManifestSummary{&manifestSummary}, - LastUpdated: &imageLastUpdated, - IsSigned: &isSigned, - SignatureInfo: signaturesInfo, - Size: &imageSize, - DownloadCount: &downloadCount, - Description: &annotations.Description, - Title: &annotations.Title, - Documentation: &annotations.Documentation, - Licenses: &annotations.Licenses, - Labels: &annotations.Labels, - Source: &annotations.Source, - Vendor: &annotations.Vendor, - Authors: &authors, - Referrers: getReferrers(repoMeta.Referrers[manifestDigest]), - } - - return &imageSummary, imageBlobsMap, nil -} - func getReferrers(referrersInfo []mTypes.ReferrerInfo) []*gql_generated.Referrer { referrers := make([]*gql_generated.Referrer, 0, len(referrersInfo)) @@ -411,84 +75,6 @@ func getAnnotationsFromMap(annotationsMap map[string]string) []*gql_generated.An return annotations } -func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descriptor ispec.Descriptor, - repoMeta mTypes.DepRepoMetadata, manifestMeta mTypes.DepManifestMetadata, - referrersInfo []mTypes.ReferrerInfo, -) (*gql_generated.ManifestSummary, map[string]int64, *ImageAnnotations, error) { - var ( - manifestContent ispec.Manifest - digest = descriptor.Digest - ) - - err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent) - if err != nil { - graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+ - "error: %s", repo, tag, digest, err.Error())) - - return &gql_generated.ManifestSummary{}, map[string]int64{}, &ImageAnnotations{}, err - } - - configContent := mcommon.InitializeImageConfig(manifestMeta.ConfigBlob) - annotations := GetAnnotations(manifestContent.Annotations, configContent.Config.Labels) - - var ( - manifestDigestStr = digest.String() - configDigest = manifestContent.Config.Digest.String() - configSize = manifestContent.Config.Size - artifactType = zcommon.GetManifestArtifactType(manifestContent) - imageLastUpdated = zcommon.GetImageLastUpdated(configContent) - downloadCount = repoMeta.Statistics[digest.String()].DownloadCount - isSigned = false - ) - - opSys := configContent.OS - arch := configContent.Architecture - variant := configContent.Variant - - if variant != "" { - arch = arch + "/" + variant - } - - platform := gql_generated.Platform{Os: &opSys, Arch: &arch} - - size, imageBlobsMap := getImageBlobsInfo( - manifestDigestStr, int64(len(manifestMeta.ManifestBlob)), - configDigest, configSize, - manifestContent.Layers) - imageSize := strconv.FormatInt(size, 10) - - historyEntries, err := getAllHistory(manifestContent, configContent) - if err != nil { - graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ - "manifest digest: %s, error: %s", tag, repo, manifestDigestStr, err.Error())) - } - - for _, signatures := range repoMeta.Signatures[manifestDigestStr] { - if len(signatures) > 0 { - isSigned = true - } - } - - signaturesInfo := GetSignaturesInfo(isSigned, repoMeta, digest) - - manifestSummary := gql_generated.ManifestSummary{ - Digest: &manifestDigestStr, - ConfigDigest: &configDigest, - LastUpdated: &imageLastUpdated, - Size: &imageSize, - Platform: &platform, - DownloadCount: &downloadCount, - Layers: getLayersSummaries(manifestContent), - History: historyEntries, - IsSigned: &isSigned, - SignatureInfo: signaturesInfo, - Referrers: getReferrers(referrersInfo), - ArtifactType: &artifactType, - } - - return &manifestSummary, imageBlobsMap, &annotations, nil -} - func getImageBlobsInfo(manifestDigest string, manifestSize int64, configDigest string, configSize int64, layers []ispec.Descriptor, ) (int64, map[string]int64) { @@ -512,42 +98,7 @@ func getImageBlobsInfo(manifestDigest string, manifestSize int64, configDigest s return imageSize, imageBlobsMap } -func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.DepRepoMetadata, - manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, - skip SkipQGLField, cveInfo cveinfo.CveInfo, -) []*gql_generated.ImageSummary { - imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) - - // Make sure the tags are sorted - // We need to implement a proper fix for this taking into account - // the sorting criteria used in the requested page - tags := make([]string, 0, len(repoMeta.Tags)) - for tag := range repoMeta.Tags { - tags = append(tags, tag) - } - - // Sorting ascending by tag name should do for now - sort.Strings(tags) - - for _, tag := range tags { - descriptor := repoMeta.Tags[tag] - - imageSummary, _, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, - repoMeta, manifestMetaMap, indexDataMap) - if err != nil { - continue - } - - // CVE scanning is expensive, only scan for final slice of results - updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) - - imageSummaries = append(imageSummaries, imageSummary) - } - - return imageSummaries -} - -func ProtoRepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.FullRepoMetadata, +func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.FullRepoMetadata, imageData map[string]mTypes.ImageData, skip SkipQGLField, cveInfo cveinfo.CveInfo, ) []*gql_generated.ImageSummary { imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) @@ -566,7 +117,7 @@ func ProtoRepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.FullRepoM for _, tag := range tags { descriptor := repoMeta.Tags[tag] - imageSummary, _, err := ProtoDescriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, + imageSummary, _, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, repoMeta, imageData) if err != nil { continue @@ -581,137 +132,6 @@ func ProtoRepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.FullRepoM return imageSummaries } -func PaginatedRepoMeta2ImageSummaries(ctx context.Context, reposMeta []mTypes.DepRepoMetadata, - manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, - skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, -) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) { - imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) - if err != nil { - return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err - } - - for _, repoMeta := range reposMeta { - for tag := range repoMeta.Tags { - descriptor := repoMeta.Tags[tag] - - imageSummary, _, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, - repoMeta, manifestMetaMap, indexDataMap) - if err != nil { - continue - } - - if ImgSumAcceptedByFilter(imageSummary, filter) { - imagePageFinder.Add(imageSummary) - } - } - } - - page, pageInfo := imagePageFinder.Page() - - for _, imageSummary := range page { - // CVE scanning is expensive, only scan for this page - updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) - } - - return page, pageInfo, nil -} - -func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.DepRepoMetadata, - manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, - skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger, -) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) { - var ( - repoName = repoMeta.Name - repoLastUpdatedTimestamp = time.Time{} - repoPlatformsSet = map[string]*gql_generated.Platform{} - repoVendorsSet = map[string]bool{} - lastUpdatedImageSummary *gql_generated.ImageSummary - repoDownloadCount = 0 - repoStarCount = repoMeta.Stars // total number of stars - isStarred = repoMeta.IsStarred // value specific to the current user - isBookmarked = repoMeta.IsBookmarked // value specific to the current user - - // map used to keep track of all blobs of a repo without duplicates as - // some images may have the same layers - repoBlob2Size = make(map[string]int64, 10) - - // made up of all manifests, configs and image layers - size = int64(0) - - imageSummaries = make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) - ) - - for tag, descriptor := range repoMeta.Tags { - imageSummary, imageBlobs, err := Descriptor2ImageSummary(ctx, descriptor, repoName, tag, - repoMeta, manifestMetaMap, indexDataMap) - if err != nil { - log.Error().Str("repository", repoName).Str("reference", tag). - Msg("metadb: error while converting descriptor for image") - - continue - } - - for _, manifestSummary := range imageSummary.Manifests { - opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch - if opSys != "" || arch != "" { - platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch)) - repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch} - } - - updateRepoBlobsMap(imageBlobs, repoBlob2Size) - } - - if *imageSummary.Vendor != "" { - repoVendorsSet[*imageSummary.Vendor] = true - } - - updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) - - lastUpdatedImageSummary = UpdateLastUpdatedTimestamp(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary) - - repoDownloadCount += *imageSummary.DownloadCount - - imageSummaries = append(imageSummaries, imageSummary) - } - - // calculate repo size = sum all manifest, config and layer blobs sizes - for _, blobSize := range repoBlob2Size { - size += blobSize - } - - repoSize := strconv.FormatInt(size, 10) - - repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) - - for _, platform := range repoPlatformsSet { - repoPlatforms = append(repoPlatforms, platform) - } - - repoVendors := make([]*string, 0, len(repoVendorsSet)) - - for vendor := range repoVendorsSet { - vendor := vendor - repoVendors = append(repoVendors, &vendor) - } - - summary := &gql_generated.RepoSummary{ - Name: &repoName, - LastUpdated: &repoLastUpdatedTimestamp, - Size: &repoSize, - Platforms: repoPlatforms, - Vendors: repoVendors, - NewestImage: lastUpdatedImageSummary, - DownloadCount: &repoDownloadCount, - StarCount: &repoStarCount, - IsBookmarked: &isBookmarked, - IsStarred: &isStarred, - } - - updateRepoSummaryVulnerabilities(ctx, summary, skip, cveInfo) - - return summary, imageSummaries -} - func FullRepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.FullRepoMetadata, imageDataMap map[string]mTypes.ImageData, skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger, ) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) { @@ -811,48 +231,7 @@ func GetPreloadString(prefix, name string) string { return name } -func GetSignaturesInfo(isSigned bool, repoMeta mTypes.DepRepoMetadata, indexDigest godigest.Digest, -) []*gql_generated.SignatureSummary { - signaturesInfo := []*gql_generated.SignatureSummary{} - - if !isSigned { - return signaturesInfo - } - - for sigType, signatures := range repoMeta.Signatures[indexDigest.String()] { - for _, sig := range signatures { - for _, layer := range sig.LayersInfo { - var ( - isTrusted bool - author string - tool string - ) - - if layer.Signer != "" { - author = layer.Signer - - if !layer.Date.IsZero() && time.Now().After(layer.Date) { - isTrusted = false - } else { - isTrusted = true - } - } else { - isTrusted = false - author = "" - } - - tool = sigType - - signaturesInfo = append(signaturesInfo, - &gql_generated.SignatureSummary{Tool: &tool, IsTrusted: &isTrusted, Author: &author}) - } - } - } - - return signaturesInfo -} - -func ProtoGetSignaturesInfo(isSigned bool, repoSignatures map[string]mTypes.ManifestSignatures, +func GetSignaturesInfo(isSigned bool, repoSignatures map[string]mTypes.ManifestSignatures, indexDigest godigest.Digest, ) []*gql_generated.SignatureSummary { signaturesInfo := []*gql_generated.SignatureSummary{} @@ -894,8 +273,7 @@ func ProtoGetSignaturesInfo(isSigned bool, repoSignatures map[string]mTypes.Mani return signaturesInfo } -func FullGetSignaturesInfo(isSigned bool, signatures mTypes.ManifestSignatures, indexDigest godigest.Digest, -) []*gql_generated.SignatureSummary { +func FullGetSignaturesInfo(isSigned bool, signatures mTypes.ManifestSignatures) []*gql_generated.SignatureSummary { signaturesInfo := []*gql_generated.SignatureSummary{} if !isSigned { @@ -935,33 +313,6 @@ func FullGetSignaturesInfo(isSigned bool, signatures mTypes.ManifestSignatures, return signaturesInfo } -func PaginatedProtoRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []mTypes.FullRepoMetadata, - imageDataMap map[string]mTypes.ImageData, filter mTypes.Filter, pageInput pagination.PageInput, - cveInfo cveinfo.CveInfo, skip SkipQGLField, -) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) { - reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) - if err != nil { - return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err - } - - for _, repoMeta := range repoMetaList { - repoSummary := ProtoRepoMeta2RepoSummary(ctx, repoMeta, imageDataMap) - - if RepoSumAcceptedByFilter(repoSummary, filter) { - reposPageFinder.Add(repoSummary) - } - } - - page, pageInfo := reposPageFinder.Page() - - // CVE scanning is expensive, only scan for the current page - for _, repoSummary := range page { - updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo) - } - - return page, pageInfo, nil -} - func PaginatedFullRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []mTypes.FullRepoMetadata, imageDataMap map[string]mTypes.ImageData, filter mTypes.Filter, pageInput pagination.PageInput, cveInfo cveinfo.CveInfo, skip SkipQGLField, @@ -989,93 +340,6 @@ func PaginatedFullRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []mTy return page, pageInfo, nil } -func ProtoRepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.FullRepoMetadata, - imageDataMap map[string]mTypes.ImageData, -) *gql_generated.RepoSummary { - var ( - repoName = repoMeta.Name - repoLastUpdatedTimestamp = time.Time{} - repoPlatformsSet = map[string]*gql_generated.Platform{} - repoVendorsSet = map[string]bool{} - lastUpdatedImageSummary *gql_generated.ImageSummary - repoDownloadCount = 0 - repoStarCount = repoMeta.StarCount // total number of stars - repoIsUserStarred = repoMeta.IsStarred // value specific to the current user - repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user - - // map used to keep track of all blobs of a repo without duplicates as - // some images may have the same layers - repoBlob2Size = make(map[string]int64, 10) - - // made up of all manifests, configs and image layers - size = int64(0) - ) - - for tag, descriptor := range repoMeta.Tags { - imageSummary, imageBlobsMap, err := ProtoDescriptor2ImageSummary(ctx, descriptor, repoMeta.Name, - tag, repoMeta, imageDataMap, - ) - if err != nil { - continue - } - - for blobDigest, blobSize := range imageBlobsMap { - repoBlob2Size[blobDigest] = blobSize - } - - for _, manifestSummary := range imageSummary.Manifests { - if *manifestSummary.Platform.Os != "" || *manifestSummary.Platform.Arch != "" { - opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch - - platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch)) - repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch} - } - } - - repoDownloadCount += *imageSummary.DownloadCount - - if *imageSummary.Vendor != "" { - repoVendorsSet[*imageSummary.Vendor] = true - } - - lastUpdatedImageSummary = UpdateLastUpdatedTimestamp(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary) - } - - // calculate repo size = sum all manifest, config and layer blobs sizes - for _, blobSize := range repoBlob2Size { - size += blobSize - } - - repoSize := strconv.FormatInt(size, 10) - - repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) - - for _, platform := range repoPlatformsSet { - repoPlatforms = append(repoPlatforms, platform) - } - - repoVendors := make([]*string, 0, len(repoVendorsSet)) - - for vendor := range repoVendorsSet { - vendor := vendor - repoVendors = append(repoVendors, &vendor) - } - - return &gql_generated.RepoSummary{ - Name: &repoName, - LastUpdated: &repoLastUpdatedTimestamp, - Size: &repoSize, - Platforms: repoPlatforms, - Vendors: repoVendors, - NewestImage: lastUpdatedImageSummary, - DownloadCount: &repoDownloadCount, - StarCount: &repoStarCount, - IsBookmarked: &repoIsUserBookMarked, - IsStarred: &repoIsUserStarred, - Rank: ref(repoMeta.Rank), - } -} - func FullRepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.FullRepoMetadata, imageDataMap map[string]mTypes.ImageData, ) *gql_generated.RepoSummary { @@ -1169,14 +433,14 @@ type ( BlobDigest = string ) -func ProtoDescriptor2ImageSummary(ctx context.Context, descriptor mTypes.Descriptor, repo, tag string, +func Descriptor2ImageSummary(ctx context.Context, descriptor mTypes.Descriptor, repo, tag string, repoMeta mTypes.FullRepoMetadata, imageDataMap map[ManifestDigest]mTypes.ImageData, ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { switch descriptor.MediaType { case ispec.MediaTypeImageManifest: - return ProtoImageManifest2ImageSummary(ctx, repo, tag, repoMeta, imageDataMap[descriptor.Digest]) + return ImageManifest2ImageSummary(ctx, repo, tag, repoMeta, imageDataMap[descriptor.Digest]) case ispec.MediaTypeImageIndex: - return ProtoImageIndex2ImageSummary(ctx, repo, tag, descriptor.Digest, repoMeta, imageDataMap) + return ImageIndex2ImageSummary(ctx, repo, tag, descriptor.Digest, repoMeta, imageDataMap) default: return nil, nil, zerr.ErrMediaTypeNotSupported } @@ -1194,7 +458,7 @@ func FullImageData2ImageSummary(ctx context.Context, imageData mTypes.FullImageD } } -func ProtoImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, repoMeta mTypes.FullRepoMetadata, +func ImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, repoMeta mTypes.FullRepoMetadata, imageDataMap map[ManifestDigest]mTypes.ImageData, ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { imageIndex := imageDataMap[digest] @@ -1212,7 +476,7 @@ func ProtoImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, ) for _, imageManifest := range imageIndex.Manifests { - imageManifestSummary, manifestBlobs, err := ProtoImageManifest2ImageSummary(ctx, repo, tag, repoMeta, + imageManifestSummary, manifestBlobs, err := ImageManifest2ImageSummary(ctx, repo, tag, repoMeta, mTypes.ImageData{ MediaType: ispec.MediaTypeImageManifest, Digest: imageManifest.Digest, @@ -1245,7 +509,7 @@ func ProtoImageIndex2ImageSummary(ctx context.Context, repo, tag, digest string, manifestSummaries = append(manifestSummaries, imageManifestSummary.Manifests[0]) } - signaturesInfo := ProtoGetSignaturesInfo(isSigned, repoMeta.Signatures, godigest.Digest(digest)) + signaturesInfo := FullGetSignaturesInfo(isSigned, repoMeta.Signatures[digest]) if manifestAnnotations == nil { // The index doesn't have manifests @@ -1332,7 +596,7 @@ func FullImageIndex2ImageSummary(ctx context.Context, imageData mTypes.FullImage manifestSummaries = append(manifestSummaries, imageManifestSummary.Manifests[0]) } - signaturesInfo := FullGetSignaturesInfo(isSigned, imageData.Signatures, imageData.Digest) + signaturesInfo := FullGetSignaturesInfo(isSigned, imageData.Signatures) if manifestAnnotations == nil { manifestAnnotations = &ImageAnnotations{} @@ -1365,7 +629,7 @@ func FullImageIndex2ImageSummary(ctx context.Context, imageData mTypes.FullImage return &indexSummary, indexBlobs, nil } -func ProtoImageManifest2ImageSummary(ctx context.Context, repo, tag string, repoMeta mTypes.FullRepoMetadata, +func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, repoMeta mTypes.FullRepoMetadata, imageData mTypes.ImageData, ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { manifest := imageData.Manifests[0] @@ -1399,7 +663,7 @@ func ProtoImageManifest2ImageSummary(ctx context.Context, repo, tag string, repo "manifest digest: %s, error: %s", tag, repo, manifest.Digest, err.Error())) } - signaturesInfo := ProtoGetSignaturesInfo(isSigned, repoMeta.Signatures, manifest.Digest) + signaturesInfo := FullGetSignaturesInfo(isSigned, repoMeta.Signatures[manifest.Digest.String()]) manifestSummary := gql_generated.ManifestSummary{ Digest: &manifestDigest, @@ -1475,7 +739,7 @@ func FullImageManifest2ImageSummary(ctx context.Context, imageData mTypes.FullIm "manifest digest: %s, error: %s", tag, repoName, manifest.Digest, err.Error())) } - signaturesInfo := FullGetSignaturesInfo(isSigned, imageData.Signatures, manifest.Digest) + signaturesInfo := FullGetSignaturesInfo(isSigned, imageData.Signatures) manifestSummary := gql_generated.ManifestSummary{ Digest: &manifestDigest, @@ -1548,41 +812,6 @@ func ref[T any](val T) *T { return &ref } -func PaginatedProtoRepoMeta2ImageSummaries(ctx context.Context, reposMeta []mTypes.FullRepoMetadata, - imageData map[string]mTypes.ImageData, skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, - pageInput pagination.PageInput, -) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) { - imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) - if err != nil { - return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err - } - - for _, repoMeta := range reposMeta { - for tag := range repoMeta.Tags { - descriptor := repoMeta.Tags[tag] - - imageSummary, _, err := ProtoDescriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, - repoMeta, imageData) - if err != nil { - continue - } - - if ImgSumAcceptedByFilter(imageSummary, filter) { - imagePageFinder.Add(imageSummary) - } - } - } - - page, pageInfo := imagePageFinder.Page() - - for _, imageSummary := range page { - // CVE scanning is expensive, only scan for this page - updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) - } - - return page, pageInfo, nil -} - func PaginatedFullImageData2ImageSummaries(ctx context.Context, imageDataList []mTypes.FullImageData, skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, ) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) { diff --git a/pkg/extensions/search/convert/metadb_deprecated.go b/pkg/extensions/search/convert/metadb_deprecated.go new file mode 100644 index 0000000000..23eaf13641 --- /dev/null +++ b/pkg/extensions/search/convert/metadb_deprecated.go @@ -0,0 +1,606 @@ +package convert + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/99designs/gqlgen/graphql" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/vektah/gqlparser/v2/gqlerror" + + zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" + cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" + "zotregistry.io/zot/pkg/extensions/search/gql_generated" + "zotregistry.io/zot/pkg/extensions/search/pagination" + "zotregistry.io/zot/pkg/log" + mcommon "zotregistry.io/zot/pkg/meta/common" + mTypes "zotregistry.io/zot/pkg/meta/types" +) + +func PaginatedDepRepoMeta2RepoSummaries(ctx context.Context, reposMeta []mTypes.DepRepoMetadata, + manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, + skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, +) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) { + reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) + if err != nil { + return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err + } + + for _, repoMeta := range reposMeta { + repoSummary := DepRepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap) + + if RepoSumAcceptedByFilter(repoSummary, filter) { + reposPageFinder.Add(repoSummary) + } + } + + page, pageInfo := reposPageFinder.Page() + + // CVE scanning is expensive, only scan for the current page + for _, repoSummary := range page { + updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo) + } + + return page, pageInfo, nil +} + +func DepDescriptor2ImageSummary(ctx context.Context, descriptor mTypes.Descriptor, repo, tag string, + repoMeta mTypes.DepRepoMetadata, manifestMetaMap map[string]mTypes.DepManifestMetadata, + indexDataMap map[string]mTypes.DepIndexData, +) (*gql_generated.ImageSummary, map[string]int64, error) { + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + return DepImageManifest2ImageSummary(ctx, repo, tag, godigest.Digest(descriptor.Digest), + repoMeta, manifestMetaMap[descriptor.Digest]) + case ispec.MediaTypeImageIndex: + return DepImageIndex2ImageSummary(ctx, repo, tag, godigest.Digest(descriptor.Digest), + repoMeta, indexDataMap[descriptor.Digest], manifestMetaMap) + default: + return &gql_generated.ImageSummary{}, map[string]int64{}, zerr.ErrMediaTypeNotSupported + } +} + +// Deprecated: DepImageIndex2ImageSummary. +func DepImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest godigest.Digest, + repoMeta mTypes.DepRepoMetadata, indexData mTypes.DepIndexData, manifestMetaMap map[string]mTypes.DepManifestMetadata, +) (*gql_generated.ImageSummary, map[string]int64, error) { + var indexContent ispec.Index + + err := json.Unmarshal(indexData.IndexBlob, &indexContent) + if err != nil { + return &gql_generated.ImageSummary{}, map[string]int64{}, err + } + + var ( + indexLastUpdated time.Time + isSigned bool + totalIndexSize int64 + indexSize string + totalDownloadCount int + manifestAnnotations *ImageAnnotations + manifestSummaries = make([]*gql_generated.ManifestSummary, 0, len(indexContent.Manifests)) + indexBlobs = make(map[string]int64, 0) + + indexDigestStr = indexDigest.String() + indexMediaType = ispec.MediaTypeImageIndex + ) + + for _, descriptor := range indexContent.Manifests { + manifestSummary, manifestBlobs, annotations, err := DepImageManifest2ManifestSummary(ctx, repo, tag, descriptor, + repoMeta, manifestMetaMap[descriptor.Digest.String()], repoMeta.Referrers[descriptor.Digest.String()]) + if err != nil { + return &gql_generated.ImageSummary{}, map[string]int64{}, err + } + + manifestSize := int64(0) + + for digest, size := range manifestBlobs { + indexBlobs[digest] = size + manifestSize += size + } + + if indexLastUpdated.Before(*manifestSummary.LastUpdated) { + indexLastUpdated = *manifestSummary.LastUpdated + } + + if manifestAnnotations == nil { + manifestAnnotations = annotations + } + + totalIndexSize += manifestSize + + manifestSummaries = append(manifestSummaries, manifestSummary) + } + + totalDownloadCount += repoMeta.Statistics[indexDigestStr].DownloadCount + + for _, signatures := range repoMeta.Signatures[indexDigest.String()] { + if len(signatures) > 0 { + isSigned = true + } + } + + indexSize = strconv.FormatInt(totalIndexSize, 10) + + signaturesInfo := DepGetSignaturesInfo(isSigned, repoMeta, indexDigest) + + if manifestAnnotations == nil { + // The index doesn't have manifests + manifestAnnotations = &ImageAnnotations{} + } + + annotations := GetIndexAnnotations(indexContent.Annotations, manifestAnnotations) + + indexSummary := gql_generated.ImageSummary{ + RepoName: &repo, + Tag: &tag, + Digest: &indexDigestStr, + MediaType: &indexMediaType, + Manifests: manifestSummaries, + LastUpdated: &indexLastUpdated, + IsSigned: &isSigned, + SignatureInfo: signaturesInfo, + Size: &indexSize, + DownloadCount: &totalDownloadCount, + Description: &annotations.Description, + Title: &annotations.Title, + Documentation: &annotations.Documentation, + Licenses: &annotations.Licenses, + Labels: &annotations.Labels, + Source: &annotations.Source, + Vendor: &annotations.Vendor, + Authors: &annotations.Authors, + Referrers: getReferrers(repoMeta.Referrers[indexDigest.String()]), + } + + return &indexSummary, indexBlobs, nil +} + +func DepImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest godigest.Digest, + repoMeta mTypes.DepRepoMetadata, manifestMeta mTypes.DepManifestMetadata, +) (*gql_generated.ImageSummary, map[string]int64, error) { + var ( + manifestContent ispec.Manifest + manifestDigest = digest.String() + mediaType = ispec.MediaTypeImageManifest + ) + + err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent) + if err != nil { + graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+ + "error: %s", repo, tag, manifestDigest, err.Error())) + + return &gql_generated.ImageSummary{}, map[string]int64{}, err + } + + configContent := mcommon.InitializeImageConfig(manifestMeta.ConfigBlob) + + var ( + repoName = repo + configDigest = manifestContent.Config.Digest.String() + configSize = manifestContent.Config.Size + artifactType = zcommon.GetManifestArtifactType(manifestContent) + imageLastUpdated = zcommon.GetImageLastUpdated(configContent) + downloadCount = repoMeta.Statistics[digest.String()].DownloadCount + isSigned = false + ) + + opSys := configContent.OS + arch := configContent.Architecture + variant := configContent.Variant + + if variant != "" { + arch = arch + "/" + variant + } + + platform := gql_generated.Platform{Os: &opSys, Arch: &arch} + + for _, signatures := range repoMeta.Signatures[digest.String()] { + if len(signatures) > 0 { + isSigned = true + } + } + + size, imageBlobsMap := getImageBlobsInfo( + manifestDigest, int64(len(manifestMeta.ManifestBlob)), + configDigest, configSize, + manifestContent.Layers) + imageSize := strconv.FormatInt(size, 10) + + annotations := GetAnnotations(manifestContent.Annotations, configContent.Config.Labels) + + authors := annotations.Authors + if authors == "" { + authors = configContent.Author + } + + historyEntries, err := getAllHistory(manifestContent, configContent) + if err != nil { + graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ + "manifest digest: %s, error: %s", tag, repo, manifestDigest, err.Error())) + } + + signaturesInfo := DepGetSignaturesInfo(isSigned, repoMeta, digest) + + manifestSummary := gql_generated.ManifestSummary{ + Digest: &manifestDigest, + ConfigDigest: &configDigest, + LastUpdated: &imageLastUpdated, + Size: &imageSize, + IsSigned: &isSigned, + SignatureInfo: signaturesInfo, + Platform: &platform, + DownloadCount: &downloadCount, + Layers: getLayersSummaries(manifestContent), + History: historyEntries, + Referrers: getReferrers(repoMeta.Referrers[manifestDigest]), + ArtifactType: &artifactType, + } + + imageSummary := gql_generated.ImageSummary{ + RepoName: &repoName, + Tag: &tag, + Digest: &manifestDigest, + MediaType: &mediaType, + Manifests: []*gql_generated.ManifestSummary{&manifestSummary}, + LastUpdated: &imageLastUpdated, + IsSigned: &isSigned, + SignatureInfo: signaturesInfo, + Size: &imageSize, + DownloadCount: &downloadCount, + Description: &annotations.Description, + Title: &annotations.Title, + Documentation: &annotations.Documentation, + Licenses: &annotations.Licenses, + Labels: &annotations.Labels, + Source: &annotations.Source, + Vendor: &annotations.Vendor, + Authors: &authors, + Referrers: getReferrers(repoMeta.Referrers[manifestDigest]), + } + + return &imageSummary, imageBlobsMap, nil +} + +func DepImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descriptor ispec.Descriptor, + repoMeta mTypes.DepRepoMetadata, manifestMeta mTypes.DepManifestMetadata, + referrersInfo []mTypes.ReferrerInfo, +) (*gql_generated.ManifestSummary, map[string]int64, *ImageAnnotations, error) { + var ( + manifestContent ispec.Manifest + digest = descriptor.Digest + ) + + err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent) + if err != nil { + graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+ + "error: %s", repo, tag, digest, err.Error())) + + return &gql_generated.ManifestSummary{}, map[string]int64{}, &ImageAnnotations{}, err + } + + configContent := mcommon.InitializeImageConfig(manifestMeta.ConfigBlob) + annotations := GetAnnotations(manifestContent.Annotations, configContent.Config.Labels) + + var ( + manifestDigestStr = digest.String() + configDigest = manifestContent.Config.Digest.String() + configSize = manifestContent.Config.Size + artifactType = zcommon.GetManifestArtifactType(manifestContent) + imageLastUpdated = zcommon.GetImageLastUpdated(configContent) + downloadCount = repoMeta.Statistics[digest.String()].DownloadCount + isSigned = false + ) + + opSys := configContent.OS + arch := configContent.Architecture + variant := configContent.Variant + + if variant != "" { + arch = arch + "/" + variant + } + + platform := gql_generated.Platform{Os: &opSys, Arch: &arch} + + size, imageBlobsMap := getImageBlobsInfo( + manifestDigestStr, int64(len(manifestMeta.ManifestBlob)), + configDigest, configSize, + manifestContent.Layers) + imageSize := strconv.FormatInt(size, 10) + + historyEntries, err := getAllHistory(manifestContent, configContent) + if err != nil { + graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ + "manifest digest: %s, error: %s", tag, repo, manifestDigestStr, err.Error())) + } + + for _, signatures := range repoMeta.Signatures[manifestDigestStr] { + if len(signatures) > 0 { + isSigned = true + } + } + + signaturesInfo := DepGetSignaturesInfo(isSigned, repoMeta, digest) + + manifestSummary := gql_generated.ManifestSummary{ + Digest: &manifestDigestStr, + ConfigDigest: &configDigest, + LastUpdated: &imageLastUpdated, + Size: &imageSize, + Platform: &platform, + DownloadCount: &downloadCount, + Layers: getLayersSummaries(manifestContent), + History: historyEntries, + IsSigned: &isSigned, + SignatureInfo: signaturesInfo, + Referrers: getReferrers(referrersInfo), + ArtifactType: &artifactType, + } + + return &manifestSummary, imageBlobsMap, &annotations, nil +} + +func PaginatedDepRepoMeta2ImageSummaries(ctx context.Context, reposMeta []mTypes.DepRepoMetadata, + manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, + skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, +) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) { + imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) + if err != nil { + return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err + } + + for _, repoMeta := range reposMeta { + for tag := range repoMeta.Tags { + descriptor := repoMeta.Tags[tag] + + imageSummary, _, err := DepDescriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, + repoMeta, manifestMetaMap, indexDataMap) + if err != nil { + continue + } + + if ImgSumAcceptedByFilter(imageSummary, filter) { + imagePageFinder.Add(imageSummary) + } + } + } + + page, pageInfo := imagePageFinder.Page() + + for _, imageSummary := range page { + // CVE scanning is expensive, only scan for this page + updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) + } + + return page, pageInfo, nil +} + +func DepRepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.DepRepoMetadata, + manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, + skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger, +) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) { + var ( + repoName = repoMeta.Name + repoLastUpdatedTimestamp = time.Time{} + repoPlatformsSet = map[string]*gql_generated.Platform{} + repoVendorsSet = map[string]bool{} + lastUpdatedImageSummary *gql_generated.ImageSummary + repoDownloadCount = 0 + repoStarCount = repoMeta.Stars // total number of stars + isStarred = repoMeta.IsStarred // value specific to the current user + isBookmarked = repoMeta.IsBookmarked // value specific to the current user + + // map used to keep track of all blobs of a repo without duplicates as + // some images may have the same layers + repoBlob2Size = make(map[string]int64, 10) + + // made up of all manifests, configs and image layers + size = int64(0) + + imageSummaries = make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) + ) + + for tag, descriptor := range repoMeta.Tags { + imageSummary, imageBlobs, err := DepDescriptor2ImageSummary(ctx, descriptor, repoName, tag, + repoMeta, manifestMetaMap, indexDataMap) + if err != nil { + log.Error().Str("repository", repoName).Str("reference", tag). + Msg("metadb: error while converting descriptor for image") + + continue + } + + for _, manifestSummary := range imageSummary.Manifests { + opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch + if opSys != "" || arch != "" { + platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch)) + repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch} + } + + updateRepoBlobsMap(imageBlobs, repoBlob2Size) + } + + if *imageSummary.Vendor != "" { + repoVendorsSet[*imageSummary.Vendor] = true + } + + updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) + + lastUpdatedImageSummary = UpdateLastUpdatedTimestamp(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary) + + repoDownloadCount += *imageSummary.DownloadCount + + imageSummaries = append(imageSummaries, imageSummary) + } + + // calculate repo size = sum all manifest, config and layer blobs sizes + for _, blobSize := range repoBlob2Size { + size += blobSize + } + + repoSize := strconv.FormatInt(size, 10) + + repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) + + for _, platform := range repoPlatformsSet { + repoPlatforms = append(repoPlatforms, platform) + } + + repoVendors := make([]*string, 0, len(repoVendorsSet)) + + for vendor := range repoVendorsSet { + vendor := vendor + repoVendors = append(repoVendors, &vendor) + } + + summary := &gql_generated.RepoSummary{ + Name: &repoName, + LastUpdated: &repoLastUpdatedTimestamp, + Size: &repoSize, + Platforms: repoPlatforms, + Vendors: repoVendors, + NewestImage: lastUpdatedImageSummary, + DownloadCount: &repoDownloadCount, + StarCount: &repoStarCount, + IsBookmarked: &isBookmarked, + IsStarred: &isStarred, + } + + updateRepoSummaryVulnerabilities(ctx, summary, skip, cveInfo) + + return summary, imageSummaries +} + +func DepGetSignaturesInfo(isSigned bool, repoMeta mTypes.DepRepoMetadata, indexDigest godigest.Digest, +) []*gql_generated.SignatureSummary { + signaturesInfo := []*gql_generated.SignatureSummary{} + + if !isSigned { + return signaturesInfo + } + + for sigType, signatures := range repoMeta.Signatures[indexDigest.String()] { + for _, sig := range signatures { + for _, layer := range sig.LayersInfo { + var ( + isTrusted bool + author string + tool string + ) + + if layer.Signer != "" { + author = layer.Signer + + if !layer.Date.IsZero() && time.Now().After(layer.Date) { + isTrusted = false + } else { + isTrusted = true + } + } else { + isTrusted = false + author = "" + } + + tool = sigType + + signaturesInfo = append(signaturesInfo, + &gql_generated.SignatureSummary{Tool: &tool, IsTrusted: &isTrusted, Author: &author}) + } + } + } + + return signaturesInfo +} + +func DepRepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.DepRepoMetadata, + manifestMetaMap map[string]mTypes.DepManifestMetadata, indexDataMap map[string]mTypes.DepIndexData, +) *gql_generated.RepoSummary { + var ( + repoName = repoMeta.Name + repoLastUpdatedTimestamp = time.Time{} + repoPlatformsSet = map[string]*gql_generated.Platform{} + repoVendorsSet = map[string]bool{} + lastUpdatedImageSummary *gql_generated.ImageSummary + repoDownloadCount = 0 + repoStarCount = repoMeta.Stars // total number of stars + repoIsUserStarred = repoMeta.IsStarred // value specific to the current user + repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user + + // map used to keep track of all blobs of a repo without duplicates as + // some images may have the same layers + repoBlob2Size = make(map[string]int64, 10) + + // made up of all manifests, configs and image layers + size = int64(0) + ) + + for tag, descriptor := range repoMeta.Tags { + imageSummary, imageBlobsMap, err := DepDescriptor2ImageSummary(ctx, descriptor, repoMeta.Name, + tag, repoMeta, manifestMetaMap, indexDataMap, + ) + if err != nil { + continue + } + + for blobDigest, blobSize := range imageBlobsMap { + repoBlob2Size[blobDigest] = blobSize + } + + for _, manifestSummary := range imageSummary.Manifests { + if *manifestSummary.Platform.Os != "" || *manifestSummary.Platform.Arch != "" { + opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch + + platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch)) + repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch} + } + } + + repoDownloadCount += *imageSummary.DownloadCount + + if *imageSummary.Vendor != "" { + repoVendorsSet[*imageSummary.Vendor] = true + } + + lastUpdatedImageSummary = UpdateLastUpdatedTimestamp(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary) + } + + // calculate repo size = sum all manifest, config and layer blobs sizes + for _, blobSize := range repoBlob2Size { + size += blobSize + } + + repoSize := strconv.FormatInt(size, 10) + + repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) + + for _, platform := range repoPlatformsSet { + repoPlatforms = append(repoPlatforms, platform) + } + + repoVendors := make([]*string, 0, len(repoVendorsSet)) + + for vendor := range repoVendorsSet { + vendor := vendor + repoVendors = append(repoVendors, &vendor) + } + + return &gql_generated.RepoSummary{ + Name: &repoName, + LastUpdated: &repoLastUpdatedTimestamp, + Size: &repoSize, + Platforms: repoPlatforms, + Vendors: repoVendors, + NewestImage: lastUpdatedImageSummary, + DownloadCount: &repoDownloadCount, + StarCount: &repoStarCount, + IsBookmarked: &repoIsUserBookMarked, + IsStarred: &repoIsUserStarred, + Rank: &repoMeta.Rank, + } +} diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 2e8d92d7fb..7d1dfd9337 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -808,7 +808,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo // Create metadb data for invalid images/negative tests image := CreateRandomImage() - digest31 := image.Digest() err = metaDB.SetRepoReference(repo3, "invalid-manifest", image.AsImageData()) So(err, ShouldBeNil) @@ -850,6 +849,13 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo err = metaDB.SetRepoReference(repoMultiarch, "tagIndex", multiarchImage.AsImageData()) So(err, ShouldBeNil) + err = metaDB.SetRepoMeta("repo-with-bad-tag-digest", mTypes.RepoMetadata{ + Name: "repo-with-bad-tag-digest", + Tags: map[string]mTypes.Descriptor{ + "tag": {MediaType: ispec.MediaTypeImageManifest, Digest: godigest.FromString("1").String()}, + }, + }) + So(err, ShouldBeNil) // Keep a record of all the image references / digest pairings // This is normally done in MetaDB, but we want to verify // the whole flow, including MetaDB @@ -875,18 +881,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo image21Media := image21.ManifestDescriptor.MediaType image21Name := repo2 + ":1.0.0" imageMap[image21Name] = image21Digest - image31Digest := digest31.String() - // image31Media := ispec.MediaTypeImageManifest - image31Name := repo3 + ":invalid-manifest" - imageMap[image31Name] = image31Digest - image41Digest := image41.ManifestDescriptor.Digest.String() - image41Media := image41.ManifestDescriptor.MediaType - image41Name := repo4 + ":invalid-config" - imageMap[image41Name] = image41Digest - image51Digest := digest51.String() - image51Media := ispec.MediaTypeImageManifest - image51Name := repo5 + ":nonexitent-manifest" - imageMap[image51Name] = digest51.String() image61Digest := image61.ManifestDescriptor.Digest.String() image61Media := image61.ManifestDescriptor.MediaType image61Name := repo6 + ":1.0.0" @@ -1105,12 +1099,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo if repo == repo2 && digest == image21Digest { return false, zerr.ErrScanNotSupported } - if repo == repo3 && digest == image31Digest { - return false, zerr.ErrTagMetaNotFound - } - if repo == repo5 && digest == image51Digest { - return false, zerr.ErrManifestDataNotFound - } if repo == repo100 { return false, zerr.ErrRepoMetaNotFound } @@ -1197,16 +1185,16 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) - // Config not valid - cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo4, "invalid-config", "", pageInput) - So(err, ShouldBeNil) + // Scan failed + cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo7, "1.0.0", "", pageInput) + So(err, ShouldEqual, ErrFailedScan) So(len(cveList), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) - // Scan failed - cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo7, "1.0.0", "", pageInput) - So(err, ShouldEqual, ErrFailedScan) + // Tag is not found + cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo-with-bad-tag-digest", "tag", "", pageInput) + So(err, ShouldEqual, zerr.ErrImageDataNotFound) So(len(cveList), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) @@ -1259,24 +1247,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(cveSummary.Count, ShouldEqual, 0) So(cveSummary.MaxSeverity, ShouldEqual, "") - // Tag is not found - // cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo3, image31Digest, image31Media) - // So(err, ShouldEqual, zerr.ErrTagMetaNotFound) - // So(cveSummary.Count, ShouldEqual, 0) - // So(cveSummary.MaxSeverity, ShouldEqual, "") - - // Config not valid - cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo4, image41Digest, image41Media) - So(err, ShouldBeNil) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "NONE") - - // Manifest is not found - cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo5, image51Digest, image51Media) - So(err, ShouldEqual, zerr.ErrManifestDataNotFound) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "") - // Scan failed cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo5, image71Digest, image71Media) So(err, ShouldBeNil) @@ -1325,16 +1295,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) - // Tag is not found, but we should not error - tagList, err = cveInfo.GetImageListWithCVEFixed(repo3, "CVE101") - So(err, ShouldBeNil) - So(len(tagList), ShouldEqual, 0) - - // Manifest is not found, we just consider exclude it from the fixed list - tagList, err = cveInfo.GetImageListWithCVEFixed(repo5, "CVE101") - So(err, ShouldBeNil) - So(len(tagList), ShouldEqual, 0) - // Repo is not found, there could potentially be unaffected tags in the repo // but we can't access their data tagList, err = cveInfo.GetImageListWithCVEFixed(repo100, "CVE100") diff --git a/pkg/extensions/search/cve/scan_test.go b/pkg/extensions/search/cve/scan_test.go index cec4dd6cf1..41cfaed1e5 100644 --- a/pkg/extensions/search/cve/scan_test.go +++ b/pkg/extensions/search/cve/scan_test.go @@ -26,6 +26,7 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta" "zotregistry.io/zot/pkg/meta/boltdb" + mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" @@ -150,6 +151,15 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo err = metaDB.SetRepoReference(repoIndex, "tagIndex", multiarchImage.AsImageData()) So(err, ShouldBeNil) + err = metaDB.SetRepoMeta("repo-with-bad-tag-digest", mTypes.RepoMetadata{ + Name: "repo-with-bad-tag-digest", + Tags: map[string]mTypes.Descriptor{ + "tag": {MediaType: ispec.MediaTypeImageManifest, Digest: godigest.FromString("1").String()}, + "tag-multi-arch": {MediaType: ispec.MediaTypeImageIndex, Digest: godigest.FromString("2").String()}, + }, + }) + So(err, ShouldBeNil) + // Keep a record of all the image references / digest pairings // This is normally done in MetaDB, but we want to verify // the whole flow, including MetaDB diff --git a/pkg/extensions/search/cve/trivy/scanner.go b/pkg/extensions/search/cve/trivy/scanner.go index 3f140ad633..aa43d93633 100644 --- a/pkg/extensions/search/cve/trivy/scanner.go +++ b/pkg/extensions/search/cve/trivy/scanner.go @@ -25,7 +25,6 @@ import ( cvecache "zotregistry.io/zot/pkg/extensions/search/cve/cache" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" - mcommon "zotregistry.io/zot/pkg/meta/common" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" ) @@ -192,7 +191,7 @@ func (scanner Scanner) IsImageFormatScannable(repo, ref string) (bool, error) { ) if zcommon.IsTag(ref) { - imgDescriptor, err := mcommon.GetImageDescriptor(scanner.metaDB, repo, ref) + imgDescriptor, err := getImageDescriptor(scanner.metaDB, repo, ref) if err != nil { return false, err } @@ -202,7 +201,7 @@ func (scanner Scanner) IsImageFormatScannable(repo, ref string) (bool, error) { } else { var found bool - found, mediaType = mcommon.FindMediaTypeForDigest(scanner.metaDB, godigest.Digest(ref)) + found, mediaType = findMediaTypeForDigest(scanner.metaDB, godigest.Digest(ref)) if !found { return false, zerr.ErrManifestNotFound } @@ -223,7 +222,7 @@ func (scanner Scanner) IsImageMediaScannable(repo, digestStr, mediaType string) return ok, nil case ispec.MediaTypeImageIndex: - ok, err := scanner.isIndexScanable(digestStr) + ok, err := scanner.isIndexScannable(digestStr) if err != nil { return ok, fmt.Errorf("image '%s' %w", image, err) } @@ -260,7 +259,28 @@ func (scanner Scanner) isManifestScanable(digestStr string) (bool, error) { return true, nil } -func (scanner Scanner) isIndexScanable(digestStr string) (bool, error) { +func (scanner Scanner) isManifestDataScannable(manifestData mTypes.ManifestData) (bool, error) { + if scanner.cache.Get(manifestData.Digest.String()) != nil { + return true, nil + } + + if manifestData.MediaType != ispec.MediaTypeImageManifest { + return false, zerr.ErrScanNotSupported + } + + for _, imageLayer := range manifestData.Layers { + switch imageLayer.MediaType { + case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): + continue + default: + return false, zerr.ErrScanNotSupported + } + } + + return true, nil +} + +func (scanner Scanner) isIndexScannable(digestStr string) (bool, error) { if scanner.cache.Get(digestStr) != nil { return true, nil } @@ -280,13 +300,13 @@ func (scanner Scanner) isIndexScanable(digestStr string) (bool, error) { return true, nil } - for _, manifest := range indexContent.Manifests { - isScannable, err := scanner.isManifestScanable(manifest.Digest.String()) + for _, manifest := range indexData.Manifests { + isScannable, err := scanner.isManifestDataScannable(manifest) if err != nil { continue } - // if at least 1 manifest is scanable, the whole index is scanable + // if at least 1 manifest is scannable, the whole index is scannable if isScannable { return true, nil } @@ -316,7 +336,7 @@ func (scanner Scanner) ScanImage(image string) (map[string]cvemodel.CVE, error) digest = ref if isTag { - imgDescriptor, err := mcommon.GetImageDescriptor(scanner.metaDB, repo, ref) + imgDescriptor, err := getImageDescriptor(scanner.metaDB, repo, ref) if err != nil { return map[string]cvemodel.CVE{}, err } @@ -326,7 +346,7 @@ func (scanner Scanner) ScanImage(image string) (map[string]cvemodel.CVE, error) } else { var found bool - found, mediaType = mcommon.FindMediaTypeForDigest(scanner.metaDB, godigest.Digest(ref)) + found, mediaType = findMediaTypeForDigest(scanner.metaDB, godigest.Digest(ref)) if !found { return map[string]cvemodel.CVE{}, zerr.ErrManifestNotFound } @@ -557,6 +577,31 @@ func (scanner Scanner) checkDBPresence() error { return nil } +func getImageDescriptor(metaDB mTypes.MetaDB, repo, tag string) (mTypes.Descriptor, error) { + repoMeta, err := metaDB.GetRepoMeta(repo) + if err != nil { + return mTypes.Descriptor{}, err + } + + imageDescriptor, ok := repoMeta.Tags[tag] + if !ok { + return mTypes.Descriptor{}, zerr.ErrTagMetaNotFound + } + + return imageDescriptor, nil +} + +// findMediaTypeForDigest will look into the buckets for a certain digest. Depending on which bucket that +// digest is found the corresponding mediatype is returned. +func findMediaTypeForDigest(metaDB mTypes.MetaDB, digest godigest.Digest) (bool, string) { + imageData, err := metaDB.GetImageData(digest) + if err == nil { + return true, imageData.MediaType + } + + return false, "" +} + func convertSeverity(detectedSeverity string) string { trivySeverity, _ := dbTypes.NewSeverity(detectedSeverity) diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 119b2c2526..63dc232c30 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -22,6 +22,7 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta" "zotregistry.io/zot/pkg/meta/boltdb" + "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/imagestore" "zotregistry.io/zot/pkg/storage/local" @@ -316,39 +317,6 @@ func TestImageScannable(t *testing.T) { panic(err) } - // // Create MetaDB data for unmarshable manifest - // unmarshableManifestBlob := []byte("Some string") - // repoMetaUnmarshable := mTypes.ManifestData{ - // ManifestBlob: unmarshableManifestBlob, - // ConfigBlob: validConfigBlob, - // } - - // digestUnmarshableManifest := godigest.FromBytes(unmarshableManifestBlob) - - // err = metaDB.SetManifestData(digestUnmarshableManifest, repoMetaUnmarshable) - // if err != nil { - // panic(err) - // } - - // err = metaDB.SetRepoReference("repo1", "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest) - // if err != nil { - // panic(err) - // } - - // // Manifest meta cannot be found - // digestMissingManifest := godigest.FromBytes([]byte("Some other string")) - - // err = metaDB.SetRepoReference("repo1", "missing", digestMissingManifest, ispec.MediaTypeImageManifest) - // if err != nil { - // panic(err) - // } - - // // RepoMeta contains invalid digest - // err = metaDB.SetRepoReference("repo1", "invalid-digest", "invalid", ispec.MediaTypeImageManifest) - // if err != nil { - // panic(err) - // } - // Continue with initializing the objects the scanner depends on metrics := monitoring.NewMetricsServer(false, log) @@ -460,14 +428,14 @@ func TestIsIndexScanable(t *testing.T) { scanner.cache.Add("digest", make(map[string]model.CVE)) - found, err := scanner.isIndexScanable("digest") + found, err := scanner.isIndexScannable("digest") So(err, ShouldBeNil) So(found, ShouldBeTrue) }) }) } -func TestIsIndexScanableErrors(t *testing.T) { +func TestIsIndexScannableErrors(t *testing.T) { Convey("Errors", t, func() { storeController := storage.StoreController{} storeController.DefaultStore = mocks.MockedImageStore{} @@ -475,7 +443,24 @@ func TestIsIndexScanableErrors(t *testing.T) { metaDB := mocks.MetaDBMock{} log := log.NewLogger("debug", "") - _ = metaDB - _ = log + Convey("all manifests of a index are not scannable", func() { + unscannableLayer := []Layer{{MediaType: "unscannable-layer-type", Digest: godigest.FromString("123")}} + img1 := CreateImageWith().Layers(unscannableLayer).RandomConfig().Build() + img2 := CreateImageWith().Layers(unscannableLayer).RandomConfig().Build() + multiarch := CreateMultiarchWith().Images([]Image{img1, img2}).Build() + + metaDB.GetImageDataFn = func(digest godigest.Digest) (types.ImageData, error) { + return map[string]types.ImageData{ + img1.DigestStr(): img1.AsImageData(), + img2.DigestStr(): img2.AsImageData(), + multiarch.DigestStr(): multiarch.AsImageData(), + }[digest.String()], nil + } + + scanner := NewScanner(storeController, metaDB, "", "", log) + ok, err := scanner.isIndexScannable(multiarch.DigestStr()) + So(err, ShouldBeNil) + So(ok, ShouldBeFalse) + }) }) } diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index c82529d46b..793b667534 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -72,7 +72,8 @@ func NewResolver(log log.Logger, storeController storage.StoreController, return resolver } -func ProtoFilterByDigest(digest string) mTypes.FilterFunc { +func FilterByDigest(digest string) mTypes.FilterFunc { + // imageData will always contain 1 manifest return func(repoMeta mTypes.RepoMetadata, imageData mTypes.ImageData) bool { lookupDigest := digest contains := false @@ -124,7 +125,7 @@ func getImageListForDigest(ctx context.Context, digest string, metaDB mTypes.Met ), } - imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, ProtoFilterByDigest(digest)) + imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, FilterByDigest(digest)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -144,7 +145,7 @@ func getImageListForDigest(ctx context.Context, digest string, metaDB mTypes.Met }, nil } -func getProtoImageSummary(ctx context.Context, repo, tag string, digest *string, skipCVE convert.SkipQGLField, +func getImageSummary(ctx context.Context, repo, tag string, digest *string, skipCVE convert.SkipQGLField, metaDB mTypes.MetaDB, cveInfo cveinfo.CveInfo, log log.Logger, //nolint:unparam ) ( *gql_generated.ImageSummary, error, @@ -175,7 +176,7 @@ func getProtoImageSummary(ctx context.Context, repo, tag string, digest *string, return &gql_generated.ImageSummary{}, err } - imageSummaries := convert.ProtoRepoMeta2ImageSummaries(ctx, repoMeta, imageDataMap, skipCVE, cveInfo) + imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, imageDataMap, skipCVE, cveInfo) if len(imageSummaries) == 0 { return &gql_generated.ImageSummary{}, nil @@ -258,7 +259,7 @@ func getCVEListForImage( }, nil } -func ProtoFilterByTagInfo(tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { +func FilterByTagInfo(tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { return func(repoMeta mTypes.RepoMetadata, imageData mTypes.ImageData) bool { manifestDigest := imageData.Manifests[0].Digest.String() @@ -281,7 +282,7 @@ func ProtoFilterByTagInfo(tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { } } -func ProtoFilterByRepoAndTagInfo(repo string, tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { +func FilterByRepoAndTagInfo(repo string, tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { return func(repoMeta mTypes.RepoMetadata, imageData mTypes.ImageData) bool { if repoMeta.Name != repo { return false @@ -326,11 +327,6 @@ func getImageListForCVE( return &gql_generated.PaginatedImagesResult{}, err } - // reposMeta, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }) - // if err != nil { - // return &gql_generated.PaginatedImagesResult{}, err - // } - affectedImages := []cvemodel.TagInfo{} for _, repoMeta := range reposMeta { @@ -377,7 +373,7 @@ func getImageListForCVE( } // get all repos - imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, ProtoFilterByTagInfo(affectedImages)) + imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, FilterByTagInfo(affectedImages)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -450,7 +446,7 @@ func getImageListWithCVEFixed( } // get all repos - imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, ProtoFilterByRepoAndTagInfo(repo, tagsInfo)) + imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, FilterByRepoAndTagInfo(repo, tagsInfo)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -537,15 +533,11 @@ func getBookmarkedRepos( // return zcommon.Contains(repoNames, repoMeta.Name) // } - filterByName := func(repo string) int { - if zcommon.Contains(bookmarkedRepos, repo) { - return 1 - } - - return -1 + filterByName := func(repo string) bool { + return zcommon.Contains(bookmarkedRepos, repo) } - return protoGetFilteredPaginatedRepos(ctx, cveInfo, filterByName, log, requestedPage, metaDB) + return getFilteredPaginatedRepos(ctx, cveInfo, filterByName, log, requestedPage, metaDB) } func getStarredRepos( @@ -560,18 +552,14 @@ func getStarredRepos( return &gql_generated.PaginatedReposResult{}, err } - protoFilterFn := func(repo string) int { - if zcommon.Contains(starredRepos, repo) { - return 1 - } - - return -1 + filterFn := func(repo string) bool { + return zcommon.Contains(starredRepos, repo) } - return protoGetFilteredPaginatedRepos(ctx, cveInfo, protoFilterFn, log, requestedPage, metaDB) + return getFilteredPaginatedRepos(ctx, cveInfo, filterFn, log, requestedPage, metaDB) } -func protoGetFilteredPaginatedRepos( +func getFilteredPaginatedRepos( ctx context.Context, cveInfo cveinfo.CveInfo, filterFn mTypes.FilterRepoNameFunc, @@ -765,7 +753,7 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB Vulnerabilities: true, } - searchedImage, err := getProtoImageSummary(ctx, imageRepo, imageTag, digest, skipReferenceImage, metaDB, cveInfo, log) + searchedImage, err := getImageSummary(ctx, imageRepo, imageTag, digest, skipReferenceImage, metaDB, cveInfo, log) if err != nil { if errors.Is(err, zerr.ErrRepoMetaNotFound) { return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("repository: not found") @@ -775,7 +763,7 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB } // we need all available tags - imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, protoFilterDerivedImages(searchedImage)) + imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, filterDerivedImages(searchedImage)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -795,7 +783,7 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB }, nil } -func protoFilterDerivedImages(image *gql_generated.ImageSummary) mTypes.FilterFunc { +func filterDerivedImages(image *gql_generated.ImageSummary) mTypes.FilterFunc { return func(repoMeta mTypes.RepoMetadata, imageData mTypes.ImageData) bool { var addImageToList bool @@ -866,7 +854,7 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy Vulnerabilities: true, } - searchedImage, err := getProtoImageSummary(ctx, imageRepo, imageTag, digest, skipReferenceImage, metaDB, cveInfo, log) + searchedImage, err := getImageSummary(ctx, imageRepo, imageTag, digest, skipReferenceImage, metaDB, cveInfo, log) if err != nil { if errors.Is(err, zerr.ErrRepoMetaNotFound) { return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("repository: not found") @@ -876,7 +864,7 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy } // we need all available tags - imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, protoFilterBaseImages(searchedImage)) + imageDataList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, filterBaseImages(searchedImage)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -896,7 +884,7 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy }, nil } -func protoFilterBaseImages(image *gql_generated.ImageSummary) mTypes.FilterFunc { +func filterBaseImages(image *gql_generated.ImageSummary) mTypes.FilterFunc { return func(repoMeta mTypes.RepoMetadata, imageData mTypes.ImageData) bool { var addImageToList bool @@ -941,7 +929,7 @@ func validateGlobalSearchInput(query string, filter *gql_generated.Filter, requestedPage *gql_generated.PageInput, ) error { if len(query) > querySizeLimit { - return fmt.Errorf("global-search: max string size limit exeeded for query parameter. max=%d current=%d %w", + return fmt.Errorf("global-search: max string size limit exceeded for query parameter. max=%d current=%d %w", querySizeLimit, len(query), zerr.ErrInvalidRequestParams) } @@ -965,14 +953,14 @@ func checkFilter(filter *gql_generated.Filter) error { for _, arch := range filter.Arch { if len(*arch) > querySizeLimit { - return fmt.Errorf("global-search: max string size limit exeeded for arch parameter. max=%d current=%d %w", + return fmt.Errorf("global-search: max string size limit exceeded for arch parameter. max=%d current=%d %w", querySizeLimit, len(*arch), zerr.ErrInvalidRequestParams) } } for _, osSys := range filter.Os { if len(*osSys) > querySizeLimit { - return fmt.Errorf("global-search: max string size limit exeeded for os parameter. max=%d current=%d %w", + return fmt.Errorf("global-search: max string size limit exceeded for os parameter. max=%d current=%d %w", querySizeLimit, len(*osSys), zerr.ErrInvalidRequestParams) } } diff --git a/pkg/extensions/search/schema.resolvers.go b/pkg/extensions/search/schema.resolvers.go index 4caa350f5d..6d614b6890 100644 --- a/pkg/extensions/search/schema.resolvers.go +++ b/pkg/extensions/search/schema.resolvers.go @@ -138,7 +138,7 @@ func (r *queryResolver) Image(ctx context.Context, image string) (*gql_generated Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Vulnerabilities"), } - return getProtoImageSummary(ctx, repo, tag, nil, skip, r.metaDB, r.cveInfo, r.log) + return getImageSummary(ctx, repo, tag, nil, skip, r.metaDB, r.cveInfo, r.log) } // Referrers is the resolver for the Referrers field. diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index c721d6d9d8..bc7d4a6a5e 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -13,7 +13,6 @@ import ( "net/url" "os" "path" - "regexp" "strconv" "strings" "testing" @@ -5355,9 +5354,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { for _, manifest := range indexContent.Manifests { tag := manifest.Annotations[ispec.AnnotationRefName] - cosignTagRule := regexp.MustCompile(`sha256\-.+\.sig`) - - if cosignTagRule.MatchString(tag) { + if zcommon.IsCosignTag(tag) { signatureTag = tag } } diff --git a/pkg/extensions/sync/local.go b/pkg/extensions/sync/local.go index 7e2e35d371..9bf1717901 100644 --- a/pkg/extensions/sync/local.go +++ b/pkg/extensions/sync/local.go @@ -164,7 +164,7 @@ func (registry *LocalRegistry) CommitImage(imageReference types.ImageReference, } if registry.metaDB != nil { - err = meta.ProtoSetImageMetaFromInput(repo, reference, mediaType, + err = meta.SetImageMetaFromInput(repo, reference, mediaType, manifestDigest, manifestBlob, imageStore, registry.metaDB, registry.log) if err != nil { return fmt.Errorf("metaDB: failed to set metadata for image '%s %s': %w", repo, reference, err) @@ -222,7 +222,7 @@ func (registry *LocalRegistry) copyManifest(repo string, manifestContent []byte, } if registry.metaDB != nil { - err = meta.ProtoSetImageMetaFromInput(repo, reference, ispec.MediaTypeImageManifest, + err = meta.SetImageMetaFromInput(repo, reference, ispec.MediaTypeImageManifest, digest, manifestContent, imageStore, registry.metaDB, registry.log) if err != nil { registry.log.Error().Str("errorType", common.TypeOf(err)). diff --git a/pkg/extensions/sync/references/cosign.go b/pkg/extensions/sync/references/cosign.go index 48b9227967..1903e85ac1 100644 --- a/pkg/extensions/sync/references/cosign.go +++ b/pkg/extensions/sync/references/cosign.go @@ -153,7 +153,7 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to sync cosign reference for image") - err = meta.ProtoSetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest, + err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest, referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go index 875bb11162..6519b05eb0 100644 --- a/pkg/extensions/sync/references/oci.go +++ b/pkg/extensions/sync/references/oci.go @@ -137,7 +137,7 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to add oci references for image") - err = meta.ProtoSetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) if err != nil { diff --git a/pkg/extensions/sync/references/oras.go b/pkg/extensions/sync/references/oras.go index 624ae4bf7a..c5a208d613 100644 --- a/pkg/extensions/sync/references/oras.go +++ b/pkg/extensions/sync/references/oras.go @@ -154,7 +154,7 @@ func (ref ORASReferences) SyncReferences(ctx context.Context, localRepo, remoteR ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to sync oras artifact for image") - err := meta.ProtoSetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + err := meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, referenceDigest, orasBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) if err != nil { diff --git a/pkg/extensions/sync/references/referrers_tag.go b/pkg/extensions/sync/references/referrers_tag.go index e76ad77fe1..2633e15b36 100644 --- a/pkg/extensions/sync/references/referrers_tag.go +++ b/pkg/extensions/sync/references/referrers_tag.go @@ -113,7 +113,7 @@ func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRe ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to add oci references for image") - err = meta.ProtoSetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), ref.metaDB, ref.log) if err != nil { diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index 014ba37af8..b216203b75 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -5,15 +5,21 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" "go.etcd.io/bbolt" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/api/constants" zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/common" + mConvert "zotregistry.io/zot/pkg/meta/convert" "zotregistry.io/zot/pkg/meta/proto_go" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/meta/version" @@ -78,6 +84,1326 @@ func New(boltDB *bbolt.DB, log log.Logger) (*BoltDB, error) { }, nil } +func (bdw *BoltDB) SetImageData(digest godigest.Digest, imageData mTypes.ImageData) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(ImageDataBuck)) + + protoImageData := &proto_go.ImageData{} + + switch imageData.MediaType { + case ispec.MediaTypeImageManifest: + manifest := imageData.Manifests[0] + + protoImageData = mConvert.GetProtoImageManifestData(manifest.Manifest, manifest.ConfigContent, + manifest.Size, manifest.Digest.String()) + case ispec.MediaTypeImageIndex: + protoImageData = mConvert.GetProtoImageIndexData(*imageData.Index, imageData.Size, imageData.Digest.String()) + } + + pImageDataBlob, err := proto.Marshal(protoImageData) + if err != nil { + return fmt.Errorf("metadb: error while calculating blob for manifest with digest %s %w", digest, err) + } + + err = buck.Put([]byte(digest), pImageDataBlob) + if err != nil { + return fmt.Errorf("metadb: error while setting manifest data with for digest %s %w", digest, err) + } + + return nil + }) + + return err +} + +func (bdw *BoltDB) SetRepoReference(repo string, reference string, imageData mTypes.ImageData, +) error { + if err := common.ValidateRepoReferenceInput(repo, reference, imageData.Digest); err != nil { + return err + } + + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(RepoMetaBuck)) + repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) + imageBuck := tx.Bucket([]byte(ImageDataBuck)) + + // 1. Add image data to db if needed + + protoImageData := mConvert.GetProtoImageData(imageData) + + imageDataBlob, err := proto.Marshal(protoImageData) + if err != nil { + return err + } + + err = imageBuck.Put([]byte(imageData.Digest), imageDataBlob) + if err != nil { + return err + } + + repoMetaBlob := repoBuck.Get([]byte(repo)) + + protoRepoMeta := &proto_go.RepoMeta{ + Name: repo, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, // This is done so Protobuf can initialize a non-nil map + Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + } + + if len(repoMetaBlob) > 0 { + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return err + } + } + + // 2. Referrers + if protoImageData.Subject != nil { + refInfo := &proto_go.ReferrersInfo{} + if protoRepoMeta.Referrers[protoImageData.Subject.Digest] != nil { + refInfo = protoRepoMeta.Referrers[protoImageData.Subject.Digest] + } + + refInfo.List = append(refInfo.List, &proto_go.ReferrerInfo{ + Digest: protoImageData.Digest, + MediaType: protoImageData.MediaType, + ArtifactType: mConvert.GetArtifactType(protoImageData), + Size: protoImageData.Size, + Annotations: protoImageData.Annotations, + }) + + protoRepoMeta.Referrers[protoImageData.Subject.Digest] = refInfo + } + + // 3. Update tag + if !common.ReferenceIsDigest(reference) { + protoRepoMeta.Tags[reference] = &proto_go.TagDescriptor{ + Digest: imageData.Digest.String(), + MediaType: imageData.MediaType, + } + } + + if _, ok := protoRepoMeta.Statistics[imageData.Digest.String()]; !ok { + protoRepoMeta.Statistics[imageData.Digest.String()] = &proto_go.DescriptorStatistics{DownloadCount: 0} + } + + if _, ok := protoRepoMeta.Signatures[imageData.Digest.String()]; !ok { + protoRepoMeta.Signatures[imageData.Digest.String()] = &proto_go.ManifestSignatures{ + Map: map[string]*proto_go.SignaturesInfo{"": {}}, + } + } + + if _, ok := protoRepoMeta.Referrers[imageData.Digest.String()]; !ok { + protoRepoMeta.Referrers[imageData.Digest.String()] = &proto_go.ReferrersInfo{ + List: []*proto_go.ReferrerInfo{}, + } + } + + // 4. Blobs + repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name)) + + repoBlobs := &proto_go.RepoBlobs{} + + if repoBlobsBytes == nil { + repoBlobs.Blobs = map[string]*proto_go.BlobInfo{} + } else { + err := proto.Unmarshal(repoBlobsBytes, repoBlobs) + if err != nil { + return err + } + } + + protoRepoMeta, repoBlobs, err = common.AddImageDataToRepoMeta(protoRepoMeta, repoBlobs, reference, imageData) + if err != nil { + return err + } + + repoBlobsBytes, err = proto.Marshal(repoBlobs) + if err != nil { + return err + } + + err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes) + if err != nil { + return err + } + + repoMetaBlob, err = proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + return repoBuck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) FilterImageData(ctx context.Context, digests []string, +) (map[string]mTypes.ImageData, error) { + imageDataMap := map[string]mTypes.ImageData{} + + err := bdw.DB.View(func(transaction *bbolt.Tx) error { + imageBuck := transaction.Bucket([]byte(ImageDataBuck)) + + for _, digest := range digests { + protoImageData, err := fetchProtoImageData(imageBuck, digest) + if err != nil { + return err + } + + if protoImageData.MediaType == ispec.MediaTypeImageIndex { + for i, manifest := range protoImageData.Manifests { + manifestDigest := manifest.Digest + + imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) + if err != nil { + return err + } + + protoImageData.Manifests[i] = imageManifestData.Manifests[0] + } + } + + imageDataMap[digest] = mConvert.GetImageData(protoImageData) + } + + return nil + }) + + return imageDataMap, err +} + +func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, +) ([]mTypes.FullRepoMetadata, error) { + repos := []mTypes.FullRepoMetadata{} + + err := bdw.DB.View(func(transaction *bbolt.Tx) error { + var ( + repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) + ) + + cursor := repoBuck.Cursor() + + for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + rank := common.RankRepoName(searchText, string(repoName)) + if rank == -1 { + continue + } + + var protoRepoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + + protoRepoMeta.Rank = int32(rank) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + + repos = append(repos, mConvert.GetFullRepoMeta(&protoRepoMeta)) + } + + return nil + }) + + return repos, err +} + +func fetchProtoImageData(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageData, error) { + imageDataBlob := imageBuck.Get([]byte(digest)) + + if len(imageDataBlob) == 0 { + return nil, zerr.ErrImageDataNotFound + } + + imageData := proto_go.ImageData{} + + err := proto.Unmarshal(imageDataBlob, &imageData) + if err != nil { + return nil, err + } + + return &imageData, nil +} + +func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, +) ([]mTypes.FullImageData, error) { + images := []mTypes.FullImageData{} + + searchedRepo, searchedTag, err := common.GetRepoTag(searchText) + if err != nil { + return []mTypes.FullImageData{}, + fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) + } + + err = bdw.DB.View(func(transaction *bbolt.Tx) error { + var ( + repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) + imageBuck = transaction.Bucket([]byte(ImageDataBuck)) + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) + ) + + repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo)) + + if string(repoName) != searchedRepo { + return nil + } + + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + return err + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + + for tag, descriptor := range protoRepoMeta.Tags { + if !strings.HasPrefix(tag, searchedTag) || tag == "" { + continue + } + + var protoImageData *proto_go.ImageData + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) + if err != nil { + return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", + manifestDigest, err) + } + + imageManifestData.Repo = ref(string(repoName)) + imageManifestData.Tag = &tag + + protoImageData = imageManifestData + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + imageIndexData, err := fetchProtoImageData(imageBuck, indexDigest) + if err != nil { + return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", + indexDigest, err) + } + + for i, manifest := range imageIndexData.Manifests { + manifestDigest := manifest.Digest + + imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) + if err != nil { + return err + } + + imageIndexData.Manifests[i] = imageManifestData.Manifests[0] + } + + imageIndexData.Repo = ref(string(repoName)) + imageIndexData.Tag = &tag + + protoImageData = imageIndexData + default: + bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") + + continue + } + + images = append(images, mConvert.GetFullImageDataFromProto(tag, protoRepoMeta, protoImageData)) + } + + return nil + }) + + return images, err +} + +func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc, +) ([]mTypes.FullImageData, error) { + images := []mTypes.FullImageData{} + + err := bdw.DB.View(func(transaction *bbolt.Tx) error { + var ( + repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) + imageDataBuck = transaction.Bucket([]byte(ImageDataBuck)) + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) + viewError error + ) + + cursor := repoBuck.Cursor() + repoName, repoMetaBlob := cursor.First() + + for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) + + for tag, descriptor := range protoRepoMeta.Tags { + if !filterRepoTag(string(repoName), tag) { + continue + } + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + imageManifestData, err := fetchProtoImageData(imageDataBuck, manifestDigest) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + imageManifestData.Repo = ref(repoMeta.Name) + imageManifestData.Tag = ref(tag) + + imageData := mConvert.GetImageData(imageManifestData) + + if filterFunc(repoMeta, imageData) { + images = append(images, mConvert.GetFullImageDataFromProto(tag, protoRepoMeta, imageManifestData)) + } + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + imageIndexData, err := fetchProtoImageData(imageDataBuck, indexDigest) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + matchedManifests := []*proto_go.ManifestData{} + + for _, manifest := range imageIndexData.Manifests { + manifestDigest := manifest.Digest + + imageManifestData, err := fetchProtoImageData(imageDataBuck, manifestDigest) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + imageData := mConvert.GetImageData(imageManifestData) + + if filterFunc(repoMeta, imageData) { + matchedManifests = append(matchedManifests, imageManifestData.Manifests[0]) + } + } + + if len(matchedManifests) > 0 { + imageIndexData.Manifests = matchedManifests + + images = append(images, mConvert.GetFullImageDataFromProto(tag, protoRepoMeta, imageIndexData)) + } + default: + bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") + + continue + } + } + } + + return viewError + }) + + return images, err +} + +func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepoNameFunc, + filter mTypes.FilterFullRepoFunc, +) ([]mTypes.FullRepoMetadata, error) { + repos := []mTypes.FullRepoMetadata{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + var ( + buck = tx.Bucket([]byte(RepoMetaBuck)) + cursor = buck.Cursor() + userBookmarks = getUserBookmarks(ctx, tx) + userStars = getUserStars(ctx, tx) + ) + + for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + if !acceptName(string(repoName)) { + continue + } + + repoMeta := proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + + fullRepoMeta := mConvert.GetFullRepoMeta(&repoMeta) + + if filter(fullRepoMeta) { + repos = append(repos, fullRepoMeta) + } + } + + return nil + }) + if err != nil { + return []mTypes.FullRepoMetadata{}, err + } + + return repos, err +} + +func (bdw *BoltDB) GetRepoMeta(repo string) (mTypes.RepoMetadata, error) { + var protoRepoMeta proto_go.RepoMeta + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + + // object not found + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + // object found + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + + return nil + }) + + return mConvert.GetRepoMeta(&protoRepoMeta), err +} + +func (bdw *BoltDB) GetFullRepoMeta(ctx context.Context, repo string) (mTypes.FullRepoMetadata, error) { + var protoRepoMeta proto_go.RepoMeta + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + userBookmarks := getUserBookmarks(ctx, tx) + userStars := getUserStars(ctx, tx) + + repoMetaBlob := buck.Get([]byte(repo)) + + // object not found + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + // object found + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) + + return nil + }) + + return mConvert.GetFullRepoMeta(&protoRepoMeta), err +} + +func (bdw *BoltDB) GetFullImageData(ctx context.Context, repo string, tag string) (mTypes.FullImageData, error) { + protoRepoMeta := &proto_go.RepoMeta{} + protoImageData := &proto_go.ImageData{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + imageBuck := tx.Bucket([]byte(ImageDataBuck)) + userBookmarks := getUserBookmarks(ctx, tx) + userStars := getUserStars(ctx, tx) + + repoMetaBlob := buck.Get([]byte(repo)) + + // object not found + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + // object found + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) + + descriptor, ok := protoRepoMeta.Tags[tag] + if !ok { + return zerr.ErrImageDataNotFound + } + + protoImageData, err := fetchProtoImageData(imageBuck, descriptor.Digest) + if err != nil { + return err + } + + if protoImageData.MediaType == ispec.MediaTypeImageIndex { + for i, manifest := range protoImageData.Manifests { + manifestDigest := manifest.Digest + + imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) + if err != nil { + return err + } + + protoImageData.Manifests[i] = imageManifestData.Manifests[0] + } + } + + return nil + }) + + return mConvert.GetFullImageDataFromProto(tag, protoRepoMeta, protoImageData), err +} + +func (bdw *BoltDB) GetImageData(digest godigest.Digest) (mTypes.ImageData, error) { + imageData := mTypes.ImageData{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + imageBuck := tx.Bucket([]byte(ImageDataBuck)) + + protoImageData, err := fetchProtoImageData(imageBuck, digest.String()) + if err != nil { + return err + } + + if protoImageData.MediaType == ispec.MediaTypeImageIndex { + for i, manifest := range protoImageData.Manifests { + manifestDigest := manifest.Digest + + imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) + if err != nil { + return err + } + + protoImageData.Manifests[i] = imageManifestData.Manifests[0] + } + } + + imageData = mConvert.GetImageData(protoImageData) + + return nil + }) + + return imageData, err +} + +func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, +) ([]mTypes.RepoMetadata, error) { + foundRepos := []mTypes.RepoMetadata{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + cursor := buck.Cursor() + + for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + protoRepoMeta := proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + + repoMeta := mConvert.GetRepoMeta(&protoRepoMeta) + + if filter(repoMeta) { + foundRepos = append(foundRepos, repoMeta) + } + } + + return nil + }) + + return foundRepos, err +} + +func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, + sygMeta mTypes.SignatureMetadata, +) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + + if len(repoMetaBlob) == 0 { + var err error + // create a new object + repoMeta := proto_go.RepoMeta{ + Name: repo, + Tags: map[string]*proto_go.TagDescriptor{}, + Signatures: map[string]*proto_go.ManifestSignatures{ + signedManifestDigest.String(): { + Map: map[string]*proto_go.SignaturesInfo{ + sygMeta.SignatureType: { + List: []*proto_go.SignatureInfo{ + { + SignatureManifestDigest: sygMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + }, + }, + }, + }, + }, + }, + Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, + } + + repoMetaBlob, err = proto.Marshal(&repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return err + } + + var ( + manifestSignatures *proto_go.ManifestSignatures + found bool + ) + + if manifestSignatures, found = protoRepoMeta.Signatures[signedManifestDigest.String()]; !found { + manifestSignatures = &proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} + } + + signatureSlice := &proto_go.SignaturesInfo{List: []*proto_go.SignatureInfo{}} + if sigSlice, found := manifestSignatures.Map[sygMeta.SignatureType]; found { + signatureSlice = sigSlice + } + + if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sygMeta) { + switch sygMeta.SignatureType { + case zcommon.NotationSignature: + signatureSlice.List = append(signatureSlice.List, &proto_go.SignatureInfo{ + SignatureManifestDigest: sygMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + }) + case zcommon.CosignSignature: + signatureSlice.List = []*proto_go.SignatureInfo{{ + SignatureManifestDigest: sygMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + }} + } + } + + manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice + protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures + + repoMetaBlob, err = proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Digest, + sigMeta mTypes.SignatureMetadata, +) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrManifestMetaNotFound + } + + protoRepoMeta := proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + manifestSignatures, found := protoRepoMeta.Signatures[signedManifestDigest.String()] + if !found { + return zerr.ErrManifestMetaNotFound + } + + signatureSlice := manifestSignatures.Map[sigMeta.SignatureType] + + newSignatureSlice := make([]*proto_go.SignatureInfo, 0, len(signatureSlice.List)) + + for _, sigInfo := range signatureSlice.List { + if sigInfo.SignatureManifestDigest != sigMeta.SignatureDigest { + newSignatureSlice = append(newSignatureSlice, sigInfo) + } + } + + manifestSignatures.Map[sigMeta.SignatureType] = &proto_go.SignaturesInfo{List: newSignatureSlice} + + protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures + + repoMetaBlob, err = proto.Marshal(&protoRepoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) IncrementRepoStars(repo string) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + repoMeta.Stars++ + + repoMetaBlob, err = proto.Marshal(&repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) DecrementRepoStars(repo string) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + if repoMeta.Stars == 0 { + return nil + } + + repoMeta.Stars-- + + repoMetaBlob, err = proto.Marshal(&repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) GetRepoStars(repo string) (int, error) { + stars := 0 + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + buck.Get([]byte(repo)) + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + stars = int(repoMeta.Stars) + + return nil + }) + + return stars, err +} + +func (bdw *BoltDB) DeleteRepoTag(repo string, tag string) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + + // object not found + if repoMetaBlob == nil { + return nil + } + + // object found + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + delete(repoMeta.Tags, tag) + + repoMetaBlob, err = proto.Marshal(&repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) GetUserRepoMeta(ctx context.Context, repo string) (mTypes.RepoMetadata, error) { + var repoMeta proto_go.RepoMeta + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + userBookmarks := getUserBookmarks(ctx, tx) + userStars := getUserStars(ctx, tx) + + repoMetaBlob := buck.Get([]byte(repo)) + + // object not found + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + // object found + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) + repoMeta.IsStarred = zcommon.Contains(userStars, repo) + + return nil + }) + + return mConvert.GetRepoMeta(&repoMeta), err +} + +func (bdw *BoltDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMetadata) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMeta.Name = repo + + repoMetaBlob, err := proto.Marshal(mConvert.GetProtoRepoMeta(repoMeta)) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) ResetRepoReferences(repo string) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + + if repoMetaBlob == nil { + return nil + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return err + } + + repoMetaBlob, err = proto.Marshal(&proto_go.RepoMeta{ + Name: repo, + Statistics: protoRepoMeta.Statistics, + Stars: protoRepoMeta.Stars, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + }) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, +) ([]mTypes.ReferrerInfo, error) { + referrersInfoResult := []mTypes.ReferrerInfo{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if len(repoMetaBlob) == 0 { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + referrersInfo := repoMeta.Referrers[referredDigest.String()].List + + for i := range referrersInfo { + if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) { + continue + } + + referrersInfoResult = append(referrersInfoResult, mTypes.ReferrerInfo{ + Digest: referrersInfo[i].Digest, + MediaType: referrersInfo[i].MediaType, + ArtifactType: referrersInfo[i].ArtifactType, + Size: int(referrersInfo[i].Size), + Annotations: referrersInfo[i].Annotations, + }) + } + + return nil + }) + + return referrersInfoResult, err +} + +func (bdw *BoltDB) IncrementImageDownloads(repo string, reference string) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + manifestDigest := reference + + if !common.ReferenceIsDigest(reference) { + // search digest for tag + descriptor, found := repoMeta.Tags[reference] + + if !found { + return zerr.ErrManifestMetaNotFound + } + + manifestDigest = descriptor.Digest + } + + manifestStatistics, ok := repoMeta.Statistics[manifestDigest] + if !ok { + return zerr.ErrManifestMetaNotFound + } + + manifestStatistics.DownloadCount++ + repoMeta.Statistics[manifestDigest] = manifestStatistics + + repoMetaBlob, err = proto.Marshal(&repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error { + err := bdw.DB.Update(func(transaction *bbolt.Tx) error { + imgTrustStore := bdw.ImageTrustStore() + + if imgTrustStore == nil { + return nil + } + + // get ManifestData of signed manifest + imageDataBuck := transaction.Bucket([]byte(ImageDataBuck)) + idBlob := imageDataBuck.Get([]byte(manifestDigest)) + + if len(idBlob) == 0 { + // manifest meta not found, updating signatures with details about validity and author will not be performed + return nil + } + + protoImageData := proto_go.ImageData{} + + err := proto.Unmarshal(idBlob, &protoImageData) + if err != nil { + return err + } + + // update signatures with details about validity and author + repoBuck := transaction.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := repoBuck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + protoRepoMeta := proto_go.RepoMeta{} + + err = proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + manifestSignatures := proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} + for sigType, sigs := range protoRepoMeta.Signatures[manifestDigest.String()].Map { + signaturesInfo := []*proto_go.SignatureInfo{} + + for _, sigInfo := range sigs.List { + layersInfo := []*proto_go.LayersInfo{} + + for _, layerInfo := range sigInfo.LayersInfo { + author, date, isTrusted, _ := imgTrustStore.VerifySignature(sigType, layerInfo.LayerContent, + layerInfo.SignatureKey, manifestDigest, mConvert.GetImageData(&protoImageData), repo) + + if isTrusted { + layerInfo.Signer = author + } + + if !date.IsZero() { + layerInfo.Signer = author + layerInfo.Date = timestamppb.New(date) + } + + layersInfo = append(layersInfo, layerInfo) + } + + signaturesInfo = append(signaturesInfo, &proto_go.SignatureInfo{ + SignatureManifestDigest: sigInfo.SignatureManifestDigest, + LayersInfo: layersInfo, + }) + } + + manifestSignatures.Map[sigType] = &proto_go.SignaturesInfo{List: signaturesInfo} + } + + protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures + + repoMetaBlob, err = proto.Marshal(&protoRepoMeta) + if err != nil { + return err + } + + return repoBuck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + imageDataBuck := tx.Bucket([]byte(ImageDataBuck)) + repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return nil + } + + repoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, repoMeta) + if err != nil { + return err + } + + imageData, err := fetchProtoImageData(imageDataBuck, manifestDigest.String()) + if err != nil { + if errors.Is(err, zerr.ErrImageDataNotFound) { + return nil + } + + return err + } + + // Remove Referrers + if imageData.Subject != nil { + referredDigest := imageData.Subject.Digest + refInfo := &proto_go.ReferrersInfo{} + + if repoMeta.Referrers[referredDigest] != nil { + refInfo = repoMeta.Referrers[referredDigest] + } + + referrers := refInfo.List + + for i := range referrers { + if referrers[i].Digest == manifestDigest.String() { + referrers = append(referrers[:i], referrers[i+1:]...) + + break + } + } + + refInfo.List = referrers + + repoMeta.Referrers[referredDigest] = refInfo + } + + if !common.ReferenceIsDigest(reference) { + delete(repoMeta.Tags, reference) + } else { + // remove all tags pointing to this digest + for tag, desc := range repoMeta.Tags { + if desc.Digest == reference { + delete(repoMeta.Tags, tag) + } + } + } + + /* try to find at least one tag pointing to manifestDigest + if not found then we can also remove everything related to this digest */ + var foundTag bool + for _, desc := range repoMeta.Tags { + if desc.Digest == manifestDigest.String() { + foundTag = true + } + } + + if !foundTag { + delete(repoMeta.Statistics, manifestDigest.String()) + delete(repoMeta.Signatures, manifestDigest.String()) + delete(repoMeta.Referrers, manifestDigest.String()) + } + + repoBlobsBytes := repoBlobsBuck.Get([]byte(repoMeta.Name)) + + repoBlobs := &proto_go.RepoBlobs{} + + if repoBlobsBytes == nil { + repoBlobs.Blobs = map[string]*proto_go.BlobInfo{} + } else { + err := proto.Unmarshal(repoBlobsBytes, repoBlobs) + if err != nil { + return err + } + } + + repoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(repoMeta, repoBlobs, reference) + if err != nil { + return err + } + + repoBlobsBytes, err = proto.Marshal(repoBlobs) + if err != nil { + return err + } + + err = repoBlobsBuck.Put([]byte(repoMeta.Name), repoBlobsBytes) + if err != nil { + return err + } + + repoMetaBlob, err = proto.Marshal(repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + func (bdw *BoltDB) ImageTrustStore() mTypes.ImageTrustStore { return bdw.imgTrustStore } @@ -679,3 +2005,9 @@ func (bdw *BoltDB) DeleteUserData(ctx context.Context) error { return err } + +func ref[T any](input T) *T { + ref := input + + return &ref +} diff --git a/pkg/meta/boltdb/boltdb_test.go b/pkg/meta/boltdb/boltdb_test.go index 0000f517f5..665362d359 100644 --- a/pkg/meta/boltdb/boltdb_test.go +++ b/pkg/meta/boltdb/boltdb_test.go @@ -21,14 +21,14 @@ import ( type imgTrustStore struct{} -func (its imgTrustStore) VerifySignature( +func (its imgTrustStore) DepVerifySignature( signatureType string, rawSignature []byte, sigKey string, manifestDigest digest.Digest, manifestContent []byte, repo string, ) (string, time.Time, bool, error) { return "", time.Time{}, false, nil } -func (its imgTrustStore) ProtoVerifySignature( +func (its imgTrustStore) VerifySignature( signatureType string, rawSignature []byte, sigKey string, manifestDigest digest.Digest, imageData mTypes.ImageData, repo string, ) (string, time.Time, bool, error) { diff --git a/pkg/meta/boltdb/proto_boltdb.go b/pkg/meta/boltdb/proto_boltdb.go deleted file mode 100644 index 8ac812b2c1..0000000000 --- a/pkg/meta/boltdb/proto_boltdb.go +++ /dev/null @@ -1,1341 +0,0 @@ -package boltdb - -import ( - "context" - "errors" - "fmt" - "strings" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "go.etcd.io/bbolt" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" - - zerr "zotregistry.io/zot/errors" - zcommon "zotregistry.io/zot/pkg/common" - "zotregistry.io/zot/pkg/meta/common" - mConvert "zotregistry.io/zot/pkg/meta/convert" - "zotregistry.io/zot/pkg/meta/proto_go" - mTypes "zotregistry.io/zot/pkg/meta/types" - reqCtx "zotregistry.io/zot/pkg/requestcontext" -) - -func (bdw *BoltDB) SetImageData(digest godigest.Digest, imageData mTypes.ImageData) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(ImageDataBuck)) - - protoImageData := &proto_go.ImageData{} - - switch imageData.MediaType { - case ispec.MediaTypeImageManifest: - manifest := imageData.Manifests[0] - - protoImageData = mConvert.GetProtoImageManifestData(manifest.Manifest, manifest.ConfigContent, - manifest.Size, manifest.Digest.String()) - case ispec.MediaTypeImageIndex: - protoImageData = mConvert.GetProtoImageIndexData(*imageData.Index, imageData.Size, imageData.Digest.String()) - } - - pImageDataBlob, err := proto.Marshal(protoImageData) - if err != nil { - return fmt.Errorf("metadb: error while calculating blob for manifest with digest %s %w", digest, err) - } - - err = buck.Put([]byte(digest), pImageDataBlob) - if err != nil { - return fmt.Errorf("metadb: error while setting manifest data with for digest %s %w", digest, err) - } - - return nil - }) - - return err -} - -func (bdw *BoltDB) SetRepoReference(repo string, reference string, imageData mTypes.ImageData, -) error { - if err := common.ValidateRepoReferenceInput(repo, reference, imageData.Digest); err != nil { - return err - } - - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(RepoMetaBuck)) - repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) - imageBuck := tx.Bucket([]byte(ImageDataBuck)) - - // 1. Add image data to db if needed - - protoImageData := mConvert.GetProtoImageData(imageData) - - imageDataBlob, err := proto.Marshal(protoImageData) - if err != nil { - return err - } - - err = imageBuck.Put([]byte(imageData.Digest), imageDataBlob) - if err != nil { - return err - } - - repoMetaBlob := repoBuck.Get([]byte(repo)) - - repoMeta := &proto_go.RepoMeta{ - Name: repo, - Tags: map[string]*proto_go.TagDescriptor{"": {}}, // This is done so Protobuff can insitialize a non-nill map - Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, - Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, - Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, - } - - if len(repoMetaBlob) > 0 { - err := proto.Unmarshal(repoMetaBlob, repoMeta) - if err != nil { - return err - } - } - - // 2. Referrers - if protoImageData.Subject != nil { - refInfo := &proto_go.ReferrersInfo{} - if repoMeta.Referrers[protoImageData.Subject.Digest] != nil { - refInfo = repoMeta.Referrers[protoImageData.Subject.Digest] - } - - refInfo.List = append(refInfo.List, &proto_go.ReferrerInfo{ - Digest: protoImageData.Digest, - MediaType: protoImageData.MediaType, - ArtifactType: mConvert.GetArtifactType(protoImageData), - Size: protoImageData.Size, - Annotations: protoImageData.Annotations, - }) - - repoMeta.Referrers[protoImageData.Subject.Digest] = refInfo - } - - // 3. Update tag - if !common.ReferenceIsDigest(reference) { - repoMeta.Tags[reference] = &proto_go.TagDescriptor{ - Digest: imageData.Digest.String(), - MediaType: imageData.MediaType, - } - } - - if _, ok := repoMeta.Statistics[imageData.Digest.String()]; !ok { - repoMeta.Statistics[imageData.Digest.String()] = &proto_go.DescriptorStatistics{DownloadCount: 0} - } - - if _, ok := repoMeta.Signatures[imageData.Digest.String()]; !ok { - repoMeta.Signatures[imageData.Digest.String()] = &proto_go.ManifestSignatures{ - Map: map[string]*proto_go.SignaturesInfo{"": {}}, - } - } - - if _, ok := repoMeta.Referrers[imageData.Digest.String()]; !ok { - repoMeta.Referrers[imageData.Digest.String()] = &proto_go.ReferrersInfo{ - List: []*proto_go.ReferrerInfo{}, - } - } - - // 4. Blobs - repoBlobsBytes := repoBlobsBuck.Get([]byte(repoMeta.Name)) - - repoBlobs := &proto_go.RepoBlobs{} - - if repoBlobsBytes == nil { - repoBlobs.Blobs = map[string]*proto_go.BlobInfo{} - } else { - err := proto.Unmarshal(repoBlobsBytes, repoBlobs) - if err != nil { - return err - } - } - - repoMeta, repoBlobs, err = common.AddImageDataToRepoMeta(repoMeta, repoBlobs, reference, imageData) - if err != nil { - return err - } - - repoBlobsBytes, err = proto.Marshal(repoBlobs) - if err != nil { - return err - } - - err = repoBlobsBuck.Put([]byte(repoMeta.Name), repoBlobsBytes) - if err != nil { - return err - } - - repoMetaBlob, err = proto.Marshal(repoMeta) - if err != nil { - return err - } - - return repoBuck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) FilterImageData(ctx context.Context, digests []string, -) (map[string]mTypes.ImageData, error) { - imageDataMap := map[string]mTypes.ImageData{} - - err := bdw.DB.View(func(transaction *bbolt.Tx) error { - imageBuck := transaction.Bucket([]byte(ImageDataBuck)) - - for _, digest := range digests { - protoImageData, err := fetchProtoImageData(imageBuck, digest) - if err != nil { - return err - } - - if protoImageData.MediaType == ispec.MediaTypeImageIndex { - for i, manifest := range protoImageData.Manifests { - manifestDigest := manifest.Digest - - imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) - if err != nil { - return err - } - - protoImageData.Manifests[i] = imageManifestData.Manifests[0] - } - } - - imageDataMap[digest] = mConvert.GetImageData(protoImageData) - } - - return nil - }) - - return imageDataMap, err -} - -func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, -) ([]mTypes.FullRepoMetadata, error) { - repos := []mTypes.FullRepoMetadata{} - - err := bdw.DB.View(func(transaction *bbolt.Tx) error { - var ( - repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) - userBookmarks = getUserBookmarks(ctx, transaction) - userStars = getUserStars(ctx, transaction) - ) - - cursor := repoBuck.Cursor() - - for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } - - rank := common.RankRepoName(searchText, string(repoName)) - if rank == -1 { - continue - } - - var protoRepoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) - if err != nil { - return err - } - - delete(protoRepoMeta.Tags, "") - - protoRepoMeta.Rank = int32(rank) - protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) - protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) - - repos = append(repos, mConvert.GetFullRepoMeta(&protoRepoMeta)) - } - - return nil - }) - - return repos, err -} - -func fetchProtoImageData(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageData, error) { - imageDataBlob := imageBuck.Get([]byte(digest)) - - if len(imageDataBlob) == 0 { - return nil, zerr.ErrImageDataNotFound - } - - imageData := proto_go.ImageData{} - - err := proto.Unmarshal(imageDataBlob, &imageData) - if err != nil { - return nil, err - } - - return &imageData, nil -} - -func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, -) ([]mTypes.FullImageData, error) { - images := []mTypes.FullImageData{} - - searchedRepo, searchedTag, err := common.GetRepoTag(searchText) - if err != nil { - return []mTypes.FullImageData{}, - fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) - } - - err = bdw.DB.View(func(transaction *bbolt.Tx) error { - var ( - repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) - imageBuck = transaction.Bucket([]byte(ImageDataBuck)) - userBookmarks = getUserBookmarks(ctx, transaction) - userStars = getUserStars(ctx, transaction) - ) - - repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo)) - - if string(repoName) != searchedRepo { - return nil - } - - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - return err - } - - protoRepoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) - if err != nil { - return err - } - - delete(protoRepoMeta.Tags, "") - - protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) - protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) - - for tag, descriptor := range protoRepoMeta.Tags { - if !strings.HasPrefix(tag, searchedTag) || tag == "" { - continue - } - - var protoImageData *proto_go.ImageData - - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest - - imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) - if err != nil { - return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", - manifestDigest, err) - } - - imageManifestData.Repo = ref(string(repoName)) - imageManifestData.Tag = &tag - - protoImageData = imageManifestData - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - imageIndexData, err := fetchProtoImageData(imageBuck, indexDigest) - if err != nil { - return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", - indexDigest, err) - } - - for i, manifest := range imageIndexData.Manifests { - manifestDigest := manifest.Digest - - imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) - if err != nil { - return err - } - - imageIndexData.Manifests[i] = imageManifestData.Manifests[0] - } - - imageIndexData.Repo = ref(string(repoName)) - imageIndexData.Tag = &tag - - protoImageData = imageIndexData - default: - bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue - } - - images = append(images, mConvert.GetFullImageDataFromProto(tag, protoRepoMeta, protoImageData)) - } - - return nil - }) - - return images, err -} - -func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, - filterFunc mTypes.FilterFunc, -) ([]mTypes.FullImageData, error) { - images := []mTypes.FullImageData{} - - err := bdw.DB.View(func(transaction *bbolt.Tx) error { - var ( - repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) - imageDataBuck = transaction.Bucket([]byte(ImageDataBuck)) - userBookmarks = getUserBookmarks(ctx, transaction) - userStars = getUserStars(ctx, transaction) - ) - - cursor := repoBuck.Cursor() - repoName, repoMetaBlob := cursor.First() - - for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } - - protoRepoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) - if err != nil { - return err - } - - delete(protoRepoMeta.Tags, "") - protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) - protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) - repoMeta := mConvert.GetRepoMeta(protoRepoMeta) - - for tag, descriptor := range protoRepoMeta.Tags { - if !filterRepoTag(string(repoName), tag) { - continue - } - - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest - - imageManifestData, err := fetchProtoImageData(imageDataBuck, manifestDigest) - if err != nil { - return fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err) - } - - imageManifestData.Repo = ref(repoMeta.Name) - imageManifestData.Tag = ref(tag) - - imageData := mConvert.GetImageData(imageManifestData) - - if filterFunc(repoMeta, imageData) { - images = append(images, mConvert.GetFullImageDataFromProto(tag, protoRepoMeta, imageManifestData)) - } - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - imageIndexData, err := fetchProtoImageData(imageDataBuck, indexDigest) - if err != nil { - return fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err) - } - - matchedManifests := []*proto_go.ManifestData{} - - for _, manifest := range imageIndexData.Manifests { - manifestDigest := manifest.Digest - - imageManifestData, err := fetchProtoImageData(imageDataBuck, manifestDigest) - if err != nil { - return fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err) - } - - imageData := mConvert.GetImageData(imageManifestData) - - if filterFunc(repoMeta, imageData) { - matchedManifests = append(matchedManifests, imageManifestData.Manifests[0]) - } - } - - if len(matchedManifests) > 0 { - imageIndexData.Manifests = matchedManifests - - images = append(images, mConvert.GetFullImageDataFromProto(tag, protoRepoMeta, imageIndexData)) - } - default: - bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue - } - } - } - - return nil - }) - - return images, err -} - -func (bdw *BoltDB) FilterRepos(ctx context.Context, rankName mTypes.FilterRepoNameFunc, - filter mTypes.FilterFullRepoFunc, -) ([]mTypes.FullRepoMetadata, error) { - repos := []mTypes.FullRepoMetadata{} - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - var ( - buck = tx.Bucket([]byte(RepoMetaBuck)) - cursor = buck.Cursor() - userBookmarks = getUserBookmarks(ctx, tx) - userStars = getUserStars(ctx, tx) - ) - - for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } - - rank := rankName(string(repoName)) - if rank < 0 { - continue - } - - repoMeta := proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - repoMeta.Rank = int32(rank) - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - - fullRepoMeta := mConvert.GetFullRepoMeta(&repoMeta) - - if filter(fullRepoMeta) { - repos = append(repos, fullRepoMeta) - } - } - - return nil - }) - if err != nil { - return []mTypes.FullRepoMetadata{}, err - } - - return repos, err -} - -func (bdw *BoltDB) GetRepoMeta(repo string) (mTypes.RepoMetadata, error) { - var protoRepoMeta proto_go.RepoMeta - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - - // object not found - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - // object found - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) - if err != nil { - return err - } - - delete(protoRepoMeta.Tags, "") - - return nil - }) - - return mConvert.GetRepoMeta(&protoRepoMeta), err -} - -func (bdw *BoltDB) GetFullRepoMeta(ctx context.Context, repo string) (mTypes.FullRepoMetadata, error) { - var protoRepoMeta proto_go.RepoMeta - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - userBookmarks := getUserBookmarks(ctx, tx) - userStars := getUserStars(ctx, tx) - - repoMetaBlob := buck.Get([]byte(repo)) - - // object not found - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - // object found - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) - if err != nil { - return err - } - - delete(protoRepoMeta.Tags, "") - protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) - protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) - - return nil - }) - - return mConvert.GetFullRepoMeta(&protoRepoMeta), err -} - -func (bdw *BoltDB) GetFullImageData(ctx context.Context, repo string, tag string) (mTypes.FullImageData, error) { - protoRepoMeta := &proto_go.RepoMeta{} - protoImageData := &proto_go.ImageData{} - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - imageBuck := tx.Bucket([]byte(ImageDataBuck)) - userBookmarks := getUserBookmarks(ctx, tx) - userStars := getUserStars(ctx, tx) - - repoMetaBlob := buck.Get([]byte(repo)) - - // object not found - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - // object found - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) - if err != nil { - return err - } - - delete(protoRepoMeta.Tags, "") - protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) - protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) - - descriptor, ok := protoRepoMeta.Tags[tag] - if !ok { - return zerr.ErrImageDataNotFound - } - - protoImageData, err := fetchProtoImageData(imageBuck, descriptor.Digest) - if err != nil { - return err - } - - if protoImageData.MediaType == ispec.MediaTypeImageIndex { - for i, manifest := range protoImageData.Manifests { - manifestDigest := manifest.Digest - - imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) - if err != nil { - return err - } - - protoImageData.Manifests[i] = imageManifestData.Manifests[0] - } - } - - return nil - }) - - return mConvert.GetFullImageDataFromProto(tag, protoRepoMeta, protoImageData), err -} - -func (bdw *BoltDB) GetImageData(digest godigest.Digest) (mTypes.ImageData, error) { - imageData := mTypes.ImageData{} - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - imageBuck := tx.Bucket([]byte(ImageDataBuck)) - - protoImageData, err := fetchProtoImageData(imageBuck, digest.String()) - if err != nil { - return err - } - - if protoImageData.MediaType == ispec.MediaTypeImageIndex { - for i, manifest := range protoImageData.Manifests { - manifestDigest := manifest.Digest - - imageManifestData, err := fetchProtoImageData(imageBuck, manifestDigest) - if err != nil { - return err - } - - protoImageData.Manifests[i] = imageManifestData.Manifests[0] - } - } - - imageData = mConvert.GetImageData(protoImageData) - - return nil - }) - - return imageData, err -} - -func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, -) ([]mTypes.RepoMetadata, error) { - foundRepos := []mTypes.RepoMetadata{} - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - cursor := buck.Cursor() - - for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } - - protoRepoMeta := proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) - if err != nil { - return err - } - - delete(protoRepoMeta.Tags, "") - - repoMeta := mConvert.GetRepoMeta(&protoRepoMeta) - - if filter(repoMeta) { - foundRepos = append(foundRepos, repoMeta) - } - } - - return nil - }) - - return foundRepos, err -} - -func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, - sygMeta mTypes.SignatureMetadata, -) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - - if len(repoMetaBlob) == 0 { - var err error - // create a new object - repoMeta := proto_go.RepoMeta{ - Name: repo, - Tags: map[string]*proto_go.TagDescriptor{}, - Signatures: map[string]*proto_go.ManifestSignatures{ - signedManifestDigest.String(): { - Map: map[string]*proto_go.SignaturesInfo{ - sygMeta.SignatureType: { - List: []*proto_go.SignatureInfo{ - { - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), - }, - }, - }, - }, - }, - }, - Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, - } - - repoMetaBlob, err = proto.Marshal(&repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - } - - protoRepoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) - if err != nil { - return err - } - - var ( - manifestSignatures *proto_go.ManifestSignatures - found bool - ) - - if manifestSignatures, found = protoRepoMeta.Signatures[signedManifestDigest.String()]; !found { - manifestSignatures = &proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} - } - - signatureSlice := &proto_go.SignaturesInfo{List: []*proto_go.SignatureInfo{}} - if sigSlice, found := manifestSignatures.Map[sygMeta.SignatureType]; found { - signatureSlice = sigSlice - } - - if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sygMeta) { - switch sygMeta.SignatureType { - case zcommon.NotationSignature: - signatureSlice.List = append(signatureSlice.List, &proto_go.SignatureInfo{ - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), - }) - case zcommon.CosignSignature: - signatureSlice.List = []*proto_go.SignatureInfo{{ - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), - }} - } - } - - manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice - protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - - repoMetaBlob, err = proto.Marshal(protoRepoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Digest, - sigMeta mTypes.SignatureMetadata, -) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrManifestMetaNotFound - } - - protoRepoMeta := proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) - if err != nil { - return err - } - - manifestSignatures, found := protoRepoMeta.Signatures[signedManifestDigest.String()] - if !found { - return zerr.ErrManifestMetaNotFound - } - - signatureSlice := manifestSignatures.Map[sigMeta.SignatureType] - - newSignatureSlice := make([]*proto_go.SignatureInfo, 0, len(signatureSlice.List)) - - for _, sigInfo := range signatureSlice.List { - if sigInfo.SignatureManifestDigest != sigMeta.SignatureDigest { - newSignatureSlice = append(newSignatureSlice, sigInfo) - } - } - - manifestSignatures.Map[sigMeta.SignatureType] = &proto_go.SignaturesInfo{List: newSignatureSlice} - - protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - - repoMetaBlob, err = proto.Marshal(&protoRepoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) IncrementRepoStars(repo string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - repoMeta.Stars++ - - repoMetaBlob, err = proto.Marshal(&repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) DecrementRepoStars(repo string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - if repoMeta.Stars == 0 { - return nil - } - - repoMeta.Stars-- - - repoMetaBlob, err = proto.Marshal(&repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) GetRepoStars(repo string) (int, error) { - stars := 0 - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - buck.Get([]byte(repo)) - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - stars = int(repoMeta.Stars) - - return nil - }) - - return stars, err -} - -func (bdw *BoltDB) DeleteRepoTag(repo string, tag string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - - // object not found - if repoMetaBlob == nil { - return nil - } - - // object found - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - delete(repoMeta.Tags, tag) - - repoMetaBlob, err = proto.Marshal(&repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) GetUserRepoMeta(ctx context.Context, repo string) (mTypes.RepoMetadata, error) { - var repoMeta proto_go.RepoMeta - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - userBookmarks := getUserBookmarks(ctx, tx) - userStars := getUserStars(ctx, tx) - - repoMetaBlob := buck.Get([]byte(repo)) - - // object not found - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - // object found - err := proto.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) - repoMeta.IsStarred = zcommon.Contains(userStars, repo) - - return nil - }) - - return mConvert.GetRepoMeta(&repoMeta), err -} - -func (bdw *BoltDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMetadata) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMeta.Name = repo - - repoMetaBlob, err := proto.Marshal(mConvert.GetProtoRepoMeta(repoMeta)) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) ResetRepoRefferences(repo string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - - if repoMetaBlob == nil { - return nil - } - - protoRepoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) - if err != nil { - return err - } - - repoMetaBlob, err = proto.Marshal(&proto_go.RepoMeta{ - Name: repo, - Statistics: protoRepoMeta.Statistics, - Stars: protoRepoMeta.Stars, - Tags: map[string]*proto_go.TagDescriptor{"": {}}, - Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, - Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, - }) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, -) ([]mTypes.ReferrerInfo, error) { - referrersInfoResult := []mTypes.ReferrerInfo{} - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - if len(repoMetaBlob) == 0 { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - referrersInfo := repoMeta.Referrers[referredDigest.String()].List - - for i := range referrersInfo { - if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) { - continue - } - - referrersInfoResult = append(referrersInfoResult, mTypes.ReferrerInfo{ - Digest: referrersInfo[i].Digest, - MediaType: referrersInfo[i].MediaType, - ArtifactType: referrersInfo[i].ArtifactType, - Size: int(referrersInfo[i].Size), - Annotations: referrersInfo[i].Annotations, - }) - } - - return nil - }) - - return referrersInfoResult, err -} - -func (bdw *BoltDB) IncrementImageDownloads(repo string, reference string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - manifestDigest := reference - - if !common.ReferenceIsDigest(reference) { - // search digest for tag - descriptor, found := repoMeta.Tags[reference] - - if !found { - return zerr.ErrManifestMetaNotFound - } - - manifestDigest = descriptor.Digest - } - - manifestStatistics, ok := repoMeta.Statistics[manifestDigest] - if !ok { - return zerr.ErrManifestMetaNotFound - } - - manifestStatistics.DownloadCount++ - repoMeta.Statistics[manifestDigest] = manifestStatistics - - repoMetaBlob, err = proto.Marshal(&repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error { - err := bdw.DB.Update(func(transaction *bbolt.Tx) error { - imgTrustStore := bdw.ImageTrustStore() - - if imgTrustStore == nil { - return nil - } - - // get ManifestData of signed manifest - imageDataBuck := transaction.Bucket([]byte(ImageDataBuck)) - idBlob := imageDataBuck.Get([]byte(manifestDigest)) - - if len(idBlob) == 0 { - // manifest meta not found, updating signatures with details about validity and author will not be performed - return nil - } - - protoImageData := proto_go.ImageData{} - - err := proto.Unmarshal(idBlob, &protoImageData) - if err != nil { - return err - } - - // update signatures with details about validity and author - repoBuck := transaction.Bucket([]byte(RepoMetaBuck)) - - repoMetaBlob := repoBuck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - protoRepoMeta := proto_go.RepoMeta{} - - err = proto.Unmarshal(repoMetaBlob, &protoRepoMeta) - if err != nil { - return err - } - - manifestSignatures := proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} - for sigType, sigs := range protoRepoMeta.Signatures[manifestDigest.String()].Map { - signaturesInfo := []*proto_go.SignatureInfo{} - - for _, sigInfo := range sigs.List { - layersInfo := []*proto_go.LayersInfo{} - - for _, layerInfo := range sigInfo.LayersInfo { - author, date, isTrusted, _ := imgTrustStore.ProtoVerifySignature(sigType, layerInfo.LayerContent, - layerInfo.SignatureKey, manifestDigest, mConvert.GetImageData(&protoImageData), repo) - - if isTrusted { - layerInfo.Signer = author - } - - if !date.IsZero() { - layerInfo.Signer = author - layerInfo.Date = timestamppb.New(date) - } - - layersInfo = append(layersInfo, layerInfo) - } - - signaturesInfo = append(signaturesInfo, &proto_go.SignatureInfo{ - SignatureManifestDigest: sigInfo.SignatureManifestDigest, - LayersInfo: layersInfo, - }) - } - - manifestSignatures.Map[sigType] = &proto_go.SignaturesInfo{List: signaturesInfo} - } - - protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures - - repoMetaBlob, err = proto.Marshal(&protoRepoMeta) - if err != nil { - return err - } - - return repoBuck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) - imageDataBuck := tx.Bucket([]byte(ImageDataBuck)) - repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return nil - } - - repoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, repoMeta) - if err != nil { - return err - } - - imageData, err := fetchProtoImageData(imageDataBuck, manifestDigest.String()) - if err != nil { - if errors.Is(err, zerr.ErrImageDataNotFound) { - return nil - } - - return err - } - - // Remove Referes - if imageData.Subject != nil { - referredDigest := imageData.Subject.Digest - refInfo := &proto_go.ReferrersInfo{} - - if repoMeta.Referrers[referredDigest] != nil { - refInfo = repoMeta.Referrers[referredDigest] - } - - referrers := refInfo.List - - for i := range referrers { - if referrers[i].Digest == manifestDigest.String() { - referrers = append(referrers[:i], referrers[i+1:]...) - - break - } - } - - refInfo.List = referrers - - repoMeta.Referrers[referredDigest] = refInfo - } - - if !common.ReferenceIsDigest(reference) { - delete(repoMeta.Tags, reference) - } else { - // remove all tags pointing to this digest - for tag, desc := range repoMeta.Tags { - if desc.Digest == reference { - delete(repoMeta.Tags, tag) - } - } - } - - /* try to find at least one tag pointing to manifestDigest - if not found then we can also remove everything related to this digest */ - var foundTag bool - for _, desc := range repoMeta.Tags { - if desc.Digest == manifestDigest.String() { - foundTag = true - } - } - - if !foundTag { - delete(repoMeta.Statistics, manifestDigest.String()) - delete(repoMeta.Signatures, manifestDigest.String()) - delete(repoMeta.Referrers, manifestDigest.String()) - } - - repoBlobsBytes := repoBlobsBuck.Get([]byte(repoMeta.Name)) - - repoBlobs := &proto_go.RepoBlobs{} - - if repoBlobsBytes == nil { - repoBlobs.Blobs = map[string]*proto_go.BlobInfo{} - } else { - err := proto.Unmarshal(repoBlobsBytes, repoBlobs) - if err != nil { - return err - } - } - - repoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(repoMeta, repoBlobs, reference) - if err != nil { - return err - } - - repoBlobsBytes, err = proto.Marshal(repoBlobs) - if err != nil { - return err - } - - err = repoBlobsBuck.Put([]byte(repoMeta.Name), repoBlobsBytes) - if err != nil { - return err - } - - repoMetaBlob, err = proto.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func ref[T any](input T) *T { - ref := input - - return &ref -} diff --git a/pkg/meta/common/common.go b/pkg/meta/common/common.go index 207a0583e5..c09e808384 100644 --- a/pkg/meta/common/common.go +++ b/pkg/meta/common/common.go @@ -262,31 +262,6 @@ func FilterDataByRepo(foundRepos []mTypes.DepRepoMetadata, manifestMetadataMap m return foundManifestMetadataMap, foundindexDataMap, nil } -// FindMediaTypeForDigest will look into the buckets for a certain digest. Depending on which bucket that -// digest is found the corresponding mediatype is returned. -func FindMediaTypeForDigest(metaDB mTypes.MetaDB, digest godigest.Digest) (bool, string) { - imageData, err := metaDB.GetImageData(digest) - if err == nil { - return true, imageData.MediaType - } - - return false, "" -} - -func GetImageDescriptor(metaDB mTypes.MetaDB, repo, tag string) (mTypes.Descriptor, error) { - repoMeta, err := metaDB.GetRepoMeta(repo) - if err != nil { - return mTypes.Descriptor{}, err - } - - imageDescriptor, ok := repoMeta.Tags[tag] - if !ok { - return mTypes.Descriptor{}, zerr.ErrTagMetaNotFound - } - - return imageDescriptor, nil -} - func InitializeImageConfig(blob []byte) ispec.Image { var configContent ispec.Image diff --git a/pkg/meta/convert/convert.go b/pkg/meta/convert/convert.go index c1a0867d48..2769cdd4b2 100644 --- a/pkg/meta/convert/convert.go +++ b/pkg/meta/convert/convert.go @@ -36,13 +36,13 @@ func getHistory(history []*proto_go.History) []ispec.History { func GetArtifactType(imageData *proto_go.ImageData) string { switch imageData.MediaType { case ispec.MediaTypeImageManifest: - if imageData.ArtifacType != "" { - return imageData.ArtifacType + if imageData.ArtifactType != "" { + return imageData.ArtifactType } return imageData.Manifests[0].Config.MediaType case ispec.MediaTypeImageIndex: - return imageData.ArtifacType + return imageData.ArtifactType } return "" @@ -396,26 +396,6 @@ func GetLastUpdated(timestamp *timestamppb.Timestamp) *time.Time { return ref(timestamp.AsTime()) } -func AddPlatforms(platforms []ispec.Platform, newPlatforms []ispec.Platform) []ispec.Platform { - for _, newPlatform := range newPlatforms { - if !ContainsPlatform(platforms, newPlatform) { - platforms = append(platforms, newPlatform) - } - } - - return platforms -} - -func ContainsPlatform(platforms []ispec.Platform, platform ispec.Platform) bool { - for i := range platforms { - if platforms[i].OS == platform.OS || platforms[i].Architecture == platform.Architecture { - return true - } - } - - return false -} - func AddProtoPlatforms(platforms []*proto_go.Platform, newPlatforms []*proto_go.Platform) []*proto_go.Platform { for _, newPlatform := range newPlatforms { if !ContainsProtoPlatform(platforms, newPlatform) { @@ -446,22 +426,6 @@ func AddVendors(vendors []string, newVendors []string) []string { return vendors } -func GetEarlierLastUpdated(lastUpdated *time.Time, newLastUpdated *time.Time) *time.Time { - if newLastUpdated == nil { - return lastUpdated - } - - if lastUpdated == nil { - return newLastUpdated - } - - if lastUpdated.Before(*newLastUpdated) { - return newLastUpdated - } - - return lastUpdated -} - func GetLastUpdatedImage(protoLastUpdated *proto_go.RepoLastUpdatedImage) *mTypes.LastUpdatedImage { if protoLastUpdated == nil { return nil @@ -508,7 +472,7 @@ func GetImageData(dbImageData *proto_go.ImageData) mTypes.ImageData { imageData.Index = &ispec.Index{ Versioned: specs.Versioned{SchemaVersion: int(dbImageData.Versioned.GetSchemaVersion())}, MediaType: ispec.MediaTypeImageIndex, - ArtifactType: dbImageData.ArtifacType, + ArtifactType: dbImageData.ArtifactType, Manifests: manifests, Subject: subject, Annotations: dbImageData.Annotations, diff --git a/pkg/meta/convert/convert_proto.go b/pkg/meta/convert/convert_proto.go index 29084b8b7d..888c26653c 100644 --- a/pkg/meta/convert/convert_proto.go +++ b/pkg/meta/convert/convert_proto.go @@ -45,14 +45,14 @@ func GetProtoImageData(imageData mTypes.ImageData) *proto_go.ImageData { func GetProtoImageManifestData(manifestContent ispec.Manifest, configContent ispec.Image, size int64, digest string, ) *proto_go.ImageData { return &proto_go.ImageData{ - Versioned: &proto_go.Versioned{SchemaVersion: int32(manifestContent.SchemaVersion)}, - MediaType: ispec.MediaTypeImageManifest, - Manifests: []*proto_go.ManifestData{GetProtoManifestData(manifestContent, configContent, size, digest)}, - ArtifacType: manifestContent.ArtifactType, - Subject: getProtoDesc(manifestContent.Subject), - Annotations: manifestContent.Annotations, - Size: size, - Digest: digest, + Versioned: &proto_go.Versioned{SchemaVersion: int32(manifestContent.SchemaVersion)}, + MediaType: ispec.MediaTypeImageManifest, + Manifests: []*proto_go.ManifestData{GetProtoManifestData(manifestContent, configContent, size, digest)}, + ArtifactType: manifestContent.ArtifactType, + Subject: getProtoDesc(manifestContent.Subject), + Annotations: manifestContent.Annotations, + Size: size, + Digest: digest, } } @@ -99,14 +99,14 @@ func GetProtoManifestData(manifestContent ispec.Manifest, configContent ispec.Im func GetProtoImageIndexData(indexContent ispec.Index, size int64, digest string) *proto_go.ImageData { return &proto_go.ImageData{ - Versioned: &proto_go.Versioned{SchemaVersion: int32(indexContent.SchemaVersion)}, - MediaType: ispec.MediaTypeImageIndex, - ArtifacType: common.GetIndexArtifactType(indexContent), - Manifests: getProtoManifests(indexContent.Manifests), - Subject: getProtoDesc(indexContent.Subject), - Annotations: indexContent.Annotations, - Size: size, - Digest: digest, + Versioned: &proto_go.Versioned{SchemaVersion: int32(indexContent.SchemaVersion)}, + MediaType: ispec.MediaTypeImageIndex, + ArtifactType: common.GetIndexArtifactType(indexContent), + Manifests: getProtoManifests(indexContent.Manifests), + Subject: getProtoDesc(indexContent.Subject), + Annotations: indexContent.Annotations, + Size: size, + Digest: digest, } } @@ -226,9 +226,6 @@ func getProtoPlatform(platform *ispec.Platform) *proto_go.Platform { return &proto_go.Platform{ Arch: getArch(platform.Architecture, platform.Variant), OS: platform.OS, - // Osversion: &platform.OSVersion, - // Osfeatures: platform.OSFeatures, - // Variant: &platform.Variant, } } @@ -280,7 +277,7 @@ func getProtoConfigVolumes(volumes map[string]struct{}) map[string]*proto_go.Emp return protoVolumes } -func GetProtoRefferrerInfo(referrer mTypes.ReferrerInfo) *proto_go.ReferrerInfo { +func GetProtoReferrerInfo(referrer mTypes.ReferrerInfo) *proto_go.ReferrerInfo { return &proto_go.ReferrerInfo{ Digest: referrer.Digest, MediaType: referrer.MediaType, diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index 8ed0de6785..434db32294 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -62,32 +62,27 @@ func New( return nil, err } - err = dynamoWrapper.createProtoRepoMetaTable() + err = dynamoWrapper.createTable(dynamoWrapper.RepoMetaTablename) if err != nil { return nil, err } - err = dynamoWrapper.createProtoRepoBlobsTable() + err = dynamoWrapper.createTable(dynamoWrapper.RepoBlobsTablename) if err != nil { return nil, err } - err = dynamoWrapper.createProtoImageDataTable() + err = dynamoWrapper.createTable(dynamoWrapper.ImageDataTablename) if err != nil { return nil, err } - err = dynamoWrapper.createRepoMetaTable() + err = dynamoWrapper.createTable(dynamoWrapper.UserDataTablename) if err != nil { return nil, err } - err = dynamoWrapper.createUserDataTable() - if err != nil { - return nil, err - } - - err = dynamoWrapper.createAPIKeyTable() + err = dynamoWrapper.createTable(dynamoWrapper.APIKeyTablename) if err != nil { return nil, err } @@ -123,7 +118,7 @@ func (dwr *DynamoDB) SetProtoImageData(digest godigest.Digest, protoImageData *p ":ImageData": mdAttributeValue, }, Key: map[string]types.AttributeValue{ - "Digest": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: protoImageData.Digest, }, }, @@ -143,7 +138,7 @@ func (dwr *DynamoDB) GetProtoImageData(ctx context.Context, digest godigest.Dige resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ TableName: aws.String(dwr.ImageDataTablename), Key: map[string]types.AttributeValue{ - "Digest": &types.AttributeValueMemberS{Value: digest.String()}, + "Key": &types.AttributeValueMemberS{Value: digest.String()}, }, }) if err != nil { @@ -192,7 +187,7 @@ func (dwr *DynamoDB) setProtoRepoMeta(repo string, repoMeta *proto_go.RepoMeta) ":RepoMetadata": repoAttributeValue, }, Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: repo, }, }, @@ -207,7 +202,7 @@ func (dwr *DynamoDB) getProtoRepoMeta(ctx context.Context, repo string) (*proto_ resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoMetaTablename), Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, + "Key": &types.AttributeValueMemberS{Value: repo}, }, }) if err != nil { @@ -328,7 +323,7 @@ func (dwr *DynamoDB) getRepoBlobsInfo(repo string) (*proto_go.RepoBlobs, error) resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoBlobsTablename), Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, + "Key": &types.AttributeValueMemberS{Value: repo}, }, }) if err != nil { @@ -378,7 +373,7 @@ func (dwr *DynamoDB) setRepoBlobsInfo(repo string, repoBlobs *proto_go.RepoBlobs ":RepoBlobsInfo": mdAttributeValue, }, Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: repo, }, }, @@ -534,6 +529,8 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter userBookmarks := getUserBookmarks(ctx, dwr) userStars := getUserStars(ctx, dwr) + var viewError error + repoMetaAttributeIterator := NewBaseDynamoAttributesIterator( dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, ) @@ -542,13 +539,16 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { if err != nil { - return []mTypes.FullImageData{}, - err + viewError = errors.Join(viewError, err) + + continue } protoRepoMeta, err := getProtoRepoMetaFromAttribute(repoMetaAttribute) if err != nil { - return nil, err + viewError = errors.Join(viewError, err) + + continue } if ok, err := reqCtx.RepoIsUserAvailable(ctx, protoRepoMeta.Name); !ok || err != nil { @@ -570,8 +570,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter imageManifestData, err := dwr.GetProtoImageData(ctx, godigest.Digest(manifestDigest)) if err != nil { - return []mTypes.FullImageData{}, - fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", manifestDigest, err) + viewError = errors.Join(viewError, err) + + continue } imageManifestData.Repo = ref(repoMeta.Name) @@ -587,8 +588,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter imageIndexData, err := dwr.GetProtoImageData(ctx, godigest.Digest(indexDigest)) if err != nil { - return []mTypes.FullImageData{}, - fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", indexDigest, err) + viewError = errors.Join(viewError, err) + + continue } matchedManifests := []*proto_go.ManifestData{} @@ -598,8 +600,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter imageManifestData, err := dwr.GetProtoImageData(ctx, godigest.Digest(manifestDigest)) if err != nil { - return []mTypes.FullImageData{}, - fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err) + viewError = errors.Join(viewError, err) + + continue } imageData := mConvert.GetImageData(imageManifestData) @@ -622,7 +625,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter } } - return images, err + viewError = errors.Join(viewError, err) + + return images, viewError } func getProtoRepoMetaFromAttribute(repoMetaAttribute types.AttributeValue) (*proto_go.RepoMeta, error) { @@ -672,7 +677,7 @@ func (dwr *DynamoDB) GetRepoMeta(repo string) (mTypes.RepoMetadata, error) { return mConvert.GetRepoMeta(protoRepoMeta), nil } -func (dwr *DynamoDB) ResetRepoRefferences(repo string) error { +func (dwr *DynamoDB) ResetRepoReferences(repo string) error { protoRepoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) if err != nil { return err @@ -817,7 +822,7 @@ func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMe return foundRepos, err } -func (dwr *DynamoDB) FilterRepos(ctx context.Context, rankName mTypes.FilterRepoNameFunc, +func (dwr *DynamoDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepoNameFunc, filterFunc mTypes.FilterFullRepoFunc, ) ([]mTypes.FullRepoMetadata, error) { repos := []mTypes.FullRepoMetadata{} @@ -845,12 +850,10 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context, rankName mTypes.FilterRepo continue } - rank := rankName(protoRepoMeta.Name) - if rank < 0 { + if !acceptName(protoRepoMeta.Name) { continue } - protoRepoMeta.Rank = int32(rank) protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) @@ -899,7 +902,7 @@ func (dwr *DynamoDB) DeleteRepoTag(repo string, tag string) error { resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoMetaTablename), Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, + "Key": &types.AttributeValueMemberS{Value: repo}, }, }) if err != nil { @@ -944,7 +947,7 @@ func (dwr *DynamoDB) DeleteRepoTag(repo string, tag string) error { ":RepoMetadata": repoAttributeValue, }, Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: repo, }, }, @@ -1054,7 +1057,7 @@ func (dwr *DynamoDB) UpdateSignaturesValidity(repo string, manifestDigest godige layersInfo := []*proto_go.LayersInfo{} for _, layerInfo := range sigInfo.LayersInfo { - author, date, isTrusted, _ := imgTrustStore.ProtoVerifySignature(sigType, layerInfo.LayerContent, + author, date, isTrusted, _ := imgTrustStore.VerifySignature(sigType, layerInfo.LayerContent, layerInfo.SignatureKey, manifestDigest, mConvert.GetImageData(protoImageData), repo) if isTrusted { @@ -1249,7 +1252,7 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest return err } - // Remove Referes + // Remove Referrers if protoImageData.Subject != nil { referredDigest := protoImageData.Subject.Digest refInfo := &proto_go.ReferrersInfo{} @@ -1466,7 +1469,7 @@ func (dwr *DynamoDB) ToggleStarRepo(ctx context.Context, repo string) ( ":UserData": userAttributeValue, }, Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: userid, }, }, @@ -1484,7 +1487,7 @@ func (dwr *DynamoDB) ToggleStarRepo(ctx context.Context, repo string) ( ":RepoMetadata": repoAttributeValue, }, Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: repo, }, }, @@ -1512,56 +1515,6 @@ func (dwr *DynamoDB) GetStarredRepos(ctx context.Context) ([]string, error) { return userMeta.StarredRepos, err } -func (dwr *DynamoDB) createUserDataTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.UserDataTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("Identity"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("Identity"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - - if err != nil && !strings.Contains(err.Error(), "Table already exists") { - return err - } - - return dwr.waitTableToBeCreated(dwr.UserDataTablename) -} - -func (dwr DynamoDB) createAPIKeyTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.APIKeyTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("HashedKey"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("HashedKey"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - - if err != nil && !strings.Contains(err.Error(), "Table already exists") { - return err - } - - return dwr.waitTableToBeCreated(dwr.APIKeyTablename) -} - func (dwr DynamoDB) SetUserGroups(ctx context.Context, groups []string) error { userData, err := dwr.GetUserData(ctx) if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { @@ -1710,7 +1663,7 @@ func (dwr DynamoDB) AddUserAPIKey(ctx context.Context, hashedKey string, apiKeyD ":UserData": userAttributeValue, }, Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: userid, }, }, @@ -1728,7 +1681,7 @@ func (dwr DynamoDB) AddUserAPIKey(ctx context.Context, hashedKey string, apiKeyD ":Identity": &types.AttributeValueMemberS{Value: userid}, }, Key: map[string]types.AttributeValue{ - "HashedKey": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: hashedKey, }, }, @@ -1755,7 +1708,7 @@ func (dwr DynamoDB) DeleteUserAPIKey(ctx context.Context, keyID string) error { _, err = dwr.Client.DeleteItem(ctx, &dynamodb.DeleteItemInput{ TableName: aws.String(dwr.APIKeyTablename), Key: map[string]types.AttributeValue{ - "HashedKey": &types.AttributeValueMemberS{Value: hash}, + "Key": &types.AttributeValueMemberS{Value: hash}, }, }) if err != nil { @@ -1777,7 +1730,7 @@ func (dwr DynamoDB) GetUserAPIKeyInfo(hashedKey string) (string, error) { resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.APIKeyTablename), Key: map[string]types.AttributeValue{ - "HashedKey": &types.AttributeValueMemberS{Value: hashedKey}, + "Key": &types.AttributeValueMemberS{Value: hashedKey}, }, }) if err != nil { @@ -1813,7 +1766,7 @@ func (dwr DynamoDB) GetUserData(ctx context.Context) (mTypes.UserData, error) { resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ TableName: aws.String(dwr.UserDataTablename), Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{Value: userid}, + "Key": &types.AttributeValueMemberS{Value: userid}, }, }) if err != nil { @@ -1857,7 +1810,7 @@ func (dwr DynamoDB) SetUserData(ctx context.Context, userData mTypes.UserData) e ":UserData": userAttributeValue, }, Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: userid, }, }, @@ -1883,7 +1836,7 @@ func (dwr DynamoDB) DeleteUserData(ctx context.Context) error { _, err = dwr.Client.DeleteItem(ctx, &dynamodb.DeleteItemInput{ TableName: aws.String(dwr.UserDataTablename), Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{Value: userid}, + "Key": &types.AttributeValueMemberS{Value: userid}, }, }) @@ -1949,68 +1902,27 @@ func (dwr *DynamoDB) PatchDB() error { return nil } -func (dwr *DynamoDB) createRepoMetaTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.RepoMetaTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("RepoName"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("RepoName"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - - if err != nil && !strings.Contains(err.Error(), "Table already exists") { - return err - } - - return dwr.waitTableToBeCreated(dwr.RepoMetaTablename) -} - -func (dwr *DynamoDB) createProtoRepoMetaTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.RepoMetaTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("RepoName"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("RepoName"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - - if err != nil && !strings.Contains(err.Error(), "Table already exists") { +func (dwr *DynamoDB) ResetTable(tableName string) error { + err := dwr.deleteTable(tableName) + if err != nil { return err } - return dwr.waitTableToBeCreated(dwr.RepoMetaTablename) + return dwr.createTable(tableName) } -func (dwr *DynamoDB) createProtoRepoBlobsTable() error { +func (dwr *DynamoDB) createTable(tableName string) error { _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.RepoBlobsTablename), + TableName: aws.String(tableName), AttributeDefinitions: []types.AttributeDefinition{ { - AttributeName: aws.String("RepoName"), + AttributeName: aws.String("Key"), AttributeType: types.ScalarAttributeTypeS, }, }, KeySchema: []types.KeySchemaElement{ { - AttributeName: aws.String("RepoName"), + AttributeName: aws.String("Key"), KeyType: types.KeyTypeHash, }, }, @@ -2021,125 +1933,19 @@ func (dwr *DynamoDB) createProtoRepoBlobsTable() error { return err } - return dwr.waitTableToBeCreated(dwr.RepoMetaTablename) + return dwr.waitTableToBeCreated(tableName) } -func (dwr *DynamoDB) deleteRepoMetaTable() error { +func (dwr *DynamoDB) deleteTable(tableName string) error { _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ - TableName: aws.String(dwr.RepoMetaTablename), + TableName: aws.String(tableName), }) if temp := new(types.ResourceNotFoundException); errors.As(err, &temp) { return nil } - return dwr.waitTableToBeDeleted(dwr.RepoMetaTablename) -} - -func (dwr *DynamoDB) deleteProtoRepoMetaTable() error { - _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ - TableName: aws.String(dwr.RepoMetaTablename), - }) - - if temp := new(types.ResourceNotFoundException); errors.As(err, &temp) { - return nil - } - - return dwr.waitTableToBeDeleted(dwr.RepoMetaTablename) -} - -func (dwr *DynamoDB) deleteProtoRepoBlobsTable() error { - _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ - TableName: aws.String(dwr.RepoBlobsTablename), - }) - - if temp := new(types.ResourceNotFoundException); errors.As(err, &temp) { - return nil - } - - return dwr.waitTableToBeDeleted(dwr.RepoBlobsTablename) -} - -func (dwr *DynamoDB) deleteProtoImageDataTable() error { - _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ - TableName: aws.String(dwr.ImageDataTablename), - }) - - if temp := new(types.ResourceNotFoundException); errors.As(err, &temp) { - return nil - } - - return dwr.waitTableToBeDeleted(dwr.ImageDataTablename) -} - -func (dwr *DynamoDB) ResetRepoMetaTable() error { - err := dwr.deleteRepoMetaTable() - if err != nil { - return err - } - - return dwr.createRepoMetaTable() -} - -func (dwr *DynamoDB) ResetProtoRepoMetaTable() error { - err := dwr.deleteProtoRepoMetaTable() - if err != nil { - return err - } - - return dwr.createProtoRepoMetaTable() -} - -func (dwr *DynamoDB) ResetProtoImageDataTable() error { - err := dwr.deleteProtoImageDataTable() - if err != nil { - return err - } - - return dwr.createProtoImageDataTable() -} - -func (dwr *DynamoDB) ResetProtoRepoBlobsTable() error { - err := dwr.deleteProtoRepoBlobsTable() - if err != nil { - return err - } - - return dwr.createProtoRepoBlobsTable() -} - -func (dwr *DynamoDB) ResetRepoRepoMetaTable() error { - err := dwr.deleteProtoRepoMetaTable() - if err != nil { - return err - } - - return dwr.createRepoMetaTable() -} - -func (dwr *DynamoDB) createProtoImageDataTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.ImageDataTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("Digest"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("Digest"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - - if err != nil && !strings.Contains(err.Error(), "Table already exists") { - return err - } - - return dwr.waitTableToBeCreated(dwr.ImageDataTablename) + return dwr.waitTableToBeDeleted(tableName) } func (dwr *DynamoDB) waitTableToBeCreated(tableName string) error { @@ -2167,13 +1973,13 @@ func (dwr *DynamoDB) createVersionTable() error { TableName: aws.String(dwr.VersionTablename), AttributeDefinitions: []types.AttributeDefinition{ { - AttributeName: aws.String("VersionKey"), + AttributeName: aws.String("Key"), AttributeType: types.ScalarAttributeTypeS, }, }, KeySchema: []types.KeySchemaElement{ { - AttributeName: aws.String("VersionKey"), + AttributeName: aws.String("Key"), KeyType: types.KeyTypeHash, }, }, @@ -2206,7 +2012,7 @@ func (dwr *DynamoDB) createVersionTable() error { ":Version": mdAttributeValue, }, Key: map[string]types.AttributeValue{ - "VersionKey": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: version.DBVersionKey, }, }, @@ -2226,7 +2032,7 @@ func (dwr *DynamoDB) getDBVersion() (string, error) { resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.VersionTablename), Key: map[string]types.AttributeValue{ - "VersionKey": &types.AttributeValueMemberS{Value: version.DBVersionKey}, + "Key": &types.AttributeValueMemberS{Value: version.DBVersionKey}, }, }) if err != nil { diff --git a/pkg/meta/dynamodb/dynamodb_internal_test.go b/pkg/meta/dynamodb/dynamodb_internal_test.go index 435bb23771..42157aab9a 100644 --- a/pkg/meta/dynamodb/dynamodb_internal_test.go +++ b/pkg/meta/dynamodb/dynamodb_internal_test.go @@ -62,13 +62,13 @@ func TestWrapperErrors(t *testing.T) { } // The table creation should fail as the endpoint is not configured correctly - err = dynamoWrapper.createRepoMetaTable() + err = dynamoWrapper.createTable(dynamoWrapper.RepoMetaTablename) So(err, ShouldNotBeNil) err = dynamoWrapper.createVersionTable() So(err, ShouldNotBeNil) - err = dynamoWrapper.createAPIKeyTable() + err = dynamoWrapper.createTable(dynamoWrapper.APIKeyTablename) So(err, ShouldNotBeNil) }) @@ -97,7 +97,7 @@ func TestWrapperErrors(t *testing.T) { } // The tables were not created so delete calls fail, but dynamoWrapper should not error - err = dynamoWrapper.deleteRepoMetaTable() + err = dynamoWrapper.deleteTable(dynamoWrapper.RepoMetaTablename) So(err, ShouldBeNil) }) } diff --git a/pkg/meta/dynamodb/dynamodb_test.go b/pkg/meta/dynamodb/dynamodb_test.go index 7a43b627bc..4dd4db4634 100644 --- a/pkg/meta/dynamodb/dynamodb_test.go +++ b/pkg/meta/dynamodb/dynamodb_test.go @@ -62,8 +62,8 @@ func TestIterator(t *testing.T) { dynamoWrapper, err := mdynamodb.New(client, params, log) So(err, ShouldBeNil) - So(dynamoWrapper.ResetProtoImageDataTable(), ShouldBeNil) - So(dynamoWrapper.ResetProtoRepoMetaTable(), ShouldBeNil) + So(dynamoWrapper.ResetTable(dynamoWrapper.ImageDataTablename), ShouldBeNil) + So(dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename), ShouldBeNil) err = dynamoWrapper.SetRepoReference("repo1", "tag1", CreateRandomImage().AsImageData()) So(err, ShouldBeNil) @@ -171,7 +171,7 @@ func TestWrapperErrors(t *testing.T) { dynamoWrapper.SetImageTrustStore(imgTrustStore) - So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) //nolint:contextcheck + So(dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename), ShouldBeNil) //nolint:contextcheck userAc := reqCtx.NewUserAccessControl() userAc.SetUsername("test") @@ -496,7 +496,7 @@ func TestWrapperErrors(t *testing.T) { Convey("ResetRepoMetaTable client errors", func() { dynamoWrapper.RepoMetaTablename = badTablename - err := dynamoWrapper.ResetRepoMetaTable() + err := dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename) So(err, ShouldNotBeNil) }) @@ -589,7 +589,7 @@ func setBadUserData(client *dynamodb.Client, userDataTablename, userID string) e ":UserData": userAttributeValue, }, Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: userID, }, }, @@ -614,7 +614,7 @@ func setVersion(client *dynamodb.Client, versionTablename string, version string ":Version": mdAttributeValue, }, Key: map[string]types.AttributeValue{ - "VersionKey": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: "DBVersion", }, }, diff --git a/pkg/meta/hooks.go b/pkg/meta/hooks.go index 3a6b66e887..74d6d9380c 100644 --- a/pkg/meta/hooks.go +++ b/pkg/meta/hooks.go @@ -22,7 +22,7 @@ func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, imgStore := storeController.GetImageStore(repo) - err := ProtoSetImageMetaFromInput(repo, reference, mediaType, digest, body, + err := SetImageMetaFromInput(repo, reference, mediaType, digest, body, imgStore, metaDB, log) if err != nil { log.Info().Str("tag", reference).Str("repository", repo).Msg("uploading image meta was unsuccessful for tag in repo") diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index 3f2f0466ac..a5a9a76268 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -132,22 +132,17 @@ func TestDynamoDBWrapper(t *testing.T) { dynamoDriver.SetImageTrustStore(imgTrustStore) resetDynamoDBTables := func() error { - err := dynamoDriver.ResetRepoMetaTable() + err := dynamoDriver.ResetTable(dynamoDriver.RepoMetaTablename) if err != nil { return err } - err = dynamoDriver.ResetProtoRepoMetaTable() + err = dynamoDriver.ResetTable(dynamoDriver.ImageDataTablename) if err != nil { return err } - err = dynamoDriver.ResetProtoImageDataTable() - if err != nil { - return err - } - - err = dynamoDriver.ResetProtoRepoBlobsTable() + err = dynamoDriver.ResetTable(dynamoDriver.RepoBlobsTablename) if err != nil { return err } @@ -733,7 +728,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference(repo2, tag2, imageData2) So(err, ShouldBeNil) - Convey("Get all Repometa", func() { + Convey("Get all RepoMeta", func() { repoMetaSlice, err := metaDB.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool { return true }) @@ -835,22 +830,21 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - - stars, err := metaDB.GetRepoStars(repo1) + repoMeta, err := metaDB.GetRepoMeta(repo1) So(err, ShouldBeNil) - So(stars, ShouldEqual, 1) + So(repoMeta.Stars, ShouldEqual, 1) err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - err = metaDB.IncrementRepoStars(repo1) + repoMeta, err = metaDB.GetRepoMeta(repo1) So(err, ShouldBeNil) + So(repoMeta.Stars, ShouldEqual, 2) - stars, err = metaDB.GetRepoStars(repo1) + err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - So(stars, ShouldEqual, 3) - - _, err = metaDB.GetRepoStars("badRepo") - So(err, ShouldNotBeNil) + repoMeta, err = metaDB.GetRepoMeta(repo1) + So(err, ShouldBeNil) + So(repoMeta.Stars, ShouldEqual, 3) }) Convey("Test repo stars for user", func() { @@ -895,14 +889,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.SetRepoReference(repo2, tag1, CreateDefaultImage().AsImageData()) So(err, ShouldBeNil) - starCount, err := metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 0) - - starCount, err = metaDB.GetRepoStars(repo2) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 0) - repos, err := metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) @@ -924,10 +910,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(repoMeta.Stars, ShouldEqual, 1) - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) - repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -950,10 +932,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(repoMeta.Stars, ShouldEqual, 2) - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 2) - repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -977,10 +955,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(repoMeta.Stars, ShouldEqual, 1) - starCount, err = metaDB.GetRepoStars(repo2) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) - repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 2) @@ -1005,10 +979,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(repoMeta.Stars, ShouldEqual, 1) - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) - repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) @@ -1040,14 +1010,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(repoMeta.Stars, ShouldEqual, 1) - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) - - starCount, err = metaDB.GetRepoStars(repo2) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) - repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 2) @@ -1062,15 +1024,11 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) - // Anonyous user attempts to toggle a star + // Anonymous user attempts to toggle a star toggleState, err = metaDB.ToggleStarRepo(ctx3, repo1) So(err, ShouldNotBeNil) So(toggleState, ShouldEqual, mTypes.NotChanged) - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) - repos, err = metaDB.GetStarredRepos(ctx3) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) @@ -1080,10 +1038,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(toggleState, ShouldEqual, mTypes.Removed) - starCount, err = metaDB.GetRepoStars(repo2) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 0) - repos, err = metaDB.GetStarredRepos(ctx3) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) @@ -1269,7 +1223,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 2) - // _, err = metaDB.GetManifestMeta(repo1, "badManiestDigest") + // _, err = metaDB.GetManifestMeta(repo1, "badManifestDigest") // So(err, ShouldNotBeNil) }) @@ -1802,7 +1756,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // So(len(repos), ShouldEqual, 0) // }) - // Convey("With no permision", func() { + // Convey("With no permission", func() { // userAc := reqCtx.NewUserAccessControl() // userAc.SetUsername("user1") // userAc.SetGlobPatterns("read", @@ -2358,7 +2312,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // referredDigest := godigest.FromString("referredDigest") // err := metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - // Digest: "inexistendManifestDigest", + // Digest: "inexistentManifestDigest", // MediaType: ispec.MediaTypeImageManifest, // }) // So(err, ShouldBeNil) diff --git a/pkg/meta/parse.go b/pkg/meta/parse.go index c98a04e55c..a1ab4688d1 100644 --- a/pkg/meta/parse.go +++ b/pkg/meta/parse.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "regexp" "time" godigest "github.com/opencontainers/go-digest" @@ -19,6 +18,11 @@ import ( storageTypes "zotregistry.io/zot/pkg/storage/types" ) +const ( + CosignType = "cosign" + NotationType = "notation" +) + // ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the // ParseStorage database. func ParseStorage(metaDB mTypes.MetaDB, storeController storage.StoreController, log log.Logger) error { @@ -75,23 +79,23 @@ func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController storage.StoreC return err } - err = metaDB.ResetRepoRefferences(repo) + err = metaDB.ResetRepoReferences(repo) if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to reset tag field in RepoMetadata for repo") return err } - for _, descriptor := range indexContent.Manifests { - tag := descriptor.Annotations[ispec.AnnotationRefName] + for _, manifest := range indexContent.Manifests { + tag := manifest.Annotations[ispec.AnnotationRefName] if zcommon.IsReferrersTag(tag) { continue } - descriptorBlob, _, _, err := imageStore.GetImageManifest(repo, descriptor.Digest.String()) + manifestBlob, _, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String()) if err != nil { - log.Error().Err(err).Str("repository", repo).Str("digest", descriptor.Digest.String()). + log.Error().Err(err).Str("repository", repo).Str("digest", manifest.Digest.String()). Msg("load-repo: failed to get blob for image") return err @@ -100,10 +104,10 @@ func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController storage.StoreC reference := tag if tag == "" { - reference = descriptor.Digest.String() + reference = manifest.Digest.String() } - err = ProtoSetImageMetaFromInput(repo, reference, descriptor.MediaType, descriptor.Digest, descriptorBlob, + err = SetImageMetaFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob, imageStore, metaDB, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("tag", tag). @@ -243,55 +247,9 @@ func getNotationSignatureLayersInfo( return layers, nil } -// NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object. -func NewManifestData(repoName string, manifestBlob []byte, imageStore storageTypes.ImageStore, -) (mTypes.DepManifestData, error) { - var ( - manifestContent ispec.Manifest - configContent ispec.Image - manifestData mTypes.DepManifestData - ) - - err := json.Unmarshal(manifestBlob, &manifestContent) - if err != nil { - return mTypes.DepManifestData{}, err - } - - var lockLatency time.Time - - imageStore.RLock(&lockLatency) - defer imageStore.RUnlock(&lockLatency) - - configBlob, err := imageStore.GetBlobContent(repoName, manifestContent.Config.Digest) - if err != nil { - return mTypes.DepManifestData{}, err - } - - if manifestContent.Config.MediaType == ispec.MediaTypeImageConfig { - err = json.Unmarshal(configBlob, &configContent) - if err != nil { - return mTypes.DepManifestData{}, err - } - } - - manifestData.ManifestBlob = manifestBlob - manifestData.ConfigBlob = configBlob - - return manifestData, nil -} - -func NewIndexData(repoName string, indexBlob []byte, imageStore storageTypes.ImageStore, -) mTypes.DepIndexData { - indexData := mTypes.DepIndexData{} - - indexData.IndexBlob = indexBlob - - return indexData -} - // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). -func ProtoSetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte, +func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, blob []byte, imageStore storageTypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger, ) error { var imageData mTypes.ImageData @@ -301,7 +259,7 @@ func ProtoSetImageMetaFromInput(repo, reference, mediaType string, digest godige manifestContent := ispec.Manifest{} configContent := ispec.Image{} - err := json.Unmarshal(descriptorBlob, &manifestContent) + err := json.Unmarshal(blob, &manifestContent) if err != nil { return err } @@ -318,9 +276,9 @@ func ProtoSetImageMetaFromInput(repo, reference, mediaType string, digest godige } } - if isSig, sigType, signedManifestDigest := isSingature(reference, manifestContent); isSig { + if isSig, sigType, signedManifestDigest := isSignature(reference, manifestContent); isSig { layers, err := GetSignatureLayersInfo(repo, reference, digest.String(), sigType, - descriptorBlob, imageStore, log) + blob, imageStore, log) if err != nil { return err } @@ -350,16 +308,16 @@ func ProtoSetImageMetaFromInput(repo, reference, mediaType string, digest godige return nil } - imageData = convert.GetImageManifestData(manifestContent, configContent, int64(len(descriptorBlob)), digest) + imageData = convert.GetImageManifestData(manifestContent, configContent, int64(len(blob)), digest) case ispec.MediaTypeImageIndex: indexContent := ispec.Index{} - err := json.Unmarshal(descriptorBlob, &indexContent) + err := json.Unmarshal(blob, &indexContent) if err != nil { return err } - imageData = convert.GetImageIndexData(indexContent, int64(len(descriptorBlob)), digest) + imageData = convert.GetImageIndexData(indexContent, int64(len(blob)), digest) default: return nil } @@ -374,12 +332,7 @@ func ProtoSetImageMetaFromInput(repo, reference, mediaType string, digest godige return nil } -const ( - CosignType = "cosign" - NotationType = "notation" -) - -func isSingature(reference string, manifestContent ispec.Manifest) (bool, string, godigest.Digest) { +func isSignature(reference string, manifestContent ispec.Manifest) (bool, string, godigest.Digest) { manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent) // check notation signature @@ -387,10 +340,7 @@ func isSingature(reference string, manifestContent ispec.Manifest) (bool, string return true, NotationType, manifestContent.Subject.Digest } - // check cosign - cosignTagRule := regexp.MustCompile(`sha256\-.+\.sig`) - - if tag := reference; cosignTagRule.MatchString(reference) { + if tag := reference; zcommon.IsCosignTag(reference) { prefixLen := len("sha256-") digestLen := 64 signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index a2fef26e74..8521e1b170 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -404,16 +404,13 @@ func TestParseStorageDynamoWrapper(t *testing.T) { dynamoWrapper, err := dynamodb.New(dynamoClient, params, log.NewLogger("debug", "")) So(err, ShouldBeNil) - err = dynamoWrapper.ResetRepoMetaTable() + err = dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename) So(err, ShouldBeNil) - err = dynamoWrapper.ResetProtoRepoMetaTable() + err = dynamoWrapper.ResetTable(dynamoWrapper.RepoBlobsTablename) So(err, ShouldBeNil) - err = dynamoWrapper.ResetProtoImageDataTable() - So(err, ShouldBeNil) - - err = dynamoWrapper.ResetProtoRepoBlobsTable() + err = dynamoWrapper.ResetTable(dynamoWrapper.ImageDataTablename) So(err, ShouldBeNil) RunParseStorageTests(rootDir, dynamoWrapper) diff --git a/pkg/meta/proto/meta.proto b/pkg/meta/proto/meta.proto index ae5ae75120..97f313fc4c 100644 --- a/pkg/meta/proto/meta.proto +++ b/pkg/meta/proto/meta.proto @@ -12,7 +12,7 @@ message TagDescriptor { message ImageData { Versioned Versioned = 1; string MediaType = 2; - string ArtifacType = 3; + string ArtifactType = 3; repeated ManifestData Manifests = 4; optional Descriptor Subject = 5; map Annotations = 6; @@ -45,17 +45,17 @@ message RepoLastUpdatedImage { message RepoMeta { string Name = 1; - map Tags = 2; + map Tags = 2; - map Statistics = 3; - map Signatures = 4; - map Referrers = 5; + map Statistics = 3; + map Signatures = 4; + map Referrers = 5; - bool IsStarred = 6; - bool IsBookmarked = 7; - int32 Rank = 8; + bool IsStarred = 6; + bool IsBookmarked = 7; + int32 Rank = 8; - int32 Stars = 9; + int32 Stars = 9; int32 Size = 10; repeated string Vendors = 11; @@ -89,11 +89,11 @@ message ReferrersInfo { message ReferrerInfo { string Digest = 1; - string MediaType = 2; - string ArtifactType = 3; - int64 Size = 4; + string MediaType = 2; + string ArtifactType = 3; + int64 Size = 4; - map Annotations = 5; + map Annotations = 5; } message ManifestSignatures { @@ -110,10 +110,10 @@ message SignatureInfo { } message LayersInfo { - string LayerDigest = 1; - bytes LayerContent = 2; - string SignatureKey = 3; - string Signer = 4; + string LayerDigest = 1; + bytes LayerContent = 2; + string SignatureKey = 3; + string Signer = 4; - google.protobuf.Timestamp Date = 5; + google.protobuf.Timestamp Date = 5; } \ No newline at end of file diff --git a/pkg/meta/proto_go/meta.pb.go b/pkg/meta/proto_go/meta.pb.go index e2c81fccc0..577172b20c 100644 --- a/pkg/meta/proto_go/meta.pb.go +++ b/pkg/meta/proto_go/meta.pb.go @@ -81,16 +81,16 @@ type ImageData struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Versioned *Versioned `protobuf:"bytes,1,opt,name=Versioned,proto3" json:"Versioned,omitempty"` - MediaType string `protobuf:"bytes,2,opt,name=MediaType,proto3" json:"MediaType,omitempty"` - ArtifacType string `protobuf:"bytes,3,opt,name=ArtifacType,proto3" json:"ArtifacType,omitempty"` - Manifests []*ManifestData `protobuf:"bytes,4,rep,name=Manifests,proto3" json:"Manifests,omitempty"` - Subject *Descriptor `protobuf:"bytes,5,opt,name=Subject,proto3,oneof" json:"Subject,omitempty"` - Annotations map[string]string `protobuf:"bytes,6,rep,name=Annotations,proto3" json:"Annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - Size int64 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"` - Digest string `protobuf:"bytes,8,opt,name=Digest,proto3" json:"Digest,omitempty"` - Repo *string `protobuf:"bytes,9,opt,name=Repo,proto3,oneof" json:"Repo,omitempty"` - Tag *string `protobuf:"bytes,10,opt,name=Tag,proto3,oneof" json:"Tag,omitempty"` + Versioned *Versioned `protobuf:"bytes,1,opt,name=Versioned,proto3" json:"Versioned,omitempty"` + MediaType string `protobuf:"bytes,2,opt,name=MediaType,proto3" json:"MediaType,omitempty"` + ArtifactType string `protobuf:"bytes,3,opt,name=ArtifactType,proto3" json:"ArtifactType,omitempty"` + Manifests []*ManifestData `protobuf:"bytes,4,rep,name=Manifests,proto3" json:"Manifests,omitempty"` + Subject *Descriptor `protobuf:"bytes,5,opt,name=Subject,proto3,oneof" json:"Subject,omitempty"` + Annotations map[string]string `protobuf:"bytes,6,rep,name=Annotations,proto3" json:"Annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Size int64 `protobuf:"varint,7,opt,name=Size,proto3" json:"Size,omitempty"` + Digest string `protobuf:"bytes,8,opt,name=Digest,proto3" json:"Digest,omitempty"` + Repo *string `protobuf:"bytes,9,opt,name=Repo,proto3,oneof" json:"Repo,omitempty"` + Tag *string `protobuf:"bytes,10,opt,name=Tag,proto3,oneof" json:"Tag,omitempty"` } func (x *ImageData) Reset() { @@ -139,9 +139,9 @@ func (x *ImageData) GetMediaType() string { return "" } -func (x *ImageData) GetArtifacType() string { +func (x *ImageData) GetArtifactType() string { if x != nil { - return x.ArtifacType + return x.ArtifactType } return "" } @@ -1067,213 +1067,213 @@ var file_meta_proto_rawDesc = []byte{ 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, - 0x65, 0x73, 0x74, 0x22, 0xd2, 0x03, 0x0a, 0x09, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x44, 0x61, 0x74, + 0x65, 0x73, 0x74, 0x22, 0xd4, 0x03, 0x0a, 0x09, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2b, 0x0a, 0x09, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x09, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x20, 0x0a, 0x0b, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0b, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x2e, - 0x0a, 0x09, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x44, - 0x61, 0x74, 0x61, 0x52, 0x09, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x12, 0x2d, - 0x0a, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0e, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x48, - 0x00, 0x52, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, 0x12, 0x40, 0x0a, - 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x44, 0x61, 0x74, - 0x61, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, - 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, - 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x08, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x04, 0x52, - 0x65, 0x70, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x04, 0x52, 0x65, 0x70, - 0x6f, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x54, 0x61, 0x67, 0x18, 0x0a, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x02, 0x52, 0x03, 0x54, 0x61, 0x67, 0x88, 0x01, 0x01, 0x1a, 0x3e, 0x0a, 0x10, 0x41, - 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0a, 0x0a, 0x08, 0x5f, - 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x52, 0x65, 0x70, 0x6f, - 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x54, 0x61, 0x67, 0x22, 0xe2, 0x03, 0x0a, 0x0c, 0x4d, 0x61, 0x6e, - 0x69, 0x66, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2b, 0x0a, 0x09, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x76, - 0x31, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x09, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x21, - 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x09, 0x48, 0x00, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, - 0x01, 0x12, 0x27, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x26, 0x0a, 0x06, 0x4c, 0x61, - 0x79, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x76, 0x31, 0x2e, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x06, 0x4c, 0x61, 0x79, 0x65, - 0x72, 0x73, 0x12, 0x2d, 0x0a, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x6f, 0x72, 0x48, 0x02, 0x52, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x88, 0x01, - 0x01, 0x12, 0x43, 0x0a, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, - 0x66, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x26, 0x0a, 0x06, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x76, 0x31, 0x2e, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x43, 0x6f, 0x6e, 0x66, - 0x69, 0x67, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, - 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, 0xb1, 0x01, - 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x41, 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1c, 0x0a, 0x09, 0x4d, 0x65, 0x64, - 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4d, 0x65, - 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, - 0x10, 0x0a, 0x03, 0x54, 0x61, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x54, 0x61, - 0x67, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x64, 0x22, 0xf4, 0x06, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x12, - 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x2e, 0x54, - 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x54, 0x61, 0x67, 0x73, 0x12, 0x3c, - 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x18, 0x03, 0x20, 0x03, + 0x09, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x22, 0x0a, 0x0c, + 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x2e, 0x0a, 0x09, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, + 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x09, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, + 0x12, 0x2d, 0x0a, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x0e, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x48, 0x00, 0x52, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, 0x12, + 0x40, 0x0a, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x76, 0x31, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x44, + 0x61, 0x74, 0x61, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, + 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, + 0x04, 0x52, 0x65, 0x70, 0x6f, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x04, 0x52, + 0x65, 0x70, 0x6f, 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x54, 0x61, 0x67, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x03, 0x54, 0x61, 0x67, 0x88, 0x01, 0x01, 0x1a, 0x3e, 0x0a, + 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0a, 0x0a, + 0x08, 0x5f, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x42, 0x07, 0x0a, 0x05, 0x5f, 0x52, 0x65, + 0x70, 0x6f, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x54, 0x61, 0x67, 0x22, 0xe2, 0x03, 0x0a, 0x0c, 0x4d, + 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12, 0x2b, 0x0a, 0x09, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, + 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x09, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, + 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, + 0x12, 0x21, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x88, 0x01, 0x01, 0x12, 0x27, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x0c, 0x41, 0x72, 0x74, + 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x26, 0x0a, 0x06, + 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x76, + 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x06, 0x4c, 0x61, + 0x79, 0x65, 0x72, 0x73, 0x12, 0x2d, 0x0a, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x48, 0x02, 0x52, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x88, 0x01, 0x01, 0x12, 0x43, 0x0a, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, + 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x44, 0x61, 0x74, 0x61, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, + 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, + 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x26, 0x0a, 0x06, + 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x76, + 0x31, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x44, 0x61, 0x74, 0x61, 0x52, 0x06, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x22, + 0xb1, 0x01, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x41, 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0b, 0x4c, 0x61, 0x73, + 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1c, 0x0a, 0x09, 0x4d, + 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, + 0x65, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x54, 0x61, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x54, 0x61, 0x67, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x64, 0x22, 0xf4, 0x06, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, + 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, + 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x54, 0x61, 0x67, 0x73, + 0x12, 0x3c, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x18, 0x03, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, + 0x74, 0x61, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x3c, + 0x0a, 0x0a, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, - 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x3c, 0x0a, 0x0a, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x2e, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, - 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x09, 0x52, 0x65, - 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1b, 0x2e, - 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x2e, 0x52, 0x65, 0x66, 0x65, - 0x72, 0x72, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x52, 0x65, 0x66, 0x65, - 0x72, 0x72, 0x65, 0x72, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x49, 0x73, 0x53, 0x74, 0x61, 0x72, 0x72, - 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x49, 0x73, 0x53, 0x74, 0x61, 0x72, - 0x72, 0x65, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x49, 0x73, 0x42, 0x6f, 0x6f, 0x6b, 0x6d, 0x61, 0x72, - 0x6b, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x49, 0x73, 0x42, 0x6f, 0x6f, - 0x6b, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x52, 0x61, 0x6e, 0x6b, 0x18, - 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x52, 0x61, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x53, - 0x74, 0x61, 0x72, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x53, 0x74, 0x61, 0x72, - 0x73, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x0a, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x09, + 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x1b, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x2e, 0x52, 0x65, + 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x52, 0x65, + 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x49, 0x73, 0x53, 0x74, 0x61, + 0x72, 0x72, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x49, 0x73, 0x53, 0x74, + 0x61, 0x72, 0x72, 0x65, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x49, 0x73, 0x42, 0x6f, 0x6f, 0x6b, 0x6d, + 0x61, 0x72, 0x6b, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x49, 0x73, 0x42, + 0x6f, 0x6f, 0x6b, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x52, 0x61, 0x6e, + 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x52, 0x61, 0x6e, 0x6b, 0x12, 0x14, 0x0a, + 0x05, 0x53, 0x74, 0x61, 0x72, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x53, 0x74, + 0x61, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, + 0x72, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, 0x72, + 0x73, 0x12, 0x2a, 0x0a, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x0c, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, + 0x72, 0x6d, 0x52, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x49, 0x0a, + 0x10, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, + 0x6f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x48, 0x00, 0x52, 0x10, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x49, 0x6d, 0x61, 0x67, 0x65, 0x88, 0x01, 0x01, 0x1a, 0x4a, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x57, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, + 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, + 0x63, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x55, 0x0a, + 0x0f, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x4f, 0x0a, 0x0e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x22, 0x97, 0x01, 0x0a, 0x09, 0x52, + 0x65, 0x70, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x05, + 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x76, 0x31, + 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x1a, 0x46, 0x0a, 0x0a, + 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x22, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x31, + 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x22, 0xd3, 0x01, 0x0a, 0x08, 0x42, 0x6c, 0x6f, 0x62, 0x49, 0x6e, 0x66, + 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x73, - 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x73, 0x12, - 0x2a, 0x0a, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x0c, 0x20, 0x03, + 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x73, 0x12, + 0x2a, 0x0a, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, - 0x52, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x49, 0x0a, 0x10, 0x4c, - 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x18, - 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4c, - 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x48, - 0x00, 0x52, 0x10, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, - 0x61, 0x67, 0x65, 0x88, 0x01, 0x01, 0x1a, 0x4a, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x57, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, - 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x55, 0x0a, 0x0f, 0x53, - 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x16, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x1a, 0x4f, 0x0a, 0x0e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x72, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, - 0x02, 0x38, 0x01, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x22, 0x97, 0x01, 0x0a, 0x09, 0x52, 0x65, 0x70, - 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2e, 0x0a, 0x05, 0x42, 0x6c, - 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x70, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x05, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x1a, 0x46, 0x0a, 0x0a, 0x42, 0x6c, - 0x6f, 0x62, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x22, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x76, 0x31, 0x2e, 0x42, - 0x6c, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0xd3, 0x01, 0x0a, 0x08, 0x42, 0x6c, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x12, - 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, - 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x73, 0x18, 0x02, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x73, 0x12, 0x2a, 0x0a, - 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x0c, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x09, - 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x53, 0x75, 0x62, - 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x53, 0x75, 0x62, - 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, - 0x64, 0x61, 0x74, 0x65, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x4c, 0x61, 0x73, - 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x22, 0x3c, 0x0a, 0x14, 0x44, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, - 0x12, 0x24, 0x0a, 0x0d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, - 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x35, 0x0a, 0x0d, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, - 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x24, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, - 0x72, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, 0x81, 0x02, - 0x0a, 0x0c, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, - 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, - 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, - 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x41, 0x72, 0x74, 0x69, - 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x43, 0x0a, 0x0b, - 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x21, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x49, - 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x22, 0x93, 0x01, 0x0a, 0x12, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x31, 0x0a, 0x03, 0x6d, 0x61, 0x70, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, - 0x65, 0x73, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x2e, 0x4d, 0x61, - 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x1a, 0x4a, 0x0a, 0x08, 0x4d, - 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, - 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x37, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x61, - 0x74, 0x75, 0x72, 0x65, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x25, 0x0a, 0x04, 0x6c, 0x69, 0x73, - 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, - 0x22, 0x79, 0x0a, 0x0d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x49, 0x6e, 0x66, - 0x6f, 0x12, 0x38, 0x0a, 0x17, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4d, 0x61, - 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x17, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4d, 0x61, 0x6e, - 0x69, 0x66, 0x65, 0x73, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x0a, 0x4c, - 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x0e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, - 0x0a, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xbe, 0x01, 0x0a, 0x0a, - 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x20, 0x0a, 0x0b, 0x4c, 0x61, - 0x79, 0x65, 0x72, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, - 0x4c, 0x61, 0x79, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0c, 0x52, 0x0c, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, - 0x12, 0x22, 0x0a, 0x0c, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4b, 0x65, 0x79, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, - 0x65, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x12, 0x2e, 0x0a, 0x04, - 0x44, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x44, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x53, + 0x75, 0x62, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x53, + 0x75, 0x62, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0b, 0x4c, 0x61, 0x73, 0x74, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x4c, + 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x22, 0x3c, 0x0a, 0x14, 0x44, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, + 0x63, 0x73, 0x12, 0x24, 0x0a, 0x0d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, + 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x35, 0x0a, 0x0d, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x72, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x24, 0x0a, 0x04, 0x6c, 0x69, 0x73, + 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x72, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, + 0x81, 0x02, 0x0a, 0x0c, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, + 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4d, 0x65, 0x64, + 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x41, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, + 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x43, + 0x0a, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x05, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, + 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, + 0x02, 0x38, 0x01, 0x22, 0x93, 0x01, 0x0a, 0x12, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x31, 0x0a, 0x03, 0x6d, 0x61, + 0x70, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, + 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x2e, + 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x1a, 0x4a, 0x0a, + 0x08, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x76, 0x31, 0x2e, + 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x37, 0x0a, 0x0e, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x25, 0x0a, 0x04, 0x6c, + 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x76, 0x31, 0x2e, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x6c, 0x69, + 0x73, 0x74, 0x22, 0x79, 0x0a, 0x0d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x49, + 0x6e, 0x66, 0x6f, 0x12, 0x38, 0x0a, 0x17, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x17, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4d, + 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, + 0x0a, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x0e, 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, + 0x6f, 0x52, 0x0a, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xbe, 0x01, + 0x0a, 0x0a, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x20, 0x0a, 0x0b, + 0x4c, 0x61, 0x79, 0x65, 0x72, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0b, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x22, + 0x0a, 0x0c, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4b, + 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x12, 0x2e, + 0x0a, 0x04, 0x44, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x44, 0x61, 0x74, 0x65, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index 1bc69e93be..d10ffe2020 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -20,17 +20,19 @@ const ( type ( DepFilterFunc func(repoMeta DepRepoMetadata, manifestMeta DepManifestMetadata) bool + // Currently imageData applied for indexes is applied for each manifest individually so imageData.manifests + // contains just 1 manifest. FilterFunc func(repoMeta RepoMetadata, imageData ImageData) bool - FilterRepoNameFunc func(repo string) int + FilterRepoNameFunc func(repo string) bool FilterFullRepoFunc func(repoMeta FullRepoMetadata) bool FilterRepoTagFunc func(repo, tag string) bool ) -func AcceptAllRepoNames(repo string) int { - return 1 +func AcceptAllRepoNames(repo string) bool { + return true } -func AcceptAllRepoMeta(repometa FullRepoMetadata) bool { +func AcceptAllRepoMeta(repoMeta FullRepoMetadata) bool { return true } @@ -51,56 +53,88 @@ type MetaDB interface { //nolint:interfacebloat SetImageData(digest godigest.Digest, imageData ImageData) error + // SetRepoReference sets the given image data to the repo metadata. SetRepoReference(repo string, reference string, imageData ImageData) error + // SearchRepos searches for repos given a search string SearchRepos(ctx context.Context, searchText string) ([]FullRepoMetadata, error) + // SearchTags searches for images(repo:tag) given a search string SearchTags(ctx context.Context, searchText string) ([]FullImageData, error) + // FilterTags filters for images given a filter function FilterTags(ctx context.Context, filterRepoTag FilterRepoTagFunc, filterFunc FilterFunc, ) ([]FullImageData, error) + // FilterRepos filters for repos given a filter function FilterRepos(ctx context.Context, rankName FilterRepoNameFunc, filterFunc FilterFullRepoFunc, ) ([]FullRepoMetadata, error) + // GetRepoMeta returns RepoMetadata of a repo from the database GetRepoMeta(repo string) (RepoMetadata, error) + // GetFullRepoMeta return the full information about a repo GetFullRepoMeta(ctx context.Context, repo string) (FullRepoMetadata, error) + // GetFullImageData return the full information about an image GetFullImageData(ctx context.Context, repo string, tag string) (FullImageData, error) + // GetImageData returns the raw information about an image GetImageData(digest godigest.Digest) (ImageData, error) + // GetMultipleRepoMeta returns information about all repositories as map[string]RepoMetadata filtered by the filter + // function GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMetadata) bool) ( []RepoMetadata, error) + // AddManifestSignature adds signature metadata to a given manifest in the database AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sm SignatureMetadata) error + // DeleteSignature deletes signature metadata to a given manifest from the database DeleteSignature(repo string, signedManifestDigest godigest.Digest, sigMeta SignatureMetadata) error + // UpdateSignaturesValidity checks and updates signatures validity of a given manifest + UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error + + // IncrementRepoStars adds 1 to the star count of an image IncrementRepoStars(repo string) error + // DecrementRepoStars subtracts 1 from the star count of an image DecrementRepoStars(repo string) error - GetRepoStars(repo string) (int, error) - + // DeleteRepoTag deletes the tag from the tag list of a repo DeleteRepoTag(repo string, tag string) error + // GetUserRepoMeta return RepoMetadata of a repo from the database along side specific information about the + // user GetUserRepoMeta(ctx context.Context, repo string) (RepoMetadata, error) + // SetRepoMeta returns RepoMetadata of a repo from the database SetRepoMeta(repo string, repoMeta RepoMetadata) error + // GetReferrersInfo returns a list of for all referrers of the given digest that match one of the + // artifact types. GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) ([]ReferrerInfo, error) + // IncrementImageDownloads adds 1 to the download count of an image IncrementImageDownloads(repo string, reference string) error - UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error - + // FilterImageData returns the image data for the given digests FilterImageData(ctx context.Context, digests []string) (map[string]ImageData, error) + /* + RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag, + + it also removes its corresponding digest from Statistics, Signatures and Referrers if there are no tags + pointing to it. + If the reference is a digest then it will remove the digest from Statistics, Signatures and Referrers only + if there are no tags pointing to the digest, otherwise it's noop + */ RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error - ResetRepoRefferences(repo string) error + // ResetRepoReferences resets all layout specific data (tags, signatures, referrers, etc.) but keep user specific + // data such as star count and other statistics + ResetRepoReferences(repo string) error PatchDB() error @@ -147,25 +181,27 @@ type UserDB interface { //nolint:interfacebloat } type ImageTrustStore interface { - VerifySignature( + DepVerifySignature( signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, blob []byte, repo string, ) (string, time.Time, bool, error) - ProtoVerifySignature( + VerifySignature( signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageData ImageData, repo string, ) (string, time.Time, bool, error) } +// ImageData can store all data related to a image, multiarch or simple. Used for writing imaged to MetaDB. type ImageData struct { MediaType string Digest godigest.Digest Size int64 - Index *ispec.Index - Manifests []ManifestData + Index *ispec.Index // If the image is multiarch the Index will be non-nil + Manifests []ManifestData // All manifests under the image, 1 for simple images and many for multiarch } +// ManifestData represents all data related to an image manifests. type ManifestData struct { Size int64 Digest godigest.Digest @@ -188,13 +224,7 @@ type RepoMetadata struct { Stars int } -type RepoStatistics struct { - Platforms []ispec.Platform - Vendors []string - Size string - LastUpdated time.Time -} - +// FullRepoMetadata is a condensed structure or all information needed about a repo when searching MetaDB. type FullRepoMetadata struct { Name string Tags map[string]Descriptor @@ -213,6 +243,7 @@ type FullRepoMetadata struct { Referrers map[string][]ReferrerInfo } +// FullImageData is a condensed structure or all information needed about an image when searching MetaDB. type FullImageData struct { Repo string Tag string diff --git a/pkg/meta/version/version_test.go b/pkg/meta/version/version_test.go index db18a60533..e1ec035154 100644 --- a/pkg/meta/version/version_test.go +++ b/pkg/meta/version/version_test.go @@ -136,7 +136,7 @@ func TestVersioningDynamoDB(t *testing.T) { dynamoWrapper, err := mdynamodb.New(dynamoClient, params, log) So(err, ShouldBeNil) - So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) + So(dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename), ShouldBeNil) Convey("dbVersion is empty", func() { err := setDynamoDBVersion(dynamoWrapper.Client, "") @@ -194,7 +194,7 @@ func setDynamoDBVersion(client *dynamodb.Client, vers string) error { ":Version": mdAttributeValue, }, Key: map[string]types.AttributeValue{ - "VersionKey": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: version.DBVersionKey, }, }, diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 56da20b74e..7d3cf4cd75 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -3,7 +3,6 @@ package storage import ( "encoding/json" "fmt" - "regexp" "strings" "github.com/docker/distribution/registry/storage/driver/factory" @@ -228,10 +227,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin return true, NotationType, manifestContent.Subject.Digest, nil } - // check cosign - cosignTagRule := regexp.MustCompile(`sha256\-.+\.sig`) - - if tag := reference; cosignTagRule.MatchString(reference) { + if tag := reference; zcommon.IsCosignTag(reference) { prefixLen := len("sha256-") digestLen := 64 signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index 1677923fc8..04a8ed7c27 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -174,7 +174,7 @@ type MetaDBMock struct { GetFullImageDataFn func(ctx context.Context, repo string, tag string) (mTypes.FullImageData, error) - ResetRepoRefferencesFn func(repo string) error + ResetRepoReferencesFn func(repo string) error } func (sdm MetaDBMock) ImageTrustStore() mTypes.ImageTrustStore { @@ -760,9 +760,9 @@ func (sdm MetaDBMock) GetFullImageData(ctx context.Context, repo string, tag str return mTypes.FullImageData{}, nil } -func (sdm MetaDBMock) ResetRepoRefferences(repo string) error { - if sdm.ResetRepoRefferencesFn != nil { - return sdm.ResetRepoRefferencesFn(repo) +func (sdm MetaDBMock) ResetRepoReferences(repo string) error { + if sdm.ResetRepoReferencesFn != nil { + return sdm.ResetRepoReferencesFn(repo) } return nil