refactor(metadb): improve UX by speeding up metadb serialize/deserialize

Use protocol buffers and update the metadb interface to better suit our search needs Signed-off-by: Ramkumar Chinchani <[email protected]> Signed-off-by: Laurentiu Niculae <[email protected]>
project-zot · Oct 25, 2023 · 37d5ea1 · 37d5ea1
1 parent a3d8202
commit 37d5ea1
Show file tree

Hide file tree

Showing 81 changed files with 9,880 additions and 11,125 deletions.
diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml
@@ -38,6 +38,7 @@ jobs:
 
           # Optional: if set to true then the action don't cache or restore ~/go/pkg.
           # skip-pkg-cache: true
+          skip-pkg-cache: false
 
           # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
           # skip-build-cache: true

diff --git a/Makefile b/Makefile
@@ -32,6 +32,22 @@ TESTDATA := $(TOP_LEVEL)/test/data
 OS ?= $(shell go env GOOS)
 ARCH ?= $(shell go env GOARCH)
 
+PROTOC := $(TOOLSDIR)/bin/protoc
+PROTOC_VERSION := 24.4
+GO_PROTOC_VERSION := 1.31.0
+HOST_OS := $(shell go env GOOS)
+HOST_ARCH := $(shell go env GOARCH)
+ifeq ($(HOST_OS),linux)
+	PROTOC_OS = linux
+else ifeq ($(HOST_OS),darwin)
+	PROTOC_OS = osx
+endif
+ifeq ($(HOST_ARCH),amd64)
+	PROTOC_ARCH = x86_64
+else ifeq ($(HOST_ARCH),arm64)
+	PROTOC_ARCH = aarch_64
+endif
+
 BENCH_OUTPUT ?= stdout
 ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs
 EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust
@@ -97,6 +113,51 @@ build-metadata: $(if $(findstring ui,$(BUILD_LABELS)), ui)
 	echo "\n Files: \n"
 	go list -tags $(BUILD_TAGS) -f '{{ join .GoFiles "\n" }}' ./... | sort -u
 
+.PHONY: gen-protobuf
+gen-protobuf: $(PROTOC)
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Moci.proto=./proto_go' \
+		--go_opt='Mmeta.proto=./proto_go' \
+		--go_opt='Mconfig.proto=./proto_go' \
+		--go_opt='Mmanifest.proto=./proto_go' \
+		--go_opt='Mindex.proto=./proto_go' \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		--go_opt='Mversioned.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/meta.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mversioned.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/versioned.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/descriptor.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		--go_opt='Mversioned.proto=./proto_go' \
+		--go_opt='Mindex.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/index.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Moci.proto=./proto_go' \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		--go_opt='Mconfig.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/config.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/ \
+		--go_opt='Mversioned.proto=./proto_go' \
+		--go_opt='Mdescriptor.proto=./proto_go' \
+		--go_opt='Mmanifest.proto=./proto_go' \
+		$(TOP_LEVEL)/pkg/meta/proto/manifest.proto
+
 .PHONY: binary-minimal
 binary-minimal: EXTENSIONS=
 binary-minimal: modcheck build-metadata
@@ -218,6 +279,13 @@ $(CRICTL):
 	mv crictl $(TOOLSDIR)/bin/crictl
 	chmod +x $(TOOLSDIR)/bin/crictl
 
+$(PROTOC):
+	mkdir -p $(TOOLSDIR)/bin
+	curl -Lo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS)-$(PROTOC_ARCH).zip
+	unzip -o -d $(TOOLSDIR) protoc.zip bin/protoc
+	rm protoc.zip
+	chmod +x $(PROTOC)
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(GO_PROTOC_VERSION)
 
 $(ACTION_VALIDATOR):
 	mkdir -p $(TOOLSDIR)/bin
@@ -242,7 +310,7 @@ $(GOLINTER):
 
 .PHONY: check
 check: $(if $(findstring ui,$(BUILD_LABELS)), ui)
-check: ./golangcilint.yaml $(GOLINTER)
+check: ./golangcilint.yaml modcheck $(GOLINTER)
 	mkdir -p pkg/extensions/build; touch pkg/extensions/build/.empty
 	$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags containers_image_openpgp ./...
 	$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags $(BUILD_LABELS),containers_image_openpgp  ./...
@@ -339,7 +407,7 @@ verify-config-commited: _verify-config
 	fi; \
 
 .PHONY: gqlgen
-gqlgen:
+gqlgen: gen-protobuf
 	cd pkg/extensions/search;\
 	go run github.com/99designs/gqlgen version;\
 	go run github.com/99designs/gqlgen generate

diff --git a/codecov.yml b/codecov.yml
@@ -6,3 +6,4 @@ ignore:
     - "./pkg/test/mocks/*.go"
     - "./swagger/*.go"
     - "./pkg/test/test_http_server.go"
+    - "./pkg/meta/proto_go/*.go"
diff --git a/errors/errors.go b/errors/errors.go
@@ -108,6 +108,8 @@ var (
 	ErrManifestConflict               = errors.New("manifest: multiple manifests found")
 	ErrManifestMetaNotFound           = errors.New("metadb: image metadata not found for given manifest reference")
 	ErrManifestDataNotFound           = errors.New("metadb: image data not found for given manifest digest")
+	ErrImageMetaNotFound              = errors.New("metadb: image data not found")
+	ErrUnexpectedMediaType            = errors.New("metadb: got unexpected media type")
 	ErrIndexDataNotFount              = errors.New("metadb: index data not found for given digest")
 	ErrRepoMetaNotFound               = errors.New("metadb: repo metadata not found for given repo name")
 	ErrTagMetaNotFound                = errors.New("metadb: tag metadata not found for given repo and tag names")
@@ -163,4 +165,5 @@ var (
 	ErrInvalidOutputFormat            = errors.New("cli: invalid output format")
 	ErrFlagValueUnsupported           = errors.New("supported values ")
 	ErrUnknownSubcommand              = errors.New("cli: unknown subcommand")
+	ErrMultipleReposSameName          = errors.New("test: can't have multiple repos with the same name")
 )
diff --git a/examples/config-all-remote.json b/examples/config-all-remote.json
@@ -18,7 +18,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageMetaTablename": "ZotImageMetaTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "versionTablename": "ZotVersion"
         }
     },

diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json
@@ -19,7 +19,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageMetaTablename": "ZotImageMetaTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "userDataTablename": "ZotUserDataTable",
             "versionTablename": "ZotVersion"
         }

diff --git a/go.mod b/go.mod
@@ -500,7 +500,7 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
 	google.golang.org/grpc v1.58.2 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go
@@ -153,14 +153,15 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 
 		endpoint := os.Getenv("DYNAMODBMOCK_ENDPOINT")
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dynamodb",
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"name":                   "dynamodb",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
@@ -169,27 +170,29 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 		// negative test cases
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
 		So(driver, ShouldBeNil)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"name":                   "dummy",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
@@ -215,43 +218,45 @@ func TestCreateMetaDBDriver(t *testing.T) {
 		}
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "BlobTable",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"userdatatablename":     "UserDatatable",
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "BlobTable",
+			"repometatablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "UserDatatable",
 		}
 
 		testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
 		So(testFunc, ShouldPanic)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versiontablename":      1,
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "",
+			"repometatablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versiontablename":       1,
 		}
 
 		testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
 		So(testFunc, ShouldPanic)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "test",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"indexdatatablename":    "IndexDataTable",
-			"userdatatablename":     "ZotUserDataTable",
-			"apikeytablename":       "APIKeyTable",
-			"versiontablename":      "1",
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "test",
+			"repometatablename":      "RepoMetadataTable",
+			"imagemetatablename":     "ZotImageMetaTable",
+			"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "ZotUserDataTable",
+			"apikeytablename":        "APIKeyTable",
+			"versiontablename":       "1",
 		}
 
 		testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
@@ -424,16 +429,16 @@ func TestObjectStorageController(t *testing.T) {
 		conf.Storage.StorageDriver = storageDriverParams
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dynamodb",
-			"endpoint":              os.Getenv("DYNAMODBMOCK_ENDPOINT"),
-			"region":                "us-east-2",
-			"cachetablename":        "test",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"indexdatatablename":    "IndexDataTable",
-			"userdatatablename":     "ZotUserDataTable",
-			"apikeytablename":       "APIKeyTable1",
-			"versiontablename":      "Version",
+			"name":                   "dynamodb",
+			"endpoint":               os.Getenv("DYNAMODBMOCK_ENDPOINT"),
+			"region":                 "us-east-2",
+			"cachetablename":         "test",
+			"repometatablename":      "RepoMetadataTable",
+			"imagemetatablename":     "ZotImageMetaTable",
+			"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "ZotUserDataTable",
+			"apikeytablename":        "APIKeyTable1",
+			"versiontablename":       "Version",
 		}
 
 		mockOIDCServer, err := authutils.MockOIDCRun()
@@ -7715,6 +7720,8 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
 }
 
 func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
+	ctx := context.Background()
+
 	Convey("Make controller", t, func() {
 		Convey("Garbage collect signatures without subject and manifests without tags", func(c C) {
 			repoName := "testrepo" //nolint:goconst
@@ -7746,7 +7753,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			ctlr.Config.Storage.Dedupe = false
 
 			cm := test.NewControllerManager(ctlr)
-			cm.StartServer()
+			cm.StartServer() //nolint: contextcheck
 			cm.WaitServerToBeReady(port)
 			defer cm.StopServer()
 
@@ -7777,7 +7784,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 
 			// generate a keypair
 			os.Setenv("COSIGN_PASSWORD", "")
-			err = generate.GenerateKeyPairCmd(context.TODO(), "", "cosign", nil)
+			err = generate.GenerateKeyPairCmd(ctx, "", "cosign", nil)
 			So(err, ShouldBeNil)
 
 			image := fmt.Sprintf("localhost:%s/%s@%s", port, repoName, digest.String())
@@ -7806,7 +7813,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			So(err, ShouldBeNil)
 
 			// sign the image
-			err = signature.SignWithNotation("good", image, tdir, true)
+			err = signature.SignWithNotation("good", image, tdir, true) //nolint: contextcheck
 			So(err, ShouldBeNil)
 
 			// get cosign signature manifest
@@ -7836,7 +7843,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			So(err, ShouldBeNil)
 
 			// make sure both signatures are stored in repodb
-			repoMeta, err := ctlr.MetaDB.GetRepoMeta(repoName)
+			repoMeta, err := ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 			So(err, ShouldBeNil)
 
 			sigMeta := repoMeta.Signatures[digest.String()]
@@ -7897,7 +7904,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 				So(err, ShouldBeNil)
 
 				// make sure repoDB reference was added
-				repoMeta, err := ctlr.MetaDB.GetRepoMeta(repoName)
+				repoMeta, err := ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 				So(err, ShouldBeNil)
 
 				_, ok := repoMeta.Referrers[untaggedManifestDigest.String()]
@@ -7926,8 +7933,8 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 				err = gc.CleanRepo(repoName)
 				So(err, ShouldBeNil)
 
-				// make sure both signatures are removed from repodb and repo reference for untagged is removed
-				repoMeta, err = ctlr.MetaDB.GetRepoMeta(repoName)
+				// make sure both signatures are removed from metaDB and repo reference for untagged is removed
+				repoMeta, err = ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 				So(err, ShouldBeNil)
 
 				sigMeta := repoMeta.Signatures[digest.String()]