refactor(metadb): improve UX by speeding up metadb serialize/deserialize

Use protocol buffers and update the metadb interface to better suit our search needs Signed-off-by: Ramkumar Chinchani <[email protected]> Signed-off-by: Laurentiu Niculae <[email protected]>
project-zot · Oct 30, 2023 · 5801e14 · 5801e14
1 parent f34af3c
commit 5801e14
Show file tree

Hide file tree

Showing 82 changed files with 9,904 additions and 11,123 deletions.
diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml
@@ -38,6 +38,7 @@ jobs:
 
           # Optional: if set to true then the action don't cache or restore ~/go/pkg.
           # skip-pkg-cache: true
+          # skip-pkg-cache: false
 
           # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
           # skip-build-cache: true

diff --git a/.gitignore b/.gitignore
@@ -18,6 +18,7 @@ pkg/extensions/build/
 hack/
 .stacker/
 oci/
+!pkg/meta/proto/oci
 roots/
 bin/
 bazel-*

diff --git a/Makefile b/Makefile
@@ -32,6 +32,22 @@ TESTDATA := $(TOP_LEVEL)/test/data
 OS ?= $(shell go env GOOS)
 ARCH ?= $(shell go env GOARCH)
 
+PROTOC := $(TOOLSDIR)/bin/protoc
+PROTOC_VERSION := 24.4
+GO_PROTOC_VERSION := 1.31.0
+HOST_OS := $(shell go env GOOS)
+HOST_ARCH := $(shell go env GOARCH)
+ifeq ($(HOST_OS),linux)
+	PROTOC_OS := linux
+else ifeq ($(HOST_OS),darwin)
+	PROTOC_OS := osx
+endif
+ifeq ($(HOST_ARCH),amd64)
+	PROTOC_ARCH := x86_64
+else ifeq ($(HOST_ARCH),arm64)
+	PROTOC_ARCH := aarch_64
+endif
+
 BENCH_OUTPUT ?= stdout
 ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs
 EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust
@@ -97,6 +113,51 @@ build-metadata: $(if $(findstring ui,$(BUILD_LABELS)), ui)
 	echo "\n Files: \n"
 	go list -tags $(BUILD_TAGS) -f '{{ join .GoFiles "\n" }}' ./... | sort -u
 
+.PHONY: gen-protobuf
+gen-protobuf: check-not-freebds $(PROTOC)
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/oci.proto=./gen' \
+		--go_opt='Mmeta/meta.proto=./gen' \
+		--go_opt='Moci/config.proto=./gen' \
+		--go_opt='Moci/manifest.proto=./gen' \
+		--go_opt='Moci/index.proto=./gen' \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		--go_opt='Moci/versioned.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/meta/meta.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/versioned.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/versioned.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/descriptor.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		--go_opt='Moci/versioned.proto=./gen' \
+		--go_opt='Moci/index.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/index.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/oci.proto=./gen' \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		--go_opt='Moci/config.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/config.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/versioned.proto=./gen' \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		--go_opt='Moci/manifest.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/manifest.proto
+
 .PHONY: binary-minimal
 binary-minimal: EXTENSIONS=
 binary-minimal: modcheck build-metadata
@@ -218,6 +279,13 @@ $(CRICTL):
 	mv crictl $(TOOLSDIR)/bin/crictl
 	chmod +x $(TOOLSDIR)/bin/crictl
 
+$(PROTOC):
+	mkdir -p $(TOOLSDIR)/bin
+	curl -Lo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS)-$(PROTOC_ARCH).zip
+	unzip -o -d $(TOOLSDIR) protoc.zip bin/protoc
+	rm protoc.zip
+	chmod +x $(PROTOC)
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(GO_PROTOC_VERSION)
 
 $(ACTION_VALIDATOR):
 	mkdir -p $(TOOLSDIR)/bin
@@ -515,6 +583,12 @@ ifneq ($(shell go env GOOS),linux)
 	$(error makefile target can be run only on linux)
 endif
 
+.PHONY: check-not-freebds
+check-not-freebds:
+ifneq ($(shell go env GOOS),freebsd)
+	$(error makefile target can't be run on freebsd)
+endif
+
 .PHONY: check-compatibility
 check-compatibility:
 ifeq ($(OS),freebsd)

diff --git a/codecov.yml b/codecov.yml
@@ -6,3 +6,4 @@ ignore:
     - "./pkg/test/mocks/*.go"
     - "./swagger/*.go"
     - "./pkg/test/test_http_server.go"
+    - "./pkg/meta/proto/gen/*.go"
diff --git a/errors/errors.go b/errors/errors.go
@@ -108,6 +108,8 @@ var (
 	ErrManifestConflict               = errors.New("manifest: multiple manifests found")
 	ErrManifestMetaNotFound           = errors.New("metadb: image metadata not found for given manifest reference")
 	ErrManifestDataNotFound           = errors.New("metadb: image data not found for given manifest digest")
+	ErrImageMetaNotFound              = errors.New("metadb: image data not found")
+	ErrUnexpectedMediaType            = errors.New("metadb: got unexpected media type")
 	ErrIndexDataNotFount              = errors.New("metadb: index data not found for given digest")
 	ErrRepoMetaNotFound               = errors.New("metadb: repo metadata not found for given repo name")
 	ErrTagMetaNotFound                = errors.New("metadb: tag metadata not found for given repo and tag names")
@@ -163,4 +165,5 @@ var (
 	ErrInvalidOutputFormat            = errors.New("cli: invalid output format")
 	ErrFlagValueUnsupported           = errors.New("supported values ")
 	ErrUnknownSubcommand              = errors.New("cli: unknown subcommand")
+	ErrMultipleReposSameName          = errors.New("test: can't have multiple repos with the same name")
 )
diff --git a/examples/config-all-remote.json b/examples/config-all-remote.json
@@ -18,7 +18,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageMetaTablename": "ZotImageMetaTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "versionTablename": "ZotVersion"
         }
     },

diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json
@@ -19,7 +19,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageMetaTablename": "ZotImageMetaTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "userDataTablename": "ZotUserDataTable",
             "versionTablename": "ZotVersion"
         }

diff --git a/go.mod b/go.mod
@@ -500,7 +500,7 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
 	google.golang.org/grpc v1.58.2 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go
@@ -143,14 +143,15 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 
 		endpoint := os.Getenv("DYNAMODBMOCK_ENDPOINT")
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dynamodb",
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"name":                   "dynamodb",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
@@ -159,27 +160,29 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
 		// negative test cases
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
 		So(driver, ShouldBeNil)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              endpoint,
-			"region":                "us-east-2",
-			"cacheTablename":        "BlobTable",
-			"repoMetaTablename":     "RepoMetadataTable",
-			"manifestDataTablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versionTablename":      "Version",
+			"name":                   "dummy",
+			"endpoint":               endpoint,
+			"region":                 "us-east-2",
+			"cacheTablename":         "BlobTable",
+			"repoMetaTablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versionTablename":       "Version",
 		}
 
 		driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
@@ -205,43 +208,45 @@ func TestCreateMetaDBDriver(t *testing.T) {
 		}
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "BlobTable",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"userdatatablename":     "UserDatatable",
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "BlobTable",
+			"repometatablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "UserDatatable",
 		}
 
 		testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
 		So(testFunc, ShouldPanic)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"userDataTablename":     "ZotUserDataTable",
-			"versiontablename":      1,
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "",
+			"repometatablename":      "RepoMetadataTable",
+			"imageMetaTablename":     "ZotImageMetaTable",
+			"repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
+			"userDataTablename":      "ZotUserDataTable",
+			"versiontablename":       1,
 		}
 
 		testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
 		So(testFunc, ShouldPanic)
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dummy",
-			"endpoint":              "http://localhost:4566",
-			"region":                "us-east-2",
-			"cachetablename":        "test",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"indexdatatablename":    "IndexDataTable",
-			"userdatatablename":     "ZotUserDataTable",
-			"apikeytablename":       "APIKeyTable",
-			"versiontablename":      "1",
+			"name":                   "dummy",
+			"endpoint":               "http://localhost:4566",
+			"region":                 "us-east-2",
+			"cachetablename":         "test",
+			"repometatablename":      "RepoMetadataTable",
+			"imagemetatablename":     "ZotImageMetaTable",
+			"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "ZotUserDataTable",
+			"apikeytablename":        "APIKeyTable",
+			"versiontablename":       "1",
 		}
 
 		testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) }
@@ -414,16 +419,16 @@ func TestObjectStorageController(t *testing.T) {
 		conf.Storage.StorageDriver = storageDriverParams
 
 		conf.Storage.CacheDriver = map[string]interface{}{
-			"name":                  "dynamodb",
-			"endpoint":              os.Getenv("DYNAMODBMOCK_ENDPOINT"),
-			"region":                "us-east-2",
-			"cachetablename":        "test",
-			"repometatablename":     "RepoMetadataTable",
-			"manifestdatatablename": "ManifestDataTable",
-			"indexdatatablename":    "IndexDataTable",
-			"userdatatablename":     "ZotUserDataTable",
-			"apikeytablename":       "APIKeyTable1",
-			"versiontablename":      "Version",
+			"name":                   "dynamodb",
+			"endpoint":               os.Getenv("DYNAMODBMOCK_ENDPOINT"),
+			"region":                 "us-east-2",
+			"cachetablename":         "test",
+			"repometatablename":      "RepoMetadataTable",
+			"imagemetatablename":     "ZotImageMetaTable",
+			"repoblobsinfotablename": "ZotRepoBlobsInfoTable",
+			"userdatatablename":      "ZotUserDataTable",
+			"apikeytablename":        "APIKeyTable1",
+			"versiontablename":       "Version",
 		}
 
 		mockOIDCServer, err := authutils.MockOIDCRun()
@@ -7773,6 +7778,8 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
 }
 
 func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
+	ctx := context.Background()
+
 	Convey("Make controller", t, func() {
 		Convey("Garbage collect signatures without subject and manifests without tags", func(c C) {
 			repoName := "testrepo" //nolint:goconst
@@ -7804,7 +7811,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			ctlr.Config.Storage.Dedupe = false
 
 			cm := test.NewControllerManager(ctlr)
-			cm.StartServer()
+			cm.StartServer() //nolint: contextcheck
 			cm.WaitServerToBeReady(port)
 			defer cm.StopServer()
 
@@ -7835,7 +7842,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 
 			// generate a keypair
 			os.Setenv("COSIGN_PASSWORD", "")
-			err = generate.GenerateKeyPairCmd(context.TODO(), "", "cosign", nil)
+			err = generate.GenerateKeyPairCmd(ctx, "", "cosign", nil)
 			So(err, ShouldBeNil)
 
 			image := fmt.Sprintf("localhost:%s/%s@%s", port, repoName, digest.String())
@@ -7864,7 +7871,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			So(err, ShouldBeNil)
 
 			// sign the image
-			err = signature.SignWithNotation("good", image, tdir, true)
+			err = signature.SignWithNotation("good", image, tdir, true) //nolint: contextcheck
 			So(err, ShouldBeNil)
 
 			// get cosign signature manifest
@@ -7894,7 +7901,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 			So(err, ShouldBeNil)
 
 			// make sure both signatures are stored in repodb
-			repoMeta, err := ctlr.MetaDB.GetRepoMeta(repoName)
+			repoMeta, err := ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 			So(err, ShouldBeNil)
 
 			sigMeta := repoMeta.Signatures[digest.String()]
@@ -7955,7 +7962,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 				So(err, ShouldBeNil)
 
 				// make sure repoDB reference was added
-				repoMeta, err := ctlr.MetaDB.GetRepoMeta(repoName)
+				repoMeta, err := ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 				So(err, ShouldBeNil)
 
 				_, ok := repoMeta.Referrers[untaggedManifestDigest.String()]
@@ -7984,8 +7991,8 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) {
 				err = gc.CleanRepo(repoName)
 				So(err, ShouldBeNil)
 
-				// make sure both signatures are removed from repodb and repo reference for untagged is removed
-				repoMeta, err = ctlr.MetaDB.GetRepoMeta(repoName)
+				// make sure both signatures are removed from metaDB and repo reference for untagged is removed
+				repoMeta, err = ctlr.MetaDB.GetRepoMeta(ctx, repoName)
 				So(err, ShouldBeNil)
 
 				sigMeta := repoMeta.Signatures[digest.String()]