refactor(metadb): improve UX by speeding up metadb serialize/deserialize

Use protocol buffers and update the metadb interface to better suit our search needs Signed-off-by: Ramkumar Chinchani <[email protected]> Signed-off-by: Laurentiu Niculae <[email protected]>
project-zot · Oct 30, 2023 · bf8e5a5 · bf8e5a5
1 parent f34af3c
commit bf8e5a5
Show file tree

Hide file tree

Showing 82 changed files with 9,904 additions and 11,125 deletions.
diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml
@@ -38,6 +38,7 @@ jobs:
 
           # Optional: if set to true then the action don't cache or restore ~/go/pkg.
           # skip-pkg-cache: true
+          # skip-pkg-cache: false
 
           # Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
           # skip-build-cache: true

diff --git a/.gitignore b/.gitignore
@@ -18,6 +18,7 @@ pkg/extensions/build/
 hack/
 .stacker/
 oci/
+!pkg/meta/proto/oci
 roots/
 bin/
 bazel-*

diff --git a/Makefile b/Makefile
@@ -32,6 +32,22 @@ TESTDATA := $(TOP_LEVEL)/test/data
 OS ?= $(shell go env GOOS)
 ARCH ?= $(shell go env GOARCH)
 
+PROTOC := $(TOOLSDIR)/bin/protoc
+PROTOC_VERSION := 24.4
+GO_PROTOC_VERSION := 1.31.0
+HOST_OS := $(shell go env GOOS)
+HOST_ARCH := $(shell go env GOARCH)
+ifeq ($(HOST_OS),linux)
+	PROTOC_OS := linux
+else ifeq ($(HOST_OS),darwin)
+	PROTOC_OS := osx
+endif
+ifeq ($(HOST_ARCH),amd64)
+	PROTOC_ARCH := x86_64
+else ifeq ($(HOST_ARCH),arm64)
+	PROTOC_ARCH := aarch_64
+endif
+
 BENCH_OUTPUT ?= stdout
 ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs
 EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust
@@ -97,6 +113,51 @@ build-metadata: $(if $(findstring ui,$(BUILD_LABELS)), ui)
 	echo "\n Files: \n"
 	go list -tags $(BUILD_TAGS) -f '{{ join .GoFiles "\n" }}' ./... | sort -u
 
+.PHONY: gen-protobuf
+gen-protobuf: check-not-freebds $(PROTOC)
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/oci.proto=./gen' \
+		--go_opt='Mmeta/meta.proto=./gen' \
+		--go_opt='Moci/config.proto=./gen' \
+		--go_opt='Moci/manifest.proto=./gen' \
+		--go_opt='Moci/index.proto=./gen' \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		--go_opt='Moci/versioned.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/meta/meta.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/versioned.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/versioned.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/descriptor.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		--go_opt='Moci/versioned.proto=./gen' \
+		--go_opt='Moci/index.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/index.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/oci.proto=./gen' \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		--go_opt='Moci/config.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/config.proto
+	$(PROTOC) --experimental_allow_proto3_optional \
+		--proto_path=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_out=$(TOP_LEVEL)/pkg/meta/proto \
+		--go_opt='Moci/versioned.proto=./gen' \
+		--go_opt='Moci/descriptor.proto=./gen' \
+		--go_opt='Moci/manifest.proto=./gen' \
+		$(TOP_LEVEL)/pkg/meta/proto/oci/manifest.proto
+
 .PHONY: binary-minimal
 binary-minimal: EXTENSIONS=
 binary-minimal: modcheck build-metadata
@@ -218,6 +279,13 @@ $(CRICTL):
 	mv crictl $(TOOLSDIR)/bin/crictl
 	chmod +x $(TOOLSDIR)/bin/crictl
 
+$(PROTOC):
+	mkdir -p $(TOOLSDIR)/bin
+	curl -Lo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS)-$(PROTOC_ARCH).zip
+	unzip -o -d $(TOOLSDIR) protoc.zip bin/protoc
+	rm protoc.zip
+	chmod +x $(PROTOC)
+	go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(GO_PROTOC_VERSION)
 
 $(ACTION_VALIDATOR):
 	mkdir -p $(TOOLSDIR)/bin
@@ -515,6 +583,12 @@ ifneq ($(shell go env GOOS),linux)
 	$(error makefile target can be run only on linux)
 endif
 
+.PHONY: check-not-freebds
+check-not-freebds:
+ifneq ($(shell go env GOOS),freebsd)
+	$(error makefile target can't be run on freebsd)
+endif
+
 .PHONY: check-compatibility
 check-compatibility:
 ifeq ($(OS),freebsd)

diff --git a/codecov.yml b/codecov.yml
@@ -6,3 +6,4 @@ ignore:
     - "./pkg/test/mocks/*.go"
     - "./swagger/*.go"
     - "./pkg/test/test_http_server.go"
+    - "./pkg/meta/proto/gen/*.go"
diff --git a/errors/errors.go b/errors/errors.go
@@ -107,8 +107,8 @@ var (
 	ErrBadLayerCount                  = errors.New("manifest: layers count doesn't correspond to config history")
 	ErrManifestConflict               = errors.New("manifest: multiple manifests found")
 	ErrManifestMetaNotFound           = errors.New("metadb: image metadata not found for given manifest reference")
-	ErrManifestDataNotFound           = errors.New("metadb: image data not found for given manifest digest")
-	ErrIndexDataNotFount              = errors.New("metadb: index data not found for given digest")
+	ErrImageMetaNotFound              = errors.New("metadb: image meta not found")
+	ErrUnexpectedMediaType            = errors.New("metadb: got unexpected media type")
 	ErrRepoMetaNotFound               = errors.New("metadb: repo metadata not found for given repo name")
 	ErrTagMetaNotFound                = errors.New("metadb: tag metadata not found for given repo and tag names")
 	ErrTypeAssertionFailed            = errors.New("storage: failed DatabaseDriver type assertion")
@@ -163,4 +163,5 @@ var (
 	ErrInvalidOutputFormat            = errors.New("cli: invalid output format")
 	ErrFlagValueUnsupported           = errors.New("supported values ")
 	ErrUnknownSubcommand              = errors.New("cli: unknown subcommand")
+	ErrMultipleReposSameName          = errors.New("test: can't have multiple repos with the same name")
 )
diff --git a/examples/config-all-remote.json b/examples/config-all-remote.json
@@ -18,7 +18,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageMetaTablename": "ZotImageMetaTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "versionTablename": "ZotVersion"
         }
     },

diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json
@@ -19,7 +19,8 @@
             "region": "us-east-2",
             "cacheTablename": "ZotBlobTable",
             "repoMetaTablename": "ZotRepoMetadataTable",
-            "manifestDataTablename": "ZotManifestDataTable",
+            "imageMetaTablename": "ZotImageMetaTable",
+            "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable",
             "userDataTablename": "ZotUserDataTable",
             "versionTablename": "ZotVersion"
         }

diff --git a/go.mod b/go.mod
@@ -500,7 +500,7 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect
 	google.golang.org/grpc v1.58.2 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect