From 39e94e15f0ef1c731488b289153e3c4f3c837618 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 9 Oct 2024 18:23:46 +0000 Subject: [PATCH] feat: add support for docker images Issue #724 A new config section under "HTTP" called "Compat" is added which currently takes a list of possible compatible legacy media-types. Only "docker2s2" (Docker Manifest V2 Schema V2) is currently supported. Signed-off-by: Ramkumar Chinchani --- examples/config-docker-compat.json | 14 +++++ examples/config-minimal.json | 7 +-- pkg/api/config/config.go | 4 +- pkg/api/routes.go | 2 +- pkg/compat/compat.go | 40 ++++++++++++++ pkg/extensions/sync/destination.go | 2 +- pkg/storage/common/common.go | 17 ++++-- pkg/storage/imagestore/imagestore.go | 8 ++- pkg/storage/local/local.go | 3 ++ pkg/storage/s3/s3.go | 5 +- pkg/storage/storage.go | 8 +-- test/blackbox/docker-compat.bats | 79 ++++++++++++++++++++++++++++ 12 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 examples/config-docker-compat.json create mode 100644 pkg/compat/compat.go create mode 100644 test/blackbox/docker-compat.bats diff --git a/examples/config-docker-compat.json b/examples/config-docker-compat.json new file mode 100644 index 000000000..a42b2ba61 --- /dev/null +++ b/examples/config-docker-compat.json @@ -0,0 +1,14 @@ +{ + "distSpecVersion": "1.1.0", + "storage": { + "rootDirectory": "/data/hdd/rchincha/tmp/zot" + }, + "http": { + "address": "0.0.0.0", + "port": "8080", + "compat": ["docker2s2"] + }, + "log": { + "level": "debug" + } +} diff --git a/examples/config-minimal.json b/examples/config-minimal.json index 6b0587985..a42b2ba61 100644 --- a/examples/config-minimal.json +++ b/examples/config-minimal.json @@ -1,11 +1,12 @@ { "distSpecVersion": "1.1.0", "storage": { - "rootDirectory": "/tmp/zot" + "rootDirectory": "/data/hdd/rchincha/tmp/zot" }, "http": { - "address": "127.0.0.1", - "port": "8080" + "address": "0.0.0.0", + "port": "8080", + "compat": ["docker2s2"] }, "log": { "level": "debug" diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index b7d5c7f71..07929c21f 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -7,6 +7,7 @@ import ( distspec "github.com/opencontainers/distribution-spec/specs-go" + "zotregistry.dev/zot/pkg/compat" extconf "zotregistry.dev/zot/pkg/extensions/config" storageConstants "zotregistry.dev/zot/pkg/storage/constants" ) @@ -122,7 +123,8 @@ type HTTPConfig struct { Auth *AuthConfig AccessControl *AccessControlConfig `mapstructure:"accessControl,omitempty"` Realm string - Ratelimit *RatelimitConfig `mapstructure:",omitempty"` + Ratelimit *RatelimitConfig `mapstructure:",omitempty"` + Compat []compat.MediaCompatibility `mapstructure:",omitempty"` } type SchedulerConfig struct { diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 97774058f..164b6bf9a 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -688,7 +688,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht } mediaType := request.Header.Get("Content-Type") - if !storageCommon.IsSupportedMediaType(mediaType) { + if !storageCommon.IsSupportedMediaType(rh.c.Config.HTTP.Compat, mediaType) { err := apiErr.NewError(apiErr.MANIFEST_INVALID).AddDetail(map[string]string{"mediaType": mediaType}) zcommon.WriteJSON(response, http.StatusUnsupportedMediaType, apiErr.NewErrorList(err)) diff --git a/pkg/compat/compat.go b/pkg/compat/compat.go new file mode 100644 index 000000000..28e3f700d --- /dev/null +++ b/pkg/compat/compat.go @@ -0,0 +1,40 @@ +package compat + +import ( + "encoding/json" + + "zotregistry.dev/zot/errors" +) + +// MediaCompatibility determines non-OCI media-compatilibility. +type MediaCompatibility string + +const ( + DockerManifestV2SchemaV2 = "docker2s2" +) + +func (mc MediaCompatibility) MarshalJSON() ([]byte, error) { + switch mc { + case DockerManifestV2SchemaV2: + return json.Marshal(DockerManifestV2SchemaV2) + default: + return nil, errors.ErrUnexpectedMediaType + } +} + +func (mc *MediaCompatibility) UnmarshalJSON(data []byte) error { + var s string + err := json.Unmarshal(data, &s) + if err != nil { + return err + } + + switch s { + case DockerManifestV2SchemaV2: + *mc = MediaCompatibility(DockerManifestV2SchemaV2) + default: + return errors.ErrUnexpectedMediaType + } + + return nil +} diff --git a/pkg/extensions/sync/destination.go b/pkg/extensions/sync/destination.go index 3384e6270..b93eec625 100644 --- a/pkg/extensions/sync/destination.go +++ b/pkg/extensions/sync/destination.go @@ -304,5 +304,5 @@ func getTempRootDirFromImageReference(imageReference types.ImageReference, repo, func getImageStore(rootDir string, log log.Logger) storageTypes.ImageStore { metrics := monitoring.NewMetricsServer(false, log) - return local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + return local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) } diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index e28239d59..1a2b540da 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -17,8 +17,11 @@ import ( imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" + dockerList "github.com/distribution/distribution/v3/manifest/manifestlist" + docker "github.com/distribution/distribution/v3/manifest/schema2" zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/scheduler" @@ -62,10 +65,10 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc } func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte, - log zlog.Logger, + compats []compat.MediaCompatibility, log zlog.Logger, ) error { // validate the manifest - if !IsSupportedMediaType(mediaType) { + if !IsSupportedMediaType(compats, mediaType) { log.Debug().Interface("actual", mediaType). Msg("bad manifest media type") @@ -795,7 +798,15 @@ func getBlobDescriptorFromManifest(imgStore storageTypes.ImageStore, repo string return ispec.Descriptor{}, zerr.ErrBlobNotFound } -func IsSupportedMediaType(mediaType string) bool { +func IsSupportedMediaType(compats []compat.MediaCompatibility, mediaType string) bool { + // check for some supported legacy formats if configured + for _, comp := range compats { + if comp == compat.DockerManifestV2SchemaV2 && + (mediaType == docker.MediaTypeManifest || mediaType == dockerList.MediaTypeManifestList) { + return true + } + } + return mediaType == ispec.MediaTypeImageIndex || mediaType == ispec.MediaTypeImageManifest } diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index 8fe1846f4..7664aeb73 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -21,6 +21,7 @@ import ( zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" syncConstants "zotregistry.dev/zot/pkg/extensions/sync/constants" zlog "zotregistry.dev/zot/pkg/log" @@ -49,6 +50,7 @@ type ImageStore struct { dedupe bool linter common.Lint commit bool + compat []compat.MediaCompatibility } func (is *ImageStore) Name() string { @@ -67,7 +69,8 @@ func (is *ImageStore) DirExists(d string) bool { // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, - metrics monitoring.MetricServer, linter common.Lint, storeDriver storageTypes.Driver, cacheDriver cache.Cache, + metrics monitoring.MetricServer, linter common.Lint, storeDriver storageTypes.Driver, + cacheDriver cache.Cache, compat []compat.MediaCompatibility, ) storageTypes.ImageStore { if err := storeDriver.EnsureDir(rootDir); err != nil { log.Error().Err(err).Str("rootDir", rootDir).Msg("failed to create root dir") @@ -85,6 +88,7 @@ func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlo linter: linter, commit: commit, cache: cacheDriver, + compat: compat, } return imgStore @@ -490,7 +494,7 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli refIsDigest = false } - err = common.ValidateManifest(is, repo, reference, mediaType, body, is.log) + err = common.ValidateManifest(is, repo, reference, mediaType, body, is.compat, is.log) if err != nil { return mDigest, "", err } diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index ecf1ffd19..3309415c2 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -1,6 +1,7 @@ package local import ( + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/storage/cache" @@ -13,6 +14,7 @@ import ( // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache, + compat []compat.MediaCompatibility, ) storageTypes.ImageStore { return imagestore.NewImageStore( rootDir, @@ -24,5 +26,6 @@ func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger, linter, New(commit), cacheDriver, + compat, ) } diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index a3ebc64ab..41a8915c7 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -6,6 +6,7 @@ import ( // Load s3 driver. _ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws" + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/storage/cache" @@ -18,7 +19,8 @@ import ( // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, - metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, cacheDriver cache.Cache, + metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, + cacheDriver cache.Cache, compat []compat.MediaCompatibility, ) storageTypes.ImageStore { return imagestore.NewImageStore( rootDir, @@ -30,5 +32,6 @@ func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlo linter, New(store), cacheDriver, + compat, ) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index edb366018..e76b2c61c 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -58,7 +58,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer //nolint:typecheck,contextcheck rootDir := config.Storage.RootDirectory defaultStore = local.NewImageStore(rootDir, - config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, cacheDriver, + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, cacheDriver, config.HTTP.Compat, ) } else { storeName := fmt.Sprintf("%v", config.Storage.StorageDriver["name"]) @@ -92,7 +92,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer // false positive lint - linter does not implement Lint method //nolint: typecheck,contextcheck defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory, - config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver) + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver, config.HTTP.Compat) } storeController.DefaultStore = defaultStore @@ -170,7 +170,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, rootDir := storageConfig.RootDirectory imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(rootDir, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, cacheDriver, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, cacheDriver, cfg.HTTP.Compat, ) subImageStore[route] = imgStoreMap[storageConfig.RootDirectory] @@ -210,7 +210,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // false positive lint - linter does not implement Lint method //nolint: typecheck subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, cfg.HTTP.Compat, ) } } diff --git a/test/blackbox/docker-compat.bats b/test/blackbox/docker-compat.bats new file mode 100644 index 000000000..2a81ce0b4 --- /dev/null +++ b/test/blackbox/docker-compat.bats @@ -0,0 +1,79 @@ +# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci" +# Makefile target installs & checks all necessary tooling +# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() + +load helpers_zot + +function verify_prerequisites { + if [ ! $(command -v curl) ]; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + return 0 +} + +function setup_file() { + # Verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + cat > ${zot_config_file}< Dockerfile < /testfile +EOF + docker build -f Dockerfile . -t localhost:${zot_port}/test + run docker push localhost:${zot_port}/test + [ "$status" -eq 0 ] + run docker pull localhost:${zot_port}/test + [ "$status" -eq 0 ] +}