Skip to content

Commit

Permalink
feat: add support for docker images
Browse files Browse the repository at this point in the history
Issue #724

A new config section under "HTTP" called "Compat" is added which
currently takes a list of possible compatible legacy media-types.

Only "docker2s2" (Docker Manifest V2 Schema V2) is currently supported.

Signed-off-by: Ramkumar Chinchani <[email protected]>
  • Loading branch information
rchincha committed Oct 21, 2024
1 parent da6bd56 commit 39e94e1
Show file tree
Hide file tree
Showing 12 changed files with 173 additions and 16 deletions.
14 changes: 14 additions & 0 deletions examples/config-docker-compat.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"distSpecVersion": "1.1.0",
"storage": {
"rootDirectory": "/data/hdd/rchincha/tmp/zot"
},
"http": {
"address": "0.0.0.0",
"port": "8080",
"compat": ["docker2s2"]
},
"log": {
"level": "debug"
}
}
7 changes: 4 additions & 3 deletions examples/config-minimal.json
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
{
"distSpecVersion": "1.1.0",
"storage": {
"rootDirectory": "/tmp/zot"
"rootDirectory": "/data/hdd/rchincha/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "8080"
"address": "0.0.0.0",
"port": "8080",
"compat": ["docker2s2"]
},
"log": {
"level": "debug"
Expand Down
4 changes: 3 additions & 1 deletion pkg/api/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

distspec "github.com/opencontainers/distribution-spec/specs-go"

"zotregistry.dev/zot/pkg/compat"
extconf "zotregistry.dev/zot/pkg/extensions/config"
storageConstants "zotregistry.dev/zot/pkg/storage/constants"
)
Expand Down Expand Up @@ -122,7 +123,8 @@ type HTTPConfig struct {
Auth *AuthConfig
AccessControl *AccessControlConfig `mapstructure:"accessControl,omitempty"`
Realm string
Ratelimit *RatelimitConfig `mapstructure:",omitempty"`
Ratelimit *RatelimitConfig `mapstructure:",omitempty"`
Compat []compat.MediaCompatibility `mapstructure:",omitempty"`
}

type SchedulerConfig struct {
Expand Down
2 changes: 1 addition & 1 deletion pkg/api/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
}

mediaType := request.Header.Get("Content-Type")
if !storageCommon.IsSupportedMediaType(mediaType) {
if !storageCommon.IsSupportedMediaType(rh.c.Config.HTTP.Compat, mediaType) {
err := apiErr.NewError(apiErr.MANIFEST_INVALID).AddDetail(map[string]string{"mediaType": mediaType})
zcommon.WriteJSON(response, http.StatusUnsupportedMediaType, apiErr.NewErrorList(err))

Expand Down
40 changes: 40 additions & 0 deletions pkg/compat/compat.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package compat

import (
"encoding/json"

"zotregistry.dev/zot/errors"
)

// MediaCompatibility determines non-OCI media-compatilibility.
type MediaCompatibility string

const (
DockerManifestV2SchemaV2 = "docker2s2"
)

func (mc MediaCompatibility) MarshalJSON() ([]byte, error) {
switch mc {
case DockerManifestV2SchemaV2:
return json.Marshal(DockerManifestV2SchemaV2)
default:
return nil, errors.ErrUnexpectedMediaType
}
}

func (mc *MediaCompatibility) UnmarshalJSON(data []byte) error {
var s string
err := json.Unmarshal(data, &s)
if err != nil {
return err
}

switch s {
case DockerManifestV2SchemaV2:
*mc = MediaCompatibility(DockerManifestV2SchemaV2)
default:
return errors.ErrUnexpectedMediaType
}

return nil
}
2 changes: 1 addition & 1 deletion pkg/extensions/sync/destination.go
Original file line number Diff line number Diff line change
Expand Up @@ -304,5 +304,5 @@ func getTempRootDirFromImageReference(imageReference types.ImageReference, repo,
func getImageStore(rootDir string, log log.Logger) storageTypes.ImageStore {
metrics := monitoring.NewMetricsServer(false, log)

return local.NewImageStore(rootDir, false, false, log, metrics, nil, nil)
return local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil)
}
17 changes: 14 additions & 3 deletions pkg/storage/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,11 @@ import (
imeta "github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"

dockerList "github.com/distribution/distribution/v3/manifest/manifestlist"
docker "github.com/distribution/distribution/v3/manifest/schema2"
zerr "zotregistry.dev/zot/errors"
zcommon "zotregistry.dev/zot/pkg/common"
"zotregistry.dev/zot/pkg/compat"
"zotregistry.dev/zot/pkg/extensions/monitoring"
zlog "zotregistry.dev/zot/pkg/log"
"zotregistry.dev/zot/pkg/scheduler"
Expand Down Expand Up @@ -62,10 +65,10 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc
}

func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte,
log zlog.Logger,
compats []compat.MediaCompatibility, log zlog.Logger,
) error {
// validate the manifest
if !IsSupportedMediaType(mediaType) {
if !IsSupportedMediaType(compats, mediaType) {
log.Debug().Interface("actual", mediaType).
Msg("bad manifest media type")

Expand Down Expand Up @@ -795,7 +798,15 @@ func getBlobDescriptorFromManifest(imgStore storageTypes.ImageStore, repo string
return ispec.Descriptor{}, zerr.ErrBlobNotFound
}

func IsSupportedMediaType(mediaType string) bool {
func IsSupportedMediaType(compats []compat.MediaCompatibility, mediaType string) bool {
// check for some supported legacy formats if configured
for _, comp := range compats {
if comp == compat.DockerManifestV2SchemaV2 &&
(mediaType == docker.MediaTypeManifest || mediaType == dockerList.MediaTypeManifestList) {
return true
}
}

return mediaType == ispec.MediaTypeImageIndex ||
mediaType == ispec.MediaTypeImageManifest
}
Expand Down
8 changes: 6 additions & 2 deletions pkg/storage/imagestore/imagestore.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

zerr "zotregistry.dev/zot/errors"
zcommon "zotregistry.dev/zot/pkg/common"
"zotregistry.dev/zot/pkg/compat"
"zotregistry.dev/zot/pkg/extensions/monitoring"
syncConstants "zotregistry.dev/zot/pkg/extensions/sync/constants"
zlog "zotregistry.dev/zot/pkg/log"
Expand Down Expand Up @@ -49,6 +50,7 @@ type ImageStore struct {
dedupe bool
linter common.Lint
commit bool
compat []compat.MediaCompatibility
}

func (is *ImageStore) Name() string {
Expand All @@ -67,7 +69,8 @@ func (is *ImageStore) DirExists(d string) bool {
// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers
// Use the last argument to properly set a cache database, or it will default to boltDB local storage.
func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger,
metrics monitoring.MetricServer, linter common.Lint, storeDriver storageTypes.Driver, cacheDriver cache.Cache,
metrics monitoring.MetricServer, linter common.Lint, storeDriver storageTypes.Driver,
cacheDriver cache.Cache, compat []compat.MediaCompatibility,
) storageTypes.ImageStore {
if err := storeDriver.EnsureDir(rootDir); err != nil {
log.Error().Err(err).Str("rootDir", rootDir).Msg("failed to create root dir")
Expand All @@ -85,6 +88,7 @@ func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlo
linter: linter,
commit: commit,
cache: cacheDriver,
compat: compat,
}

return imgStore
Expand Down Expand Up @@ -490,7 +494,7 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli
refIsDigest = false
}

err = common.ValidateManifest(is, repo, reference, mediaType, body, is.log)
err = common.ValidateManifest(is, repo, reference, mediaType, body, is.compat, is.log)
if err != nil {
return mDigest, "", err
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/storage/local/local.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package local

import (
"zotregistry.dev/zot/pkg/compat"
"zotregistry.dev/zot/pkg/extensions/monitoring"
zlog "zotregistry.dev/zot/pkg/log"
"zotregistry.dev/zot/pkg/storage/cache"
Expand All @@ -13,6 +14,7 @@ import (
// Use the last argument to properly set a cache database, or it will default to boltDB local storage.
func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger,
metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache,
compat []compat.MediaCompatibility,
) storageTypes.ImageStore {
return imagestore.NewImageStore(
rootDir,
Expand All @@ -24,5 +26,6 @@ func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger,
linter,
New(commit),
cacheDriver,
compat,
)
}
5 changes: 4 additions & 1 deletion pkg/storage/s3/s3.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
// Load s3 driver.
_ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws"

"zotregistry.dev/zot/pkg/compat"
"zotregistry.dev/zot/pkg/extensions/monitoring"
zlog "zotregistry.dev/zot/pkg/log"
"zotregistry.dev/zot/pkg/storage/cache"
Expand All @@ -18,7 +19,8 @@ import (
// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers
// Use the last argument to properly set a cache database, or it will default to boltDB local storage.
func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger,
metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, cacheDriver cache.Cache,
metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver,
cacheDriver cache.Cache, compat []compat.MediaCompatibility,
) storageTypes.ImageStore {
return imagestore.NewImageStore(
rootDir,
Expand All @@ -30,5 +32,6 @@ func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlo
linter,
New(store),
cacheDriver,
compat,
)
}
8 changes: 4 additions & 4 deletions pkg/storage/storage.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer
//nolint:typecheck,contextcheck
rootDir := config.Storage.RootDirectory
defaultStore = local.NewImageStore(rootDir,
config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, cacheDriver,
config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, cacheDriver, config.HTTP.Compat,
)
} else {
storeName := fmt.Sprintf("%v", config.Storage.StorageDriver["name"])
Expand Down Expand Up @@ -92,7 +92,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer
// false positive lint - linter does not implement Lint method
//nolint: typecheck,contextcheck
defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory,
config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver)
config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver, config.HTTP.Compat)
}

storeController.DefaultStore = defaultStore
Expand Down Expand Up @@ -170,7 +170,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig,

rootDir := storageConfig.RootDirectory
imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(rootDir,
storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, cacheDriver,
storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, cacheDriver, cfg.HTTP.Compat,
)

subImageStore[route] = imgStoreMap[storageConfig.RootDirectory]
Expand Down Expand Up @@ -210,7 +210,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig,
// false positive lint - linter does not implement Lint method
//nolint: typecheck
subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory,
storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver,
storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, cfg.HTTP.Compat,
)
}
}
Expand Down
79 changes: 79 additions & 0 deletions test/blackbox/docker-compat.bats
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
# Makefile target installs & checks all necessary tooling
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()

load helpers_zot

function verify_prerequisites {
if [ ! $(command -v curl) ]; then
echo "you need to install curl as a prerequisite to running the tests" >&3
return 1
fi

if [ ! $(command -v jq) ]; then
echo "you need to install jq as a prerequisite to running the tests" >&3
return 1
fi

return 0
}

function setup_file() {
# Verify prerequisites are available
if ! $(verify_prerequisites); then
exit 1
fi
# Download test data to folder common for the entire suite, not just this file
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20
# Setup zot server
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
local oci_data_dir=${BATS_FILE_TMPDIR}/oci
mkdir -p ${zot_root_dir}
mkdir -p ${oci_data_dir}
zot_port=$(get_free_port)
echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port
cat > ${zot_config_file}<<EOF
{
"distSpecVersion": "1.1.0",
"storage": {
"rootDirectory": "${zot_root_dir}"
},
"http": {
"address": "0.0.0.0",
"port": "${zot_port}",
"compat": ["docker2s2"]
},
"log": {
"level": "debug",
"output": "${BATS_FILE_TMPDIR}/zot.log"
}
}
EOF
git -C ${BATS_FILE_TMPDIR} clone https://github.com/project-zot/helm-charts.git
zot_serve ${ZOT_PATH} ${zot_config_file}
wait_zot_reachable ${zot_port}
}

function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot.log
}

function teardown_file() {
zot_stop_all
}

@test "push docker image to compatible zot" {
zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port`
cat > Dockerfile <<EOF
FROM scratch
ECHO "hello world" > /testfile
EOF
docker build -f Dockerfile . -t localhost:${zot_port}/test
run docker push localhost:${zot_port}/test
[ "$status" -eq 0 ]
run docker pull localhost:${zot_port}/test
[ "$status" -eq 0 ]
}

0 comments on commit 39e94e1

Please sign in to comment.