Skip to content

Commit

Permalink
feat: add support for docker images (#2714)
Browse files Browse the repository at this point in the history
* feat: add support for docker images

Issue #724

A new config section under "HTTP" called "Compat" is added which
currently takes a list of possible compatible legacy media-types.

https://github.com/opencontainers/image-spec/blob/main/media-types.md#compatibility-matrix

Only "docker2s2" (Docker Manifest V2 Schema V2) is currently supported.

Garbage collection also needs to be made aware of non-OCI compatible
layer types.
feat: add cve support for non-OCI compatible layer types

Signed-off-by: Ramkumar Chinchani <[email protected]>

* 

Signed-off-by: Ramkumar Chinchani <[email protected]>

* test: add more docker compat tests

Signed-off-by: Ramkumar Chinchani <[email protected]>

* feat: add additional validation checks for non-OCI images

Signed-off-by: Ramkumar Chinchani <[email protected]>

* ci: make "full" images docker-compatible

Signed-off-by: Ramkumar Chinchani <[email protected]>

---------

Signed-off-by: Ramkumar Chinchani <[email protected]>
  • Loading branch information
rchincha authored Oct 31, 2024
1 parent 403fd4e commit cb2af94
Show file tree
Hide file tree
Showing 44 changed files with 436 additions and 191 deletions.
3 changes: 2 additions & 1 deletion build/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ RUN echo '{\n\
},\n\
"http": {\n\
"address": "0.0.0.0",\n\
"port": "5000"\n\
"port": "5000",\n\
"compat": ["docker2s2"]\n\
},\n\
"log": {\n\
"level": "debug"\n\
Expand Down
3 changes: 2 additions & 1 deletion build/stacker.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ build:
},
"http":{
"address":"0.0.0.0",
"port":"5000"
"port":"5000",
"compat": ["docker2s2"]
},
"log":{
"level":"debug"
Expand Down
14 changes: 14 additions & 0 deletions examples/config-docker-compat.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{
"distSpecVersion": "1.1.0",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "8080",
"compat": ["docker2s2"]
},
"log": {
"level": "debug"
}
}
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ require (
github.com/containers/image/v5 v5.32.2
github.com/dchest/siphash v1.2.3
github.com/didip/tollbooth/v7 v7.0.2
github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20240729175644-f0bd0f689923
github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20241007110747-0ab7f326e651
github.com/dustin/go-humanize v1.0.1
github.com/fsnotify/fsnotify v1.7.0
github.com/go-ldap/ldap/v3 v3.4.8
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -605,8 +605,8 @@ github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 h1:lxmTCgmHE1G
github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7/go.mod h1:GvWntX9qiTlOud0WkQ6ewFm0LPy5JUR1Xo0Ngbd1w6Y=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20240729175644-f0bd0f689923 h1:j3L5ly19rMyoy+okWyeMdf9+X8NyU3ZuIje8Hc6B+IA=
github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20240729175644-f0bd0f689923/go.mod h1:3PKHxunRSv4DqZ1Q9HgsocS2/PBIHFfm0ICBKkJQE1g=
github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20241007110747-0ab7f326e651 h1:IznKfIVhvCjhopIrrvaXrTMloRCkfJGmYiDANR6dBwo=
github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20241007110747-0ab7f326e651/go.mod h1:Unn8+BXBntRw4BZHI7UVY9wJ7yGW/xtOspXPxo6hFIs=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
Expand Down
4 changes: 2 additions & 2 deletions pkg/api/authn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -954,7 +954,7 @@ func TestCookiestoreCleanup(t *testing.T) {
err = os.Chtimes(sessionPath, changeTime, changeTime)
So(err, ShouldBeNil)

imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil)
imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil)

storeController := storage.StoreController{
DefaultStore: imgStore,
Expand Down Expand Up @@ -989,7 +989,7 @@ func TestCookiestoreCleanup(t *testing.T) {
err = os.WriteFile(sessionPath, []byte("session"), storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)

imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil)
imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil)

storeController := storage.StoreController{
DefaultStore: imgStore,
Expand Down
4 changes: 3 additions & 1 deletion pkg/api/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

distspec "github.com/opencontainers/distribution-spec/specs-go"

"zotregistry.dev/zot/pkg/compat"
extconf "zotregistry.dev/zot/pkg/extensions/config"
storageConstants "zotregistry.dev/zot/pkg/storage/constants"
)
Expand Down Expand Up @@ -122,7 +123,8 @@ type HTTPConfig struct {
Auth *AuthConfig
AccessControl *AccessControlConfig `mapstructure:"accessControl,omitempty"`
Realm string
Ratelimit *RatelimitConfig `mapstructure:",omitempty"`
Ratelimit *RatelimitConfig `mapstructure:",omitempty"`
Compat []compat.MediaCompatibility `mapstructure:",omitempty"`
}

type SchedulerConfig struct {
Expand Down
2 changes: 1 addition & 1 deletion pkg/api/routes.go
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
}

mediaType := request.Header.Get("Content-Type")
if !storageCommon.IsSupportedMediaType(mediaType) {
if !storageCommon.IsSupportedMediaType(rh.c.Config.HTTP.Compat, mediaType) {
err := apiErr.NewError(apiErr.MANIFEST_INVALID).AddDetail(map[string]string{"mediaType": mediaType})
zcommon.WriteJSON(response, http.StatusUnsupportedMediaType, apiErr.NewErrorList(err))

Expand Down
2 changes: 1 addition & 1 deletion pkg/cli/client/cve_cmd_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ func TestNegativeServerResponse(t *testing.T) {
dir := t.TempDir()

imageStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil)

storeController := storage.StoreController{
DefaultStore: imageStore,
Expand Down
69 changes: 69 additions & 0 deletions pkg/compat/compat.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package compat

import (
dockerList "github.com/distribution/distribution/v3/manifest/manifestlist"
docker "github.com/distribution/distribution/v3/manifest/schema2"
v1 "github.com/opencontainers/image-spec/specs-go/v1"

"zotregistry.dev/zot/errors"
)

// MediaCompatibility determines non-OCI media-compatilibility.
type MediaCompatibility string

const (
DockerManifestV2SchemaV2 = "docker2s2"
)

// docker

func CompatibleManifestMediaTypes() []string {
return []string{docker.MediaTypeManifest}
}

func IsCompatibleManifestMediaType(mediatype string) bool {
for _, mt := range CompatibleManifestMediaTypes() {
if mt == mediatype {
return true
}
}

return false
}

func CompatibleManifestListMediaTypes() []string {
return []string{dockerList.MediaTypeManifestList}
}

func IsCompatibleManifestListMediaType(mediatype string) bool {
for _, mt := range CompatibleManifestListMediaTypes() {
if mt == mediatype {
return true
}
}

return false
}

func Validate(body []byte, mediaType string) ([]v1.Descriptor, error) {
switch mediaType {
case docker.MediaTypeManifest:
var desm docker.DeserializedManifest

if err := desm.UnmarshalJSON(body); err != nil {
return nil, err
}

return desm.References(), nil
case dockerList.MediaTypeManifestList:
var desm dockerList.DeserializedManifestList

if err := desm.UnmarshalJSON(body); err != nil {
return nil, err
}

return desm.References(), nil
}

return nil, errors.ErrMediaTypeNotSupported
}
10 changes: 5 additions & 5 deletions pkg/extensions/extension_image_trust_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
logger.Logger = logger.Output(writers)

imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil)

storeController := storage.StoreController{
DefaultStore: imageStore,
Expand Down Expand Up @@ -321,7 +321,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
logger.Logger = logger.Output(writers)

imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil)

storeController := storage.StoreController{
DefaultStore: imageStore,
Expand Down Expand Up @@ -429,7 +429,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
logger.Logger = logger.Output(writers)

imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil)

storeController := storage.StoreController{
DefaultStore: imageStore,
Expand Down Expand Up @@ -592,7 +592,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
logger.Logger = logger.Output(writers)

imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil)

storeController := storage.StoreController{
DefaultStore: imageStore,
Expand Down Expand Up @@ -856,7 +856,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
logger.Logger = logger.Output(writers)

imageStore := local.NewImageStore(globalDir, false, false,
logger, monitoring.NewMetricsServer(false, logger), nil, nil)
logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil)

storeController := storage.StoreController{
DefaultStore: imageStore,
Expand Down
14 changes: 7 additions & 7 deletions pkg/extensions/lint/lint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -489,7 +489,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {

linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil)

indexContent, err := imgStore.GetIndexContent("zot-test")
So(err, ShouldBeNil)
Expand Down Expand Up @@ -521,7 +521,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {

linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil)

indexContent, err := imgStore.GetIndexContent("zot-test")
So(err, ShouldBeNil)
Expand Down Expand Up @@ -591,7 +591,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {

linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil)

pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldBeNil)
Expand Down Expand Up @@ -653,7 +653,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {

linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil)

pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldNotBeNil)
Expand Down Expand Up @@ -717,7 +717,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {

linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil)

pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldBeNil)
Expand Down Expand Up @@ -780,7 +780,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {

linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil)

err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000)
if err != nil {
Expand Down Expand Up @@ -878,7 +878,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {

linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := local.NewImageStore(dir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil)
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil)

err = os.Chmod(path.Join(dir, "zot-test", "blobs", "sha256", manifest.Config.Digest.Encoded()), 0o000)
if err != nil {
Expand Down
6 changes: 3 additions & 3 deletions pkg/extensions/scrub/scrub_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ func TestRunScrubRepo(t *testing.T) {
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true,
true, log, metrics, nil, cacheDriver)
true, log, metrics, nil, cacheDriver, nil)

srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log)
image := CreateDefaultVulnerableImage()
Expand Down Expand Up @@ -231,7 +231,7 @@ func TestRunScrubRepo(t *testing.T) {
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true,
true, log, metrics, nil, cacheDriver)
true, log, metrics, nil, cacheDriver, nil)

srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log)
image := CreateDefaultVulnerableImage()
Expand Down Expand Up @@ -272,7 +272,7 @@ func TestRunScrubRepo(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil)

srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log)
image := CreateDefaultVulnerableImage()
Expand Down
Loading

0 comments on commit cb2af94

Please sign in to comment.