diff --git a/build/Dockerfile b/build/Dockerfile index b5a19bd61..78a48ee39 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -17,7 +17,8 @@ RUN echo '{\n\ },\n\ "http": {\n\ "address": "0.0.0.0",\n\ - "port": "5000"\n\ + "port": "5000",\n\ + "compat": ["docker2s2"]\n\ },\n\ "log": {\n\ "level": "debug"\n\ diff --git a/build/stacker.yaml b/build/stacker.yaml index 1e14b4211..b1028d4a6 100644 --- a/build/stacker.yaml +++ b/build/stacker.yaml @@ -21,7 +21,8 @@ build: }, "http":{ "address":"0.0.0.0", - "port":"5000" + "port":"5000", + "compat": ["docker2s2"] }, "log":{ "level":"debug" diff --git a/examples/config-docker-compat.json b/examples/config-docker-compat.json new file mode 100644 index 000000000..ff9a19d64 --- /dev/null +++ b/examples/config-docker-compat.json @@ -0,0 +1,14 @@ +{ + "distSpecVersion": "1.1.0", + "storage": { + "rootDirectory": "/tmp/zot" + }, + "http": { + "address": "127.0.0.1", + "port": "8080", + "compat": ["docker2s2"] + }, + "log": { + "level": "debug" + } +} diff --git a/go.mod b/go.mod index 5602413fc..59e632d1e 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/containers/image/v5 v5.32.2 github.com/dchest/siphash v1.2.3 github.com/didip/tollbooth/v7 v7.0.2 - github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20240729175644-f0bd0f689923 + github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20241007110747-0ab7f326e651 github.com/dustin/go-humanize v1.0.1 github.com/fsnotify/fsnotify v1.7.0 github.com/go-ldap/ldap/v3 v3.4.8 diff --git a/go.sum b/go.sum index 6e57c10a2..9d03ae001 100644 --- a/go.sum +++ b/go.sum @@ -605,8 +605,8 @@ github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 h1:lxmTCgmHE1G github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7/go.mod h1:GvWntX9qiTlOud0WkQ6ewFm0LPy5JUR1Xo0Ngbd1w6Y= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20240729175644-f0bd0f689923 h1:j3L5ly19rMyoy+okWyeMdf9+X8NyU3ZuIje8Hc6B+IA= -github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20240729175644-f0bd0f689923/go.mod h1:3PKHxunRSv4DqZ1Q9HgsocS2/PBIHFfm0ICBKkJQE1g= +github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20241007110747-0ab7f326e651 h1:IznKfIVhvCjhopIrrvaXrTMloRCkfJGmYiDANR6dBwo= +github.com/distribution/distribution/v3 v3.0.0-beta.1.0.20241007110747-0ab7f326e651/go.mod h1:Unn8+BXBntRw4BZHI7UVY9wJ7yGW/xtOspXPxo6hFIs= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= diff --git a/pkg/api/authn_test.go b/pkg/api/authn_test.go index a05fd726c..e5e29f6c8 100644 --- a/pkg/api/authn_test.go +++ b/pkg/api/authn_test.go @@ -954,7 +954,7 @@ func TestCookiestoreCleanup(t *testing.T) { err = os.Chtimes(sessionPath, changeTime, changeTime) So(err, ShouldBeNil) - imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imgStore, @@ -989,7 +989,7 @@ func TestCookiestoreCleanup(t *testing.T) { err = os.WriteFile(sessionPath, []byte("session"), storageConstants.DefaultFilePerms) So(err, ShouldBeNil) - imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imgStore, diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index b7d5c7f71..07929c21f 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -7,6 +7,7 @@ import ( distspec "github.com/opencontainers/distribution-spec/specs-go" + "zotregistry.dev/zot/pkg/compat" extconf "zotregistry.dev/zot/pkg/extensions/config" storageConstants "zotregistry.dev/zot/pkg/storage/constants" ) @@ -122,7 +123,8 @@ type HTTPConfig struct { Auth *AuthConfig AccessControl *AccessControlConfig `mapstructure:"accessControl,omitempty"` Realm string - Ratelimit *RatelimitConfig `mapstructure:",omitempty"` + Ratelimit *RatelimitConfig `mapstructure:",omitempty"` + Compat []compat.MediaCompatibility `mapstructure:",omitempty"` } type SchedulerConfig struct { diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 97774058f..164b6bf9a 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -688,7 +688,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht } mediaType := request.Header.Get("Content-Type") - if !storageCommon.IsSupportedMediaType(mediaType) { + if !storageCommon.IsSupportedMediaType(rh.c.Config.HTTP.Compat, mediaType) { err := apiErr.NewError(apiErr.MANIFEST_INVALID).AddDetail(map[string]string{"mediaType": mediaType}) zcommon.WriteJSON(response, http.StatusUnsupportedMediaType, apiErr.NewErrorList(err)) diff --git a/pkg/cli/client/cve_cmd_test.go b/pkg/cli/client/cve_cmd_test.go index 8d05d1cf8..addfff951 100644 --- a/pkg/cli/client/cve_cmd_test.go +++ b/pkg/cli/client/cve_cmd_test.go @@ -119,7 +119,7 @@ func TestNegativeServerResponse(t *testing.T) { dir := t.TempDir() imageStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, diff --git a/pkg/compat/compat.go b/pkg/compat/compat.go new file mode 100644 index 000000000..a2b61f0e0 --- /dev/null +++ b/pkg/compat/compat.go @@ -0,0 +1,69 @@ +package compat + +import ( + dockerList "github.com/distribution/distribution/v3/manifest/manifestlist" + docker "github.com/distribution/distribution/v3/manifest/schema2" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + + "zotregistry.dev/zot/errors" +) + +// MediaCompatibility determines non-OCI media-compatilibility. +type MediaCompatibility string + +const ( + DockerManifestV2SchemaV2 = "docker2s2" +) + +// docker + +func CompatibleManifestMediaTypes() []string { + return []string{docker.MediaTypeManifest} +} + +func IsCompatibleManifestMediaType(mediatype string) bool { + for _, mt := range CompatibleManifestMediaTypes() { + if mt == mediatype { + return true + } + } + + return false +} + +func CompatibleManifestListMediaTypes() []string { + return []string{dockerList.MediaTypeManifestList} +} + +func IsCompatibleManifestListMediaType(mediatype string) bool { + for _, mt := range CompatibleManifestListMediaTypes() { + if mt == mediatype { + return true + } + } + + return false +} + +func Validate(body []byte, mediaType string) ([]v1.Descriptor, error) { + switch mediaType { + case docker.MediaTypeManifest: + var desm docker.DeserializedManifest + + if err := desm.UnmarshalJSON(body); err != nil { + return nil, err + } + + return desm.References(), nil + case dockerList.MediaTypeManifestList: + var desm dockerList.DeserializedManifestList + + if err := desm.UnmarshalJSON(body); err != nil { + return nil, err + } + + return desm.References(), nil + } + + return nil, errors.ErrMediaTypeNotSupported +} diff --git a/pkg/extensions/extension_image_trust_test.go b/pkg/extensions/extension_image_trust_test.go index bbdb3f76c..a7de46bc0 100644 --- a/pkg/extensions/extension_image_trust_test.go +++ b/pkg/extensions/extension_image_trust_test.go @@ -200,7 +200,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -321,7 +321,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -429,7 +429,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -592,7 +592,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -856,7 +856,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[ logger.Logger = logger.Output(writers) imageStore := local.NewImageStore(globalDir, false, false, - logger, monitoring.NewMetricsServer(false, logger), nil, nil) + logger, monitoring.NewMetricsServer(false, logger), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, diff --git a/pkg/extensions/lint/lint_test.go b/pkg/extensions/lint/lint_test.go index 7187ffab1..d29ed76bc 100644 --- a/pkg/extensions/lint/lint_test.go +++ b/pkg/extensions/lint/lint_test.go @@ -489,7 +489,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) indexContent, err := imgStore.GetIndexContent("zot-test") So(err, ShouldBeNil) @@ -521,7 +521,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) indexContent, err := imgStore.GetIndexContent("zot-test") So(err, ShouldBeNil) @@ -591,7 +591,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) So(err, ShouldBeNil) @@ -653,7 +653,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) So(err, ShouldNotBeNil) @@ -717,7 +717,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) So(err, ShouldBeNil) @@ -780,7 +780,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000) if err != nil { @@ -878,7 +878,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) imgStore := local.NewImageStore(dir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil, nil) err = os.Chmod(path.Join(dir, "zot-test", "blobs", "sha256", manifest.Config.Digest.Encoded()), 0o000) if err != nil { diff --git a/pkg/extensions/scrub/scrub_test.go b/pkg/extensions/scrub/scrub_test.go index 3406ea50d..90f87e2cd 100644 --- a/pkg/extensions/scrub/scrub_test.go +++ b/pkg/extensions/scrub/scrub_test.go @@ -195,7 +195,7 @@ func TestRunScrubRepo(t *testing.T) { UseRelPaths: true, }, log) imgStore := local.NewImageStore(dir, true, - true, log, metrics, nil, cacheDriver) + true, log, metrics, nil, cacheDriver, nil) srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log) image := CreateDefaultVulnerableImage() @@ -231,7 +231,7 @@ func TestRunScrubRepo(t *testing.T) { UseRelPaths: true, }, log) imgStore := local.NewImageStore(dir, true, - true, log, metrics, nil, cacheDriver) + true, log, metrics, nil, cacheDriver, nil) srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log) image := CreateDefaultVulnerableImage() @@ -272,7 +272,7 @@ func TestRunScrubRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) srcStorageCtlr := ociutils.GetDefaultStoreController(dir, log) image := CreateDefaultVulnerableImage() diff --git a/pkg/extensions/search/cve/cve.go b/pkg/extensions/search/cve/cve.go index 7a7c0bfac..f897e4949 100644 --- a/pkg/extensions/search/cve/cve.go +++ b/pkg/extensions/search/cve/cve.go @@ -11,6 +11,7 @@ import ( zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/compat" cvemodel "zotregistry.dev/zot/pkg/extensions/search/cve/model" "zotregistry.dev/zot/pkg/extensions/search/cve/trivy" "zotregistry.dev/zot/pkg/log" @@ -69,8 +70,10 @@ func (cveinfo BaseCveInfo) GetImageListForCVE(ctx context.Context, repo, cveID s } for tag, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest, ispec.MediaTypeImageIndex: + if descriptor.MediaType == ispec.MediaTypeImageManifest || + descriptor.MediaType == ispec.MediaTypeImageIndex || + compat.IsCompatibleManifestMediaType(descriptor.MediaType) || + compat.IsCompatibleManifestListMediaType(descriptor.MediaType) { manifestDigestStr := descriptor.Digest manifestDigest := godigest.Digest(manifestDigestStr) @@ -102,7 +105,7 @@ func (cveinfo BaseCveInfo) GetImageListForCVE(ctx context.Context, repo, cveID s }, }) } - default: + } else { cveinfo.Log.Debug().Str("image", repo+":"+tag).Str("mediaType", descriptor.MediaType). Msg("image media type not supported for scanning") } @@ -129,8 +132,9 @@ func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(ctx context.Context, repo, c return []cvemodel.TagInfo{}, ctx.Err() } - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: + //nolint:gocritic // cannot convert to switch-case + if descriptor.MediaType == ispec.MediaTypeImageManifest || + compat.IsCompatibleManifestMediaType(descriptor.MediaType) { manifestDigestStr := descriptor.Digest tagInfo, err := getTagInfoForManifest(tag, manifestDigestStr, cveinfo.MetaDB) @@ -146,7 +150,8 @@ func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(ctx context.Context, repo, c if cveinfo.isManifestVulnerable(ctx, repo, tag, manifestDigestStr, cveID) { vulnerableTags = append(vulnerableTags, tagInfo) } - case ispec.MediaTypeImageIndex: + } else if descriptor.MediaType == ispec.MediaTypeImageIndex || + compat.IsCompatibleManifestListMediaType(descriptor.MediaType) { indexDigestStr := descriptor.Digest indexContent, err := getIndexContent(cveinfo.MetaDB, indexDigestStr) @@ -201,7 +206,7 @@ func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(ctx context.Context, repo, c Timestamp: mostRecentUpdate(vulnerableManifests), }) } - default: + } else { cveinfo.Log.Debug().Str("mediaType", descriptor.MediaType). Msg("image media type not supported for scanning") } @@ -533,18 +538,20 @@ func GetFixedTags(allTags, vulnerableTags []cvemodel.TagInfo) []cvemodel.TagInfo for _, tag := range vulnerableTags { vulnerableTagMap[tag.Tag] = tag - switch tag.Descriptor.MediaType { - case ispec.MediaTypeImageManifest: + //nolint:gocritic // cannot convert to switch-case + if tag.Descriptor.MediaType == ispec.MediaTypeImageManifest || + compat.IsCompatibleManifestMediaType(tag.Descriptor.MediaType) { if tag.Timestamp.Before(earliestVulnerable.Timestamp) { earliestVulnerable = tag } - case ispec.MediaTypeImageIndex: + } else if tag.Descriptor.MediaType == ispec.MediaTypeImageIndex || + compat.IsCompatibleManifestListMediaType(tag.Descriptor.MediaType) { for _, manifestDesc := range tag.Manifests { if manifestDesc.Timestamp.Before(earliestVulnerable.Timestamp) { earliestVulnerable = tag } } - default: + } else { continue } } @@ -557,8 +564,9 @@ func GetFixedTags(allTags, vulnerableTags []cvemodel.TagInfo) []cvemodel.TagInfo // There may be older images which have a fix or // newer images which don't for _, tag := range allTags { - switch tag.Descriptor.MediaType { - case ispec.MediaTypeImageManifest: + //nolint:gocritic // cannot convert to switch-case + if tag.Descriptor.MediaType == ispec.MediaTypeImageManifest || + compat.IsCompatibleManifestMediaType(tag.Descriptor.MediaType) { if tag.Timestamp.Before(earliestVulnerable.Timestamp) { // The vulnerability did not exist at the time this // image was built @@ -570,7 +578,8 @@ func GetFixedTags(allTags, vulnerableTags []cvemodel.TagInfo) []cvemodel.TagInfo if _, ok := vulnerableTagMap[tag.Tag]; !ok { fixedTags = append(fixedTags, tag) } - case ispec.MediaTypeImageIndex: + } else if tag.Descriptor.MediaType == ispec.MediaTypeImageIndex || + compat.IsCompatibleManifestListMediaType(tag.Descriptor.MediaType) { fixedManifests := []cvemodel.DescriptorInfo{} // If the latest update inside the index is before the earliest vulnerability found then @@ -599,7 +608,7 @@ func GetFixedTags(allTags, vulnerableTags []cvemodel.TagInfo) []cvemodel.TagInfo fixedTags = append(fixedTags, fixedTag) } - default: + } else { continue } } diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 9022ec796..d74c6cdc5 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -319,7 +319,7 @@ func TestImageFormat(t *testing.T) { dbDir := t.TempDir() metrics := monitoring.NewMetricsServer(false, log) - defaultStore := local.NewImageStore(imgDir, false, false, log, metrics, nil, nil) + defaultStore := local.NewImageStore(imgDir, false, false, log, metrics, nil, nil, nil) storeController := storage.StoreController{DefaultStore: defaultStore} params := boltdb.DBParameters{ diff --git a/pkg/extensions/search/cve/scan_test.go b/pkg/extensions/search/cve/scan_test.go index 363a2e7f5..d8e53624f 100644 --- a/pkg/extensions/search/cve/scan_test.go +++ b/pkg/extensions/search/cve/scan_test.go @@ -505,7 +505,7 @@ func TestScanGeneratorWithRealData(t *testing.T) { metrics := monitoring.NewMetricsServer(true, logger) imageStore := local.NewImageStore(rootDir, false, false, - logger, metrics, nil, nil) + logger, metrics, nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} image := CreateRandomVulnerableImage() diff --git a/pkg/extensions/search/cve/trivy/scanner.go b/pkg/extensions/search/cve/trivy/scanner.go index 42c9e92b9..496dd0a91 100644 --- a/pkg/extensions/search/cve/trivy/scanner.go +++ b/pkg/extensions/search/cve/trivy/scanner.go @@ -26,6 +26,7 @@ import ( zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/compat" cvecache "zotregistry.dev/zot/pkg/extensions/search/cve/cache" cvemodel "zotregistry.dev/zot/pkg/extensions/search/cve/model" "zotregistry.dev/zot/pkg/log" @@ -243,22 +244,23 @@ func (scanner Scanner) IsImageFormatScannable(repo, ref string) (bool, error) { func (scanner Scanner) IsImageMediaScannable(repo, digestStr, mediaType string) (bool, error) { image := repo + "@" + digestStr - switch mediaType { - case ispec.MediaTypeImageManifest: + if mediaType == ispec.MediaTypeImageManifest || //nolint:gocritic // not converting to switch-case + compat.IsCompatibleManifestMediaType(mediaType) { ok, err := scanner.isManifestScanable(digestStr) if err != nil { return ok, fmt.Errorf("image '%s' %w", image, err) } return ok, nil - case ispec.MediaTypeImageIndex: + } else if mediaType == ispec.MediaTypeImageIndex || + compat.IsCompatibleManifestListMediaType(mediaType) { ok, err := scanner.isIndexScannable(digestStr) if err != nil { return ok, fmt.Errorf("image '%s' %w", image, err) } return ok, nil - default: + } else { return false, nil } } @@ -379,10 +381,11 @@ func (scanner Scanner) ScanImage(ctx context.Context, image string) (map[string] err error ) - switch mediaType { - case ispec.MediaTypeImageIndex: + if mediaType == ispec.MediaTypeImageIndex || + compat.IsCompatibleManifestListMediaType(mediaType) { cveIDMap, err = scanner.scanIndex(ctx, repo, digest) - default: + } else if mediaType == ispec.MediaTypeImageManifest || + compat.IsCompatibleManifestMediaType(mediaType) { cveIDMap, err = scanner.scanManifest(ctx, repo, digest) } diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 00c94a431..c201d88b3 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -54,11 +54,11 @@ func TestMultipleStoragePath(t *testing.T) { // Create ImageStore - firstStore := local.NewImageStore(firstRootDir, false, false, log, metrics, nil, nil) + firstStore := local.NewImageStore(firstRootDir, false, false, log, metrics, nil, nil, nil) - secondStore := local.NewImageStore(secondRootDir, false, false, log, metrics, nil, nil) + secondStore := local.NewImageStore(secondRootDir, false, false, log, metrics, nil, nil, nil) - thirdStore := local.NewImageStore(thirdRootDir, false, false, log, metrics, nil, nil) + thirdStore := local.NewImageStore(thirdRootDir, false, false, log, metrics, nil, nil, nil) storeController := storage.StoreController{} @@ -172,7 +172,7 @@ func TestTrivyLibraryErrors(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -313,7 +313,7 @@ func TestImageScannable(t *testing.T) { // Continue with initializing the objects the scanner depends on metrics := monitoring.NewMetricsServer(false, log) - store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -367,7 +367,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store diff --git a/pkg/extensions/search/cve/trivy/scanner_test.go b/pkg/extensions/search/cve/trivy/scanner_test.go index 3597a0dc7..a280f5b08 100644 --- a/pkg/extensions/search/cve/trivy/scanner_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_test.go @@ -168,7 +168,7 @@ func TestVulnerableLayer(t *testing.T) { log := log.NewLogger("debug", "") imageStore := local.NewImageStore(tempDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -239,7 +239,7 @@ func TestVulnerableLayer(t *testing.T) { log := log.NewLogger("debug", "") imageStore := local.NewImageStore(tempDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index b7f835cf4..234c6300c 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -1158,7 +1158,7 @@ func TestExpandedRepoInfo(t *testing.T) { ctlr := api.NewController(conf) imageStore := local.NewImageStore(tempDir, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -1280,7 +1280,7 @@ func TestExpandedRepoInfo(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - testStorage := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + testStorage := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) @@ -1636,7 +1636,7 @@ func TestExpandedRepoInfo(t *testing.T) { ctlr := api.NewController(conf) imageStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -5774,7 +5774,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil) + storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil, nil) indexBlob, err := storage.GetIndexContent(repo) So(err, ShouldBeNil) @@ -5848,7 +5848,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil) + storage := local.NewImageStore(dir, false, false, log, metrics, nil, nil, nil) indexBlob, err := storage.GetIndexContent(repo) So(err, ShouldBeNil) diff --git a/pkg/extensions/sync/destination.go b/pkg/extensions/sync/destination.go index 3384e6270..b93eec625 100644 --- a/pkg/extensions/sync/destination.go +++ b/pkg/extensions/sync/destination.go @@ -304,5 +304,5 @@ func getTempRootDirFromImageReference(imageReference types.ImageReference, repo, func getImageStore(rootDir string, log log.Logger) storageTypes.ImageStore { metrics := monitoring.NewMetricsServer(false, log) - return local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + return local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) } diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 5a03ae0fa..3609cb4a1 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -72,7 +72,7 @@ func TestInjectSyncUtils(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(t.TempDir(), false, false, log, metrics, nil, nil) + imageStore := local.NewImageStore(t.TempDir(), false, false, log, metrics, nil, nil, nil) injected = inject.InjectFailure(0) ols := NewOciLayoutStorage(storage.StoreController{DefaultStore: imageStore}) @@ -216,7 +216,7 @@ func TestDestinationRegistry(t *testing.T) { UseRelPaths: true, }, log) - syncImgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + syncImgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := "repo" storeController := storage.StoreController{DefaultStore: syncImgStore} @@ -334,7 +334,7 @@ func TestDestinationRegistry(t *testing.T) { MandatoryAnnotations: []string{"annot1"}, }, log) - syncImgStore := local.NewImageStore(dir, true, true, log, metrics, linter, cacheDriver) + syncImgStore := local.NewImageStore(dir, true, true, log, metrics, linter, cacheDriver, nil) repoName := "repo" storeController := storage.StoreController{DefaultStore: syncImgStore} diff --git a/pkg/meta/hooks_test.go b/pkg/meta/hooks_test.go index 03d742071..48e8b0216 100644 --- a/pkg/meta/hooks_test.go +++ b/pkg/meta/hooks_test.go @@ -26,7 +26,7 @@ func TestOnUpdateManifest(t *testing.T) { storeController := storage.StoreController{} log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storeController.DefaultStore = local.NewImageStore(rootDir, true, true, log, metrics, nil, nil) + storeController.DefaultStore = local.NewImageStore(rootDir, true, true, log, metrics, nil, nil, nil) params := boltdb.DBParameters{ RootDir: rootDir, diff --git a/pkg/meta/parse.go b/pkg/meta/parse.go index ab671c008..056b0e1e3 100644 --- a/pkg/meta/parse.go +++ b/pkg/meta/parse.go @@ -11,6 +11,7 @@ import ( zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/meta/convert" mTypes "zotregistry.dev/zot/pkg/meta/types" @@ -309,8 +310,7 @@ func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType strin ) error { var imageMeta mTypes.ImageMeta - switch mediaType { - case ispec.MediaTypeImageManifest: + if mediaType == ispec.MediaTypeImageManifest || compat.IsCompatibleManifestMediaType(mediaType) { //nolint:gocritic,lll // mixing checking mechanisms manifestContent := ispec.Manifest{} configContent := ispec.Image{} @@ -367,7 +367,7 @@ func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType strin } imageMeta = convert.GetImageManifestMeta(manifestContent, configContent, int64(len(blob)), digest) - case ispec.MediaTypeImageIndex: + } else if mediaType == ispec.MediaTypeImageIndex || compat.IsCompatibleManifestListMediaType(mediaType) { indexContent := ispec.Index{} err := json.Unmarshal(blob, &indexContent) @@ -376,7 +376,7 @@ func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType strin } imageMeta = convert.GetImageIndexMeta(indexContent, int64(len(blob)), digest) - default: + } else { return nil } diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index 09bb40ee9..9fc0c64f8 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -344,7 +344,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) Convey("Test with simple case", func() { imageStore := local.NewImageStore(rootDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} manifests := []ispec.Manifest{} @@ -419,7 +419,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) Convey("Accept orphan signatures", func() { imageStore := local.NewImageStore(rootDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} @@ -464,7 +464,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) Convey("Check statistics after load", func() { imageStore := local.NewImageStore(rootDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} // add an image @@ -505,7 +505,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) // make sure pushTimestamp is always populated to not interfere with retention logic Convey("Always update pushTimestamp if its value is 0(time.Time{})", func() { imageStore := local.NewImageStore(rootDir, false, false, - log, monitoring.NewMetricsServer(false, log), nil, nil) + log, monitoring.NewMetricsServer(false, log), nil, nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} // add an image diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index 022e9680d..67bd4de92 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -11,6 +11,8 @@ import ( "strings" "time" + dockerList "github.com/distribution/distribution/v3/manifest/manifestlist" + docker "github.com/distribution/distribution/v3/manifest/schema2" "github.com/distribution/distribution/v3/registry/storage/driver" godigest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/schema" @@ -19,6 +21,7 @@ import ( zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/scheduler" @@ -62,10 +65,10 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc } func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte, - log zlog.Logger, + compats []compat.MediaCompatibility, log zlog.Logger, ) error { // validate the manifest - if !IsSupportedMediaType(mediaType) { + if !IsSupportedMediaType(compats, mediaType) { log.Debug().Interface("actual", mediaType). Msg("bad manifest media type") @@ -145,6 +148,23 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy log.Error().Err(err).Str("digest", manifest.Digest.String()). Msg("failed to stat manifest due to missing manifest blob") + return zerr.ErrBadManifest + } + } + default: + // non-OCI compatible + descriptors, err := compat.Validate(body, mediaType) + if err != nil { + log.Error().Err(err).Msg("failed to unmarshal JSON") + + return zerr.ErrBadManifest + } + + for _, desc := range descriptors { + if ok, _, _, err := imgStore.StatBlob(repo, desc.Digest); !ok || err != nil { + log.Error().Err(err).Str("digest", desc.Digest.String()). + Msg("failed to stat non-OCI descriptor due to missing blob") + return zerr.ErrBadManifest } } @@ -796,7 +816,15 @@ func getBlobDescriptorFromManifest(imgStore storageTypes.ImageStore, repo string return ispec.Descriptor{}, zerr.ErrBlobNotFound } -func IsSupportedMediaType(mediaType string) bool { +func IsSupportedMediaType(compats []compat.MediaCompatibility, mediaType string) bool { + // check for some supported legacy formats if configured + for _, comp := range compats { + if comp == compat.DockerManifestV2SchemaV2 && + (mediaType == docker.MediaTypeManifest || mediaType == dockerList.MediaTypeManifestList) { + return true + } + } + return mediaType == ispec.MediaTypeImageIndex || mediaType == ispec.MediaTypeImageManifest } diff --git a/pkg/storage/common/common_test.go b/pkg/storage/common/common_test.go index c618d7627..b6a575147 100644 --- a/pkg/storage/common/common_test.go +++ b/pkg/storage/common/common_test.go @@ -37,7 +37,7 @@ func TestValidateManifest(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) content := []byte("this is a blob") digest := godigest.FromBytes(content) @@ -180,7 +180,7 @@ func TestGetReferrersErrors(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, false, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, false, true, log, metrics, nil, cacheDriver, nil) artifactType := "application/vnd.example.icecream.v1" validDigest := godigest.FromBytes([]byte("blob")) @@ -401,7 +401,7 @@ func TestGetBlobDescriptorFromRepo(t *testing.T) { driver := local.New(true) imgStore := imagestore.NewImageStore(tdir, tdir, true, - true, log, metrics, nil, driver, cacheDriver) + true, log, metrics, nil, driver, cacheDriver, nil) repoName := "zot-test" diff --git a/pkg/storage/gc/gc.go b/pkg/storage/gc/gc.go index 4c7300348..a28125c6f 100644 --- a/pkg/storage/gc/gc.go +++ b/pkg/storage/gc/gc.go @@ -17,6 +17,7 @@ import ( zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/api/config" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/compat" zlog "zotregistry.dev/zot/pkg/log" mTypes "zotregistry.dev/zot/pkg/meta/types" "zotregistry.dev/zot/pkg/retention" @@ -219,8 +220,7 @@ func (gc GarbageCollect) removeIndexReferrers(repo string, rootIndex *ispec.Inde var err error for _, desc := range index.Manifests { - switch desc.MediaType { - case ispec.MediaTypeImageIndex: + if (desc.MediaType == ispec.MediaTypeImageIndex) || compat.IsCompatibleManifestListMediaType(desc.MediaType) { indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log) if err != nil { gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", desc.Digest.String()). @@ -249,7 +249,7 @@ func (gc GarbageCollect) removeIndexReferrers(repo string, rootIndex *ispec.Inde if gced { count++ } - case ispec.MediaTypeImageManifest: + } else if (desc.MediaType == ispec.MediaTypeImageManifest) || compat.IsCompatibleManifestMediaType(desc.MediaType) { image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log) if err != nil { gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo).Str("digest", desc.Digest.String()). @@ -513,8 +513,7 @@ func (gc GarbageCollect) identifyManifestsReferencedInIndex(index ispec.Index, r referenced map[godigest.Digest]bool, ) error { for _, desc := range index.Manifests { - switch desc.MediaType { - case ispec.MediaTypeImageIndex: + if (desc.MediaType == ispec.MediaTypeImageIndex) || compat.IsCompatibleManifestListMediaType(desc.MediaType) { indexImage, err := common.GetImageIndex(gc.imgStore, repo, desc.Digest, gc.log) if err != nil { gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). @@ -534,7 +533,7 @@ func (gc GarbageCollect) identifyManifestsReferencedInIndex(index ispec.Index, r if err := gc.identifyManifestsReferencedInIndex(indexImage, repo, referenced); err != nil { return err } - case ispec.MediaTypeImageManifest: + } else if (desc.MediaType == ispec.MediaTypeImageManifest) || compat.IsCompatibleManifestMediaType(desc.MediaType) { image, err := common.GetImageManifest(gc.imgStore, repo, desc.Digest, gc.log) if err != nil { gc.log.Error().Err(err).Str("module", "gc").Str("repo", repo). @@ -675,15 +674,14 @@ func (gc GarbageCollect) removeUnreferencedBlobs(repo string, delay time.Duratio func (gc GarbageCollect) addIndexBlobsToReferences(repo string, index ispec.Index, refBlobs map[string]bool, ) error { for _, desc := range index.Manifests { - switch desc.MediaType { - case ispec.MediaTypeImageIndex: + if (desc.MediaType == ispec.MediaTypeImageIndex) || compat.IsCompatibleManifestListMediaType(desc.MediaType) { if err := gc.addImageIndexBlobsToReferences(repo, desc.Digest, refBlobs); err != nil { gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). Str("digest", desc.Digest.String()).Msg("failed to read blobs in multiarch(index) image") return err } - case ispec.MediaTypeImageManifest: + } else if (desc.MediaType == ispec.MediaTypeImageManifest) || compat.IsCompatibleManifestMediaType(desc.MediaType) { if err := gc.addImageManifestBlobsToReferences(repo, desc.Digest, refBlobs); err != nil { gc.log.Error().Err(err).Str("module", "gc").Str("repository", repo). Str("digest", desc.Digest.String()).Msg("failed to read blobs in image manifest") diff --git a/pkg/storage/gc/gc_internal_test.go b/pkg/storage/gc/gc_internal_test.go index 3bd40287c..59ef0d76c 100644 --- a/pkg/storage/gc/gc_internal_test.go +++ b/pkg/storage/gc/gc_internal_test.go @@ -47,7 +47,7 @@ func TestGarbageCollectManifestErrors(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ Delay: storageConstants.DefaultGCDelay, @@ -171,7 +171,7 @@ func TestGarbageCollectIndexErrors(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) gc := NewGarbageCollect(imgStore, mocks.MetaDBMock{}, Options{ Delay: storageConstants.DefaultGCDelay, diff --git a/pkg/storage/gc/gc_test.go b/pkg/storage/gc/gc_test.go index f8e73a2a0..bcec4a674 100644 --- a/pkg/storage/gc/gc_test.go +++ b/pkg/storage/gc/gc_test.go @@ -140,13 +140,13 @@ func TestGarbageCollectAndRetention(t *testing.T) { panic(err) } - imgStore = s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, nil) + imgStore = s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, nil, nil) } else { // Create temporary directory rootDir := t.TempDir() // Create ImageStore - imgStore = local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + imgStore = local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) // init metaDB params := boltdb.DBParameters{ @@ -1105,7 +1105,7 @@ func TestGarbageCollectDeletion(t *testing.T) { rootDir := t.TempDir() // Create ImageStore - imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil) + imgStore := local.NewImageStore(rootDir, false, false, log, metrics, nil, nil, nil) // init metaDB params := boltdb.DBParameters{ diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index 8fe1846f4..7664aeb73 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -21,6 +21,7 @@ import ( zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" syncConstants "zotregistry.dev/zot/pkg/extensions/sync/constants" zlog "zotregistry.dev/zot/pkg/log" @@ -49,6 +50,7 @@ type ImageStore struct { dedupe bool linter common.Lint commit bool + compat []compat.MediaCompatibility } func (is *ImageStore) Name() string { @@ -67,7 +69,8 @@ func (is *ImageStore) DirExists(d string) bool { // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, - metrics monitoring.MetricServer, linter common.Lint, storeDriver storageTypes.Driver, cacheDriver cache.Cache, + metrics monitoring.MetricServer, linter common.Lint, storeDriver storageTypes.Driver, + cacheDriver cache.Cache, compat []compat.MediaCompatibility, ) storageTypes.ImageStore { if err := storeDriver.EnsureDir(rootDir); err != nil { log.Error().Err(err).Str("rootDir", rootDir).Msg("failed to create root dir") @@ -85,6 +88,7 @@ func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlo linter: linter, commit: commit, cache: cacheDriver, + compat: compat, } return imgStore @@ -490,7 +494,7 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli refIsDigest = false } - err = common.ValidateManifest(is, repo, reference, mediaType, body, is.log) + err = common.ValidateManifest(is, repo, reference, mediaType, body, is.compat, is.log) if err != nil { return mDigest, "", err } diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index ecf1ffd19..3309415c2 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -1,6 +1,7 @@ package local import ( + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/storage/cache" @@ -13,6 +14,7 @@ import ( // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache, + compat []compat.MediaCompatibility, ) storageTypes.ImageStore { return imagestore.NewImageStore( rootDir, @@ -24,5 +26,6 @@ func NewImageStore(rootDir string, dedupe, commit bool, log zlog.Logger, linter, New(commit), cacheDriver, + compat, ) } diff --git a/pkg/storage/local/local_elevated_test.go b/pkg/storage/local/local_elevated_test.go index 0ea8ffb2a..9c75ac617 100644 --- a/pkg/storage/local/local_elevated_test.go +++ b/pkg/storage/local/local_elevated_test.go @@ -35,7 +35,7 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) upload, err := imgStore.NewBlobUpload("dedupe1") So(err, ShouldBeNil) diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index 610b68045..e38c98484 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -83,7 +83,7 @@ func TestStorageFSAPIs(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) Convey("Repo layout", t, func(c C) { Convey("Bad image manifest", func() { @@ -217,7 +217,7 @@ func FuzzNewBlobUpload(f *testing.F) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) _, err := imgStore.NewBlobUpload(data) if err != nil { @@ -244,7 +244,7 @@ func FuzzPutBlobChunk(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := data @@ -280,7 +280,7 @@ func FuzzPutBlobChunkStreamed(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := data @@ -314,7 +314,7 @@ func FuzzGetBlobUpload(f *testing.F) { UseRelPaths: true, }, log) imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, - cacheDriver) + cacheDriver, nil) _, err := imgStore.GetBlobUpload(data1, data2) if err != nil { @@ -340,7 +340,7 @@ func FuzzTestPutGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) cblob, cdigest := GetRandomImageConfig() @@ -396,7 +396,7 @@ func FuzzTestPutDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) cblob, cdigest := GetRandomImageConfig() @@ -457,7 +457,7 @@ func FuzzTestDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) digest, _, err := newRandomBlobForFuzz(data) if err != nil { @@ -494,7 +494,7 @@ func FuzzInitRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) err := imgStore.InitRepo(data) if err != nil { @@ -520,7 +520,7 @@ func FuzzInitValidateRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) err := imgStore.InitRepo(data) if err != nil { @@ -555,7 +555,7 @@ func FuzzGetImageTags(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) _, err := imgStore.GetImageTags(data) if err != nil { @@ -581,7 +581,7 @@ func FuzzBlobUploadPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) _ = imgStore.BlobUploadPath(repo, uuid) }) @@ -600,7 +600,7 @@ func FuzzBlobUploadInfo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) repo := data _, err := imgStore.BlobUploadInfo(repo, uuid) @@ -626,7 +626,7 @@ func FuzzTestGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := data @@ -655,7 +655,7 @@ func FuzzFinishBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := data @@ -707,7 +707,7 @@ func FuzzFullBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) ldigest, lblob, err := newRandomBlobForFuzz(data) if err != nil { @@ -750,7 +750,7 @@ func TestStorageCacheErrors(t *testing.T) { GetBlobFn: func(digest godigest.Digest) (string, error) { return getBlobPath, nil }, - }) + }, nil) err := imgStore.InitRepo(originRepo) So(err, ShouldBeNil) @@ -780,7 +780,7 @@ func FuzzDedupeBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) blobDigest := godigest.FromString(data) @@ -821,7 +821,7 @@ func FuzzDeleteBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) uuid, err := imgStore.NewBlobUpload(repoName) if err != nil { @@ -853,7 +853,7 @@ func FuzzBlobPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) digest := godigest.FromString(data) _ = imgStore.BlobPath(repoName, digest) @@ -874,7 +874,7 @@ func FuzzCheckBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -907,7 +907,7 @@ func FuzzGetBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -948,7 +948,7 @@ func FuzzDeleteBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -985,7 +985,7 @@ func FuzzGetIndexContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1022,7 +1022,7 @@ func FuzzGetBlobContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, *log, metrics, nil, cacheDriver, nil) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1060,7 +1060,7 @@ func FuzzRunGCRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ Delay: storageConstants.DefaultGCDelay, @@ -1104,9 +1104,9 @@ func TestDedupeLinks(t *testing.T) { var imgStore storageTypes.ImageStore if testCase.dedupe { - imgStore = local.NewImageStore(dir, testCase.dedupe, true, log, metrics, nil, cacheDriver) + imgStore = local.NewImageStore(dir, testCase.dedupe, true, log, metrics, nil, cacheDriver, nil) } else { - imgStore = local.NewImageStore(dir, testCase.dedupe, true, log, metrics, nil, nil) + imgStore = local.NewImageStore(dir, testCase.dedupe, true, log, metrics, nil, nil, nil) } // run on empty image store @@ -1282,7 +1282,7 @@ func TestDedupeLinks(t *testing.T) { Convey("test RunDedupeForDigest directly, trigger stat error on original blob", func() { // rebuild with dedupe true - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) duplicateBlobs := []string{ path.Join(dir, "dedupe1", "blobs", "sha256", blobDigest1), @@ -1303,7 +1303,7 @@ func TestDedupeLinks(t *testing.T) { defer taskScheduler.Shutdown() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) @@ -1317,7 +1317,7 @@ func TestDedupeLinks(t *testing.T) { defer taskScheduler.Shutdown() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1337,7 +1337,7 @@ func TestDedupeLinks(t *testing.T) { taskScheduler := runAndGetScheduler() defer taskScheduler.Shutdown() - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil, nil) // rebuild with dedupe true imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) @@ -1367,7 +1367,7 @@ func TestDedupeLinks(t *testing.T) { PutBlobFn: func(digest godigest.Digest, path string) error { return errCache }, - }) + }, nil) // rebuild with dedupe true, should have samefile blobs imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1400,7 +1400,7 @@ func TestDedupeLinks(t *testing.T) { return nil }, - }) + }, nil) // rebuild with dedupe true, should have samefile blobs imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1495,7 +1495,7 @@ func TestDedupe(t *testing.T) { UseRelPaths: true, }, log) - il := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + il := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) So(il.DedupeBlob("", "", "", ""), ShouldNotBeNil) }) @@ -1516,7 +1516,7 @@ func TestNegativeCases(t *testing.T) { }, log) So(local.NewImageStore(dir, true, - true, log, metrics, nil, cacheDriver), ShouldNotBeNil) + true, log, metrics, nil, cacheDriver, nil), ShouldNotBeNil) if os.Geteuid() != 0 { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ @@ -1524,7 +1524,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - So(local.NewImageStore("/deadBEEF", true, true, log, metrics, nil, cacheDriver), ShouldBeNil) + So(local.NewImageStore("/deadBEEF", true, true, log, metrics, nil, cacheDriver, nil), ShouldBeNil) } }) @@ -1539,7 +1539,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) err := os.Chmod(dir, 0o000) // remove all perms if err != nil { @@ -1589,7 +1589,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1703,7 +1703,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1730,7 +1730,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1778,7 +1778,7 @@ func TestNegativeCases(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1956,7 +1956,7 @@ func TestInjectWriteFile(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, false, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, false, log, metrics, nil, cacheDriver, nil) Convey("Failure path not reached", func() { err := imgStore.InitRepo("repo1") @@ -1987,7 +1987,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := "gc-all-repos-short" //nolint:goconst // test data gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2035,7 +2035,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := "gc-all-repos-short" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2073,7 +2073,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := "gc-sig" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2151,7 +2151,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := "gc-all-repos-short" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2226,7 +2226,7 @@ func TestGarbageCollectImageUnknownManifest(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) storeController := storage.StoreController{ DefaultStore: imgStore, @@ -2409,7 +2409,7 @@ func TestGarbageCollectErrors(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := "gc-index" gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2656,7 +2656,7 @@ func TestInitRepo(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -2678,7 +2678,7 @@ func TestValidateRepo(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -2698,7 +2698,7 @@ func TestValidateRepo(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) _, err := imgStore.ValidateRepo(".") So(err, ShouldNotBeNil) @@ -2743,7 +2743,7 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) // Create valid directory with permissions err := os.Mkdir(path.Join(dir, "test-dir"), 0o755) //nolint: gosec @@ -2838,7 +2838,7 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) // Root dir does not contain repos repos, err := imgStore.GetRepositories() @@ -2885,7 +2885,7 @@ func TestGetRepositories(t *testing.T) { }, log) imgStore := local.NewImageStore(rootDir, - true, true, log, metrics, nil, cacheDriver, + true, true, log, metrics, nil, cacheDriver, nil, ) // Root dir does not contain repos @@ -2928,7 +2928,7 @@ func TestGetNextRepository(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) firstRepoName := "repo1" secondRepoName := "repo2" @@ -2981,7 +2981,7 @@ func TestPutBlobChunkStreamed(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) uuid, err := imgStore.NewBlobUpload("test") So(err, ShouldBeNil) @@ -3011,7 +3011,7 @@ func TestPullRange(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) repoName := "pull-range" upload, err := imgStore.NewBlobUpload(repoName) @@ -3053,7 +3053,7 @@ func TestStatIndex(t *testing.T) { dir := t.TempDir() log := zlog.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, nil, nil) err := WriteImageToFileSystem(CreateRandomImage(), "repo", "tag", storage.StoreController{DefaultStore: imgStore}) @@ -3077,7 +3077,7 @@ func TestStorageDriverErr(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, log, metrics, nil, cacheDriver, nil) Convey("Init repo", t, func() { err := imgStore.InitRepo(repoName) diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index a3ebc64ab..41a8915c7 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -6,6 +6,7 @@ import ( // Load s3 driver. _ "github.com/distribution/distribution/v3/registry/storage/driver/s3-aws" + "zotregistry.dev/zot/pkg/compat" "zotregistry.dev/zot/pkg/extensions/monitoring" zlog "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/storage/cache" @@ -18,7 +19,8 @@ import ( // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlog.Logger, - metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, cacheDriver cache.Cache, + metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, + cacheDriver cache.Cache, compat []compat.MediaCompatibility, ) storageTypes.ImageStore { return imagestore.NewImageStore( rootDir, @@ -30,5 +32,6 @@ func NewImageStore(rootDir string, cacheDir string, dedupe, commit bool, log zlo linter, New(store), cacheDriver, + compat, ) } diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 39798ee19..f21b3ffc8 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -75,7 +75,7 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive }, log) } - il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil) return il } @@ -86,7 +86,7 @@ func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.St log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := s3.NewImageStore(rootDir, "", dedupe, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, "", dedupe, false, log, metrics, nil, store, cacheDriver, nil) return il } @@ -147,7 +147,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( }, log) } - il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil) return store, il, err } @@ -181,7 +181,7 @@ func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tabl panic(err) } - il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver, nil) return store, il, err } diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index 66617146b..468b7a00b 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -49,7 +49,7 @@ func TestLocalCheckAllBlobsIntegrity(t *testing.T) { UseRelPaths: true, }, log) driver := local.New(true) - imgStore := local.NewImageStore(tdir, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(tdir, true, true, log, metrics, nil, cacheDriver, nil) RunCheckAllBlobsIntegrityTests(t, imgStore, driver, log) }) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index edb366018..e76b2c61c 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -58,7 +58,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer //nolint:typecheck,contextcheck rootDir := config.Storage.RootDirectory defaultStore = local.NewImageStore(rootDir, - config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, cacheDriver, + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, cacheDriver, config.HTTP.Compat, ) } else { storeName := fmt.Sprintf("%v", config.Storage.StorageDriver["name"]) @@ -92,7 +92,7 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer // false positive lint - linter does not implement Lint method //nolint: typecheck,contextcheck defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory, - config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver) + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, cacheDriver, config.HTTP.Compat) } storeController.DefaultStore = defaultStore @@ -170,7 +170,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, rootDir := storageConfig.RootDirectory imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(rootDir, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, cacheDriver, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, cacheDriver, cfg.HTTP.Compat, ) subImageStore[route] = imgStoreMap[storageConfig.RootDirectory] @@ -210,7 +210,7 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // false positive lint - linter does not implement Lint method //nolint: typecheck subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, cacheDriver, cfg.HTTP.Compat, ) } } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index a94591339..6ac19a4a8 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -102,7 +102,7 @@ func createObjectsStore(rootDir string, cacheDir string) ( UseRelPaths: false, }, log) - il := s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, cacheDir, true, false, log, metrics, nil, store, cacheDriver, nil) return store, il, err } @@ -167,7 +167,7 @@ func TestGetAllDedupeReposCandidates(t *testing.T) { driver := local.New(true) - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver) + imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) } Convey("Push repos with deduped blobs", t, func(c C) { @@ -237,7 +237,7 @@ func TestStorageAPIs(t *testing.T) { driver := local.New(true) - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver) + imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) } Convey("Repo layout", t, func(c C) { @@ -952,7 +952,7 @@ func TestMandatoryAnnotations(t *testing.T) { LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, driver, nil) + }, driver, nil, nil) defer cleanupStorage(store, testDir) } else { @@ -968,7 +968,7 @@ func TestMandatoryAnnotations(t *testing.T) { LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, driver, cacheDriver) + }, driver, cacheDriver, nil) } Convey("Setup manifest", t, func() { @@ -1022,7 +1022,7 @@ func TestMandatoryAnnotations(t *testing.T) { //nolint: goerr113 return false, errors.New("linter error") }, - }, driver, nil) + }, driver, nil, nil) } else { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: tdir, @@ -1036,7 +1036,7 @@ func TestMandatoryAnnotations(t *testing.T) { //nolint: goerr113 return false, errors.New("linter error") }, - }, driver, cacheDriver) + }, driver, cacheDriver, nil) } _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) @@ -1150,7 +1150,7 @@ func TestDeleteBlobsInUse(t *testing.T) { }, log) driver := local.New(true) imgStore = imagestore.NewImageStore(tdir, tdir, true, - true, log, metrics, nil, driver, cacheDriver) + true, log, metrics, nil, driver, cacheDriver, nil) } Convey("Setup manifest", t, func() { @@ -1458,7 +1458,7 @@ func TestReuploadCorruptedBlob(t *testing.T) { }, log) driver = local.New(true) imgStore = imagestore.NewImageStore(tdir, tdir, true, - true, log, metrics, nil, driver, cacheDriver) + true, log, metrics, nil, driver, cacheDriver, nil) } Convey("Test errors paths", t, func() { @@ -1604,11 +1604,14 @@ func TestStorageHandler(t *testing.T) { driver := local.New(true) // Create ImageStore - firstStore = imagestore.NewImageStore(firstRootDir, firstRootDir, false, false, log, metrics, nil, driver, nil) + firstStore = imagestore.NewImageStore(firstRootDir, firstRootDir, false, false, + log, metrics, nil, driver, nil, nil) - secondStore = imagestore.NewImageStore(secondRootDir, secondRootDir, false, false, log, metrics, nil, driver, nil) + secondStore = imagestore.NewImageStore(secondRootDir, secondRootDir, false, false, + log, metrics, nil, driver, nil, nil) - thirdStore = imagestore.NewImageStore(thirdRootDir, thirdRootDir, false, false, log, metrics, nil, driver, nil) + thirdStore = imagestore.NewImageStore(thirdRootDir, thirdRootDir, false, false, log, + metrics, nil, driver, nil, nil) } Convey("Test storage handler", t, func() { @@ -1693,7 +1696,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { driver := local.New(true) - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver) + imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -1865,7 +1868,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { driver := local.New(true) imgStore = imagestore.NewImageStore(dir, dir, true, - true, log, metrics, nil, driver, cacheDriver) + true, log, metrics, nil, driver, cacheDriver, nil) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2150,7 +2153,7 @@ func TestGarbageCollectImageManifest(t *testing.T) { driver := local.New(true) - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver) + imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2395,7 +2398,7 @@ func TestGarbageCollectImageIndex(t *testing.T) { driver := local.New(true) - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver) + imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2524,7 +2527,7 @@ func TestGarbageCollectImageIndex(t *testing.T) { driver := local.New(true) - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver) + imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ @@ -2812,7 +2815,7 @@ func TestGarbageCollectChainedImageIndexes(t *testing.T) { driver := local.New(true) - imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver) + imgStore = imagestore.NewImageStore(dir, dir, true, true, log, metrics, nil, driver, cacheDriver, nil) } gc := gc.NewGarbageCollect(imgStore, mocks.MetaDBMock{}, gc.Options{ diff --git a/pkg/test/oci-utils/oci_layout_test.go b/pkg/test/oci-utils/oci_layout_test.go index 75908bfcc..293822ab5 100644 --- a/pkg/test/oci-utils/oci_layout_test.go +++ b/pkg/test/oci-utils/oci_layout_test.go @@ -433,7 +433,7 @@ func TestExtractImageDetails(t *testing.T) { dir := t.TempDir() testLogger := log.NewLogger("debug", "") imageStore := local.NewImageStore(dir, false, false, - testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) + testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -457,7 +457,7 @@ func TestExtractImageDetails(t *testing.T) { dir := t.TempDir() testLogger := log.NewLogger("debug", "") imageStore := local.NewImageStore(dir, false, false, - testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) + testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, @@ -477,7 +477,7 @@ func TestExtractImageDetails(t *testing.T) { dir := t.TempDir() testLogger := log.NewLogger("debug", "") imageStore := local.NewImageStore(dir, false, false, - testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) + testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil, nil) storeController := storage.StoreController{ DefaultStore: imageStore, diff --git a/pkg/test/oci-utils/store.go b/pkg/test/oci-utils/store.go index afb0e913c..2bb15e78b 100644 --- a/pkg/test/oci-utils/store.go +++ b/pkg/test/oci-utils/store.go @@ -19,7 +19,7 @@ func GetDefaultImageStore(rootDir string, log zLog.Logger) stypes.ImageStore { return true, nil }, }, - mocks.CacheMock{}, + mocks.CacheMock{}, nil, ) } diff --git a/test/blackbox/ci.sh b/test/blackbox/ci.sh index 14974440b..3820d1955 100755 --- a/test/blackbox/ci.sh +++ b/test/blackbox/ci.sh @@ -9,7 +9,7 @@ PATH=$PATH:${SCRIPTPATH}/../../hack/tools/bin tests=("pushpull" "pushpull_authn" "delete_images" "referrers" "metadata" "anonymous_policy" "annotations" "detect_manifest_collision" "cve" "sync" "sync_docker" "sync_replica_cluster" - "scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index") + "scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index" "docker_compat") for test in ${tests[*]}; do ${BATS} ${BATS_FLAGS} ${SCRIPTPATH}/${test}.bats > ${test}.log & pids+=($!) diff --git a/test/blackbox/docker_compat.bats b/test/blackbox/docker_compat.bats new file mode 100644 index 000000000..1dea7d6f2 --- /dev/null +++ b/test/blackbox/docker_compat.bats @@ -0,0 +1,94 @@ +# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci" +# Makefile target installs & checks all necessary tooling +# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() + +load helpers_zot + +function verify_prerequisites { + if [ ! $(command -v curl) ]; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + return 0 +} + +function setup_file() { + # Verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + zot_port=$(get_free_port) + echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port + cat > ${zot_config_file}< Dockerfile < /testfile +EOF + docker build -f Dockerfile . -t localhost:${zot_port}/test:latest + run docker push localhost:${zot_port}/test:latest + [ "$status" -eq 0 ] + [ $(cat ${zot_root_dir}/test/index.json | jq .manifests[0].mediaType) = '"application/vnd.docker.distribution.manifest.v2+json"' ] + run docker pull localhost:${zot_port}/test:latest + [ "$status" -eq 0 ] + # inspect and trigger a CVE scan + run skopeo inspect --tls-verify=false docker://localhost:${zot_port}/test:latest + [ "$status" -eq 0 ] + # delete + run skopeo delete --tls-verify=false docker://localhost:${zot_port}/test:latest + [ "$status" -eq 0 ] + run skopeo inspect --tls-verify=false docker://localhost:${zot_port}/test:latest + [ "$status" -ne 0 ] + # re-push + run docker push localhost:${zot_port}/test:latest + [ "$status" -eq 0 ] + run skopeo inspect --tls-verify=false docker://localhost:${zot_port}/test:latest + [ "$status" -eq 0 ] +} diff --git a/test/blackbox/pushpull.bats b/test/blackbox/pushpull.bats index cebf1b4b2..c1ea075a0 100644 --- a/test/blackbox/pushpull.bats +++ b/test/blackbox/pushpull.bats @@ -359,3 +359,16 @@ EOF [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.manifests | length') -eq 0 ] } + +@test "push docker image" { + zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` + cat > Dockerfile < /testfile +EOF + docker build -f Dockerfile . -t localhost:${zot_port}/test + run docker push localhost:${zot_port}/test + [ "$status" -eq 1 ] + run docker pull localhost:${zot_port}/test + [ "$status" -eq 1 ] +}