diff --git a/errors/errors.go b/errors/errors.go index 3e4c65e98e..f1d8e0be2f 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -110,4 +110,7 @@ var ( ErrInvalidPublicKeyContent = errors.New("signatures: invalid public key content") ErrInvalidStateCookie = errors.New("auth: state cookie not present or differs from original state") ErrSyncNoURLsLeft = errors.New("sync: no valid registry urls left after filtering local ones") + ErrFileAlreadyCancelled = errors.New("storageDriver: file already cancelled") + ErrFileAlreadyClosed = errors.New("storageDriver: file already closed") + ErrFileAlreadyCommitted = errors.New("storageDriver: file already committed") ) diff --git a/go.mod b/go.mod index c82a52afe6..1ad8d3c740 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/99designs/gqlgen v0.17.35 github.com/Masterminds/semver v1.5.0 - github.com/apex/log v1.9.0 + github.com/apex/log v1.9.0 // indirect github.com/aquasecurity/trivy-db v0.0.0-20230703082116-dc52e83376ce github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/briandowns/spinner v1.23.0 @@ -23,7 +23,6 @@ require ( github.com/gorilla/mux v1.8.0 github.com/hashicorp/golang-lru/v2 v2.0.3 github.com/json-iterator/go v1.1.12 - github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/mapstructure v1.5.0 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba github.com/olekukonko/tablewriter v0.0.5 @@ -367,7 +366,6 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422 // indirect @@ -422,7 +420,7 @@ require ( github.com/sigstore/fulcio v1.3.1 // indirect github.com/sigstore/rekor v1.2.2-0.20230530122220-67cc9e58bd23 // indirect github.com/sigstore/sigstore v1.7.1 - github.com/sirupsen/logrus v1.9.3 // indirect + github.com/sirupsen/logrus v1.9.3 github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.1 // indirect diff --git a/go.sum b/go.sum index 923a80aaf5..5dfb86474a 100644 --- a/go.sum +++ b/go.sum @@ -1128,8 +1128,6 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 h1:BcxbplxjtczA1a6d3wYoa7a0WL3rq9DKBMGHeKyjEF0= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -1256,8 +1254,6 @@ github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/migueleliasweb/go-github-mock v0.0.19 h1:z/88f6wPqZVFnE7s9DbwXMhCtmV/0FofNxc4M7FuSdU= github.com/migueleliasweb/go-github-mock v0.0.19/go.mod h1:dBoCB3W9NjzyABhoGkfI0iSlFpzulAXhI7M+9A4ONYI= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -2062,7 +2058,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index e034bbc11d..fc1713f023 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -971,7 +971,7 @@ func TestInterruptedBlobUpload(t *testing.T) { defer cm.StopServer() client := resty.New() - blob := make([]byte, 50*1024*1024) + blob := make([]byte, 200*1024*1024) digest := godigest.FromBytes(blob).String() //nolint: dupl @@ -1024,6 +1024,7 @@ func TestInterruptedBlobUpload(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) }) + //nolint: dupl Convey("Test negative interrupt PATCH blob upload", func() { resp, err := client.R().Post(baseURL + "/v2/" + AuthorizedNamespace + "/blobs/uploads/") So(err, ShouldBeNil) @@ -1126,6 +1127,7 @@ func TestInterruptedBlobUpload(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) }) + //nolint: dupl Convey("Test negative interrupt PUT blob upload", func() { resp, err := client.R().Post(baseURL + "/v2/" + AuthorizedNamespace + "/blobs/uploads/") So(err, ShouldBeNil) @@ -6679,6 +6681,12 @@ func TestManifestImageIndex(t *testing.T) { So(digestHdr, ShouldEqual, digest.String()) }) + Convey("Deleting manifest contained by a multiarch image should not be allowed", func() { + resp, err = resty.R().Delete(baseURL + fmt.Sprintf("/v2/index/manifests/%s", m2dgst.String())) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + }) + Convey("Deleting an image index", func() { // delete manifest by tag should pass resp, err = resty.R().Delete(baseURL + "/v2/index/manifests/test:index3") @@ -7229,7 +7237,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { So(digest, ShouldNotBeNil) // monolithic blob upload - injected := inject.InjectFailure(0) + injected := inject.InjectFailure(2) if injected { request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc, bytes.NewReader(content)) tokens := strings.Split(loc, "/") @@ -7302,7 +7310,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { // Testing router path: @Router /v2/{name}/manifests/{reference} [put] //nolint:lll // gofumpt conflicts with lll Convey("Uploading an image manifest blob (when injected simulates that PutImageManifest failed due to 'too many open files' error)", func() { - injected := inject.InjectFailure(1) + injected := inject.InjectFailure(2) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) @@ -7363,48 +7371,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { So(resp.StatusCode, ShouldEqual, http.StatusCreated) } }) - Convey("code coverage: error inside PutImageManifest method of img store (umoci.OpenLayout error)", func() { - injected := inject.InjectFailure(3) - - request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) - request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) - request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - response := httptest.NewRecorder() - - rthdlr.UpdateManifest(response, request) - - resp := response.Result() - defer resp.Body.Close() - - So(resp, ShouldNotBeNil) - - if injected { - So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError) - } else { - So(resp.StatusCode, ShouldEqual, http.StatusCreated) - } - }) - Convey("code coverage: error inside PutImageManifest method of img store (oci.GC)", func() { - injected := inject.InjectFailure(4) - - request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) - request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) - request.Header.Set("Content-Type", "application/vnd.oci.image.manifest.v1+json") - response := httptest.NewRecorder() - - rthdlr.UpdateManifest(response, request) - resp := response.Result() - defer resp.Body.Close() - - So(resp, ShouldNotBeNil) - - if injected { - So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError) - } else { - So(resp.StatusCode, ShouldEqual, http.StatusCreated) - } - }) Convey("when index.json is not in json format", func() { resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). SetBody(content).Put(baseURL + "/v2/repotest/manifests/v1.0") diff --git a/pkg/api/routes.go b/pkg/api/routes.go index fe7ed7a59d..d3c2dba5aa 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -814,6 +814,10 @@ func (rh *RouteHandler) DeleteManifest(response http.ResponseWriter, request *ht } else if errors.Is(err, zerr.ErrBadManifest) { zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(apiErr.NewError(apiErr.UNSUPPORTED, map[string]string{"reference": reference}))) + } else if errors.Is(err, zerr.ErrBlobReferenced) { + zcommon.WriteJSON(response, + http.StatusMethodNotAllowed, + apiErr.NewErrorList(apiErr.NewError(apiErr.DENIED, map[string]string{"reference": reference}))) } else { rh.c.Log.Error().Err(err).Msg("unexpected error") response.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/cli/root.go b/pkg/cli/root.go index ecb5d3c944..801965325c 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -25,7 +25,6 @@ import ( extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" storageConstants "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/storage/s3" ) // metadataConfig reports metadata after parsing, which we use to track @@ -631,7 +630,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) { // s3 dedup=false, check for previous dedup usage and set to true if cachedb found if !config.Storage.Dedupe && config.Storage.StorageDriver != nil { cacheDir, _ := config.Storage.StorageDriver["rootdirectory"].(string) - cachePath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + cachePath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err := os.Stat(cachePath); err == nil { log.Info().Msg("Config: dedupe set to false for s3 driver but used to be true.") @@ -652,7 +651,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) { // s3 dedup=false, check for previous dedup usage and set to true if cachedb found if !storageConfig.Dedupe && storageConfig.StorageDriver != nil { subpathCacheDir, _ := storageConfig.StorageDriver["rootdirectory"].(string) - subpathCachePath := path.Join(subpathCacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + subpathCachePath := path.Join(subpathCacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err := os.Stat(subpathCachePath); err == nil { log.Info().Msg("Config: dedupe set to false for s3 driver but used to be true. ") diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index 65aed63fe0..ac06dd0f63 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -14,7 +14,6 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/cli" storageConstants "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/storage/s3" . "zotregistry.io/zot/pkg/test" ) @@ -521,7 +520,7 @@ func TestVerify(t *testing.T) { // s3 dedup=false, check for previous dedup usage and set to true if cachedb found cacheDir := t.TempDir() - existingDBPath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + existingDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) _, err = os.Create(existingDBPath) So(err, ShouldBeNil) @@ -537,7 +536,7 @@ func TestVerify(t *testing.T) { // subpath s3 dedup=false, check for previous dedup usage and set to true if cachedb found cacheDir = t.TempDir() - existingDBPath = path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + existingDBPath = path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) _, err = os.Create(existingDBPath) So(err, ShouldBeNil) diff --git a/pkg/extensions/extension_scrub.go b/pkg/extensions/extension_scrub.go index 94bb2435d9..99a3d822ed 100644 --- a/pkg/extensions/extension_scrub.go +++ b/pkg/extensions/extension_scrub.go @@ -30,19 +30,25 @@ func EnableScrubExtension(config *config.Config, log log.Logger, storeController log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll } - generator := &taskGenerator{ - imgStore: storeController.DefaultStore, - log: log, + // is local imagestore (because of umoci dependency which works only locally) + if config.Storage.StorageDriver == nil { + generator := &taskGenerator{ + imgStore: storeController.DefaultStore, + log: log, + } + sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } - sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) if config.Storage.SubPaths != nil { for route := range config.Storage.SubPaths { - generator := &taskGenerator{ - imgStore: storeController.SubStore[route], - log: log, + // is local imagestore (because of umoci dependency which works only locally) + if config.Storage.SubPaths[route].StorageDriver == nil { + generator := &taskGenerator{ + imgStore: storeController.SubStore[route], + log: log, + } + sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } - sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } } } else { diff --git a/pkg/extensions/extensions_lint_disabled.go b/pkg/extensions/extensions_lint_disabled.go index d2d803d1da..a059bdf2c5 100644 --- a/pkg/extensions/extensions_lint_disabled.go +++ b/pkg/extensions/extensions_lint_disabled.go @@ -11,7 +11,7 @@ import ( func GetLinter(config *config.Config, log log.Logger) *lint.Linter { log.Warn().Msg("lint extension is disabled because given zot binary doesn't" + - "include this feature please build a binary that does so") + " include this feature please build a binary that does so") return nil } diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 67fd6f800f..9006c8d24b 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -25,6 +25,7 @@ import ( mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" "zotregistry.io/zot/pkg/storage/local" storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" @@ -514,7 +515,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) { func TestIsIndexScanable(t *testing.T) { Convey("IsIndexScanable", t, func() { storeController := storage.StoreController{} - storeController.DefaultStore = &local.ImageStoreLocal{} + storeController.DefaultStore = &imagestore.ImageStore{} metaDB := &boltdb.BoltDB{} log := log.NewLogger("debug", "") diff --git a/pkg/extensions/sync/constants/consts.go b/pkg/extensions/sync/constants/consts.go index 9dff8e9502..69484d241d 100644 --- a/pkg/extensions/sync/constants/consts.go +++ b/pkg/extensions/sync/constants/consts.go @@ -2,7 +2,8 @@ package constants // references type. const ( - Oras = "OrasReference" - Cosign = "CosignSignature" - OCI = "OCIReference" + Oras = "OrasReference" + Cosign = "CosignSignature" + OCI = "OCIReference" + SyncBlobUploadDir = ".sync" ) diff --git a/pkg/extensions/sync/oci_layout.go b/pkg/extensions/sync/oci_layout.go index 1d1377193b..5806d96bc1 100644 --- a/pkg/extensions/sync/oci_layout.go +++ b/pkg/extensions/sync/oci_layout.go @@ -12,6 +12,7 @@ import ( "github.com/containers/image/v5/types" "github.com/gofrs/uuid" + "zotregistry.io/zot/pkg/extensions/sync/constants" "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/test/inject" @@ -39,7 +40,7 @@ func (oci OciLayoutStorageImpl) GetContext() *types.SystemContext { func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) (types.ImageReference, error) { localImageStore := oci.storeController.GetImageStore(repo) - tempSyncPath := path.Join(localImageStore.RootDir(), repo, SyncBlobUploadDir) + tempSyncPath := path.Join(localImageStore.RootDir(), repo, constants.SyncBlobUploadDir) // create session folder uuid, err := uuid.NewV4() diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index cfcb60200a..66cd1014eb 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -42,6 +42,7 @@ import ( extconf "zotregistry.io/zot/pkg/extensions/config" syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/extensions/sync" + syncConstants "zotregistry.io/zot/pkg/extensions/sync/constants" "zotregistry.io/zot/pkg/meta/signatures" mTypes "zotregistry.io/zot/pkg/meta/types" storageConstants "zotregistry.io/zot/pkg/storage/constants" @@ -580,7 +581,7 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - err = os.MkdirAll(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o000) + err = os.MkdirAll(path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir), 0o000) if err != nil { panic(err) } @@ -593,7 +594,7 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - err = os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o755) + err = os.Chmod(path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir), 0o755) if err != nil { panic(err) } @@ -1675,7 +1676,7 @@ func TestPermsDenied(t *testing.T) { defer dcm.StopServer() - syncSubDir := path.Join(destDir, testImage, sync.SyncBlobUploadDir) + syncSubDir := path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir) err := os.MkdirAll(syncSubDir, 0o755) So(err, ShouldBeNil) @@ -1686,7 +1687,7 @@ func TestPermsDenied(t *testing.T) { dcm.StartAndWait(destPort) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't get a local image reference", 20*time.Second) + "couldn't get a local image reference", 50*time.Second) if err != nil { panic(err) } @@ -4878,7 +4879,7 @@ func TestOnDemandPullsOnce(t *testing.T) { done := make(chan bool) var maxLen int - syncBlobUploadDir := path.Join(destDir, testImage, sync.SyncBlobUploadDir) + syncBlobUploadDir := path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir) go func() { for { @@ -6486,7 +6487,7 @@ func pushRepo(url, repoName string) godigest.Digest { func waitSync(rootDir, repoName string) { // wait for .sync subdirs to be removed for { - dirs, err := os.ReadDir(path.Join(rootDir, repoName, sync.SyncBlobUploadDir)) + dirs, err := os.ReadDir(path.Join(rootDir, repoName, syncConstants.SyncBlobUploadDir)) if err == nil && len(dirs) == 0 { // stop watching /.sync/ subdirs return diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 5eb210065c..7513cb562b 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -29,10 +29,6 @@ import ( "zotregistry.io/zot/pkg/test/inject" ) -const ( - SyncBlobUploadDir = ".sync" -) - // Get sync.FileCredentials from file. func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) { credsFile, err := os.ReadFile(filepath) diff --git a/pkg/storage/cache/boltdb.go b/pkg/storage/cache/boltdb.go index 89d068911c..f146e0dd2d 100644 --- a/pkg/storage/cache/boltdb.go +++ b/pkg/storage/cache/boltdb.go @@ -77,6 +77,10 @@ func NewBoltDBCache(parameters interface{}, log zlog.Logger) Cache { } } +func (d *BoltDBDriver) UsesRelativePaths() bool { + return d.useRelPaths +} + func (d *BoltDBDriver) Name() string { return "boltdb" } diff --git a/pkg/storage/cache/cacheinterface.go b/pkg/storage/cache/cacheinterface.go index d02fe95cf9..de1df6b2c1 100644 --- a/pkg/storage/cache/cacheinterface.go +++ b/pkg/storage/cache/cacheinterface.go @@ -19,4 +19,7 @@ type Cache interface { // Delete a blob from the cachedb. DeleteBlob(digest godigest.Digest, path string) error + + // UsesRelativePaths returns if cache is storing blobs relative to cache rootDir + UsesRelativePaths() bool } diff --git a/pkg/storage/cache/dynamodb.go b/pkg/storage/cache/dynamodb.go index 890d5177f3..4aafed0900 100644 --- a/pkg/storage/cache/dynamodb.go +++ b/pkg/storage/cache/dynamodb.go @@ -99,6 +99,10 @@ func NewDynamoDBCache(parameters interface{}, log zlog.Logger) Cache { return driver } +func (d *DynamoDBDriver) UsesRelativePaths() bool { + return false +} + func (d *DynamoDBDriver) Name() string { return "dynamodb" } diff --git a/pkg/storage/cache_test.go b/pkg/storage/cache_test.go index 01b31b7712..6158f25eae 100644 --- a/pkg/storage/cache_test.go +++ b/pkg/storage/cache_test.go @@ -35,6 +35,8 @@ func TestCache(t *testing.T) { }, log) So(cacheDriver, ShouldNotBeNil) + So(cacheDriver.UsesRelativePaths(), ShouldBeTrue) + name := cacheDriver.Name() So(name, ShouldEqual, "boltdb") diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index ea73c5dc0f..023c465703 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -7,6 +7,7 @@ import ( "path" "strings" + "github.com/docker/distribution/registry/storage/driver" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/schema" @@ -99,7 +100,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy continue } - ok, _, err := imgStore.StatBlob(repo, layer.Digest) + ok, _, _, err := imgStore.StatBlob(repo, layer.Digest) if !ok || err != nil { log.Error().Err(err).Str("digest", layer.Digest.String()).Msg("missing layer blob") @@ -130,7 +131,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy } for _, manifest := range indexManifest.Manifests { - if ok, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil { + if ok, _, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil { log.Error().Err(err).Str("digest", manifest.Digest.String()).Msg("missing manifest blob") return "", zerr.ErrBadManifest @@ -239,6 +240,10 @@ func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) buf, err := imgStore.GetIndexContent(repo) if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return index, zerr.ErrRepoNotFound + } + return index, err } @@ -401,6 +406,7 @@ func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, inUse[manifest.Digest.Encoded()]++ } + // inc cele de mai sus for _, otherIndex := range otherImgIndexes { oindex, err := GetImageIndex(imgStore, repo, otherIndex.Digest, log) if err != nil { @@ -451,7 +457,7 @@ func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, return prunedManifests, nil } -func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string, +func isBlobReferencedInImageManifest(imgStore storageTypes.ImageStore, repo string, bdigest, mdigest godigest.Digest, log zerolog.Logger, ) (bool, error) { if bdigest == mdigest { @@ -479,7 +485,7 @@ func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string, return false, nil } -func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, +func IsBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, index ispec.Index, log zerolog.Logger, ) (bool, error) { for _, desc := range index.Manifests { @@ -487,8 +493,6 @@ func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, switch desc.MediaType { case ispec.MediaTypeImageIndex: - /* this branch is not needed, because every manifests in index is already checked - when this one is hit, all manifests are referenced in index.json */ indexImage, err := GetImageIndex(imgStore, repo, desc.Digest, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). @@ -497,9 +501,9 @@ func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, return false, err } - found, _ = isBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log) + found, _ = IsBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log) case ispec.MediaTypeImageManifest: - found, _ = isBlobReferencedInManifest(imgStore, repo, digest, desc.Digest, log) + found, _ = isBlobReferencedInImageManifest(imgStore, repo, digest, desc.Digest, log) } if found { @@ -523,7 +527,101 @@ func IsBlobReferenced(imgStore storageTypes.ImageStore, repo string, return false, err } - return isBlobReferencedInImageIndex(imgStore, repo, digest, index, log) + return IsBlobReferencedInImageIndex(imgStore, repo, digest, index, log) +} + +/* Garbage Collection */ + +func AddManifestBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zerolog.Logger, +) error { + manifestContent, err := GetImageManifest(imgStore, repo, mdigest, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + refBlobs[mdigest.String()] = true + refBlobs[manifestContent.Config.Digest.String()] = true + + // if there is a Subject, it may not exist yet and that is ok + if manifestContent.Subject != nil { + refBlobs[manifestContent.Subject.Digest.String()] = true + } + + for _, layer := range manifestContent.Layers { + refBlobs[layer.Digest.String()] = true + } + + return nil +} + +func AddIndexBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zerolog.Logger, +) error { + index, err := GetImageIndex(imgStore, repo, mdigest, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + refBlobs[mdigest.String()] = true + + // if there is a Subject, it may not exist yet and that is ok + if index.Subject != nil { + refBlobs[index.Subject.Digest.String()] = true + } + + for _, manifest := range index.Manifests { + refBlobs[manifest.Digest.String()] = true + } + + return nil +} + +func AddRepoBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, index ispec.Index, refBlobs map[string]bool, log zerolog.Logger, +) error { + for _, desc := range index.Manifests { + switch desc.MediaType { + case ispec.MediaTypeImageIndex: + if err := AddIndexBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read blobs in multiarch(index) image") + + return err + } + case ispec.MediaTypeImageManifest: + if err := AddManifestBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read blobs in image manifest") + + return err + } + } + } + + return nil +} + +func GetReferencedBlobs(imgStore storageTypes.ImageStore, + repo string, refBlobs map[string]bool, log zerolog.Logger, +) error { + dir := path.Join(imgStore.RootDir(), repo) + if !imgStore.DirExists(dir) { + return zerr.ErrRepoNotFound + } + + index, err := GetIndex(imgStore, repo, log) + if err != nil { + return err + } + + return AddRepoBlobsToReferences(imgStore, repo, index, refBlobs, log) } func ApplyLinter(imgStore storageTypes.ImageStore, linter Lint, repo string, descriptor ispec.Descriptor, diff --git a/pkg/storage/common/common_test.go b/pkg/storage/common/common_test.go index 76ae36eca2..844e943c77 100644 --- a/pkg/storage/common/common_test.go +++ b/pkg/storage/common/common_test.go @@ -103,8 +103,9 @@ func TestValidateManifest(t *testing.T) { body, err := json.Marshal(manifest) So(err, ShouldBeNil) + // this was actually an umoci error on config blob _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body) - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) }) Convey("manifest with non-distributable layers", func() { @@ -425,3 +426,219 @@ func TestIsSignature(t *testing.T) { So(isSingature, ShouldBeFalse) }) } + +func TestGarbageCollectManifestErrors(t *testing.T) { + Convey("Make imagestore and upload manifest", t, func(c C) { + dir := t.TempDir() + + repoName := "test" + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, + true, log, metrics, nil, cacheDriver) + + Convey("trigger repo not found in GetReferencedBlobs()", func() { + err := common.GetReferencedBlobs(imgStore, repoName, map[string]bool{}, log.Logger) + So(err, ShouldNotBeNil) + }) + + content := []byte("this is a blob") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + _, blen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), digest) + So(err, ShouldBeNil) + So(blen, ShouldEqual, len(content)) + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + body, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(body) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, body) + So(err, ShouldBeNil) + + Convey("trigger GetIndex error in GetReferencedBlobs", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o755) + So(err, ShouldBeNil) + }() + + err = common.GetReferencedBlobs(imgStore, repoName, map[string]bool{}, log.Logger) + So(err, ShouldNotBeNil) + }) + + Convey("trigger GetImageManifest error in GetReferencedBlobsInImageManifest", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o755) + So(err, ShouldBeNil) + }() + + err = common.GetReferencedBlobs(imgStore, repoName, map[string]bool{}, log.Logger) + So(err, ShouldNotBeNil) + }) + }) +} + +func TestGarbageCollectIndexErrors(t *testing.T) { + Convey("Make imagestore and upload manifest", t, func(c C) { + dir := t.TempDir() + + repoName := "test" + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, + true, log, metrics, nil, cacheDriver) + + content := []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + for i := 0; i < 4; i++ { + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + err = common.GetReferencedBlobs(imgStore, repoName, map[string]bool{}, log.Logger) + So(err, ShouldBeNil) + + Convey("trigger GetImageIndex error in GetReferencedBlobsInImageIndex", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o755) + So(err, ShouldBeNil) + }() + + err = common.GetReferencedBlobs(imgStore, repoName, map[string]bool{}, log.Logger) + So(err, ShouldNotBeNil) + }) + + Convey("trigger GetReferencedBlobsInImageIndex error when called recursively", func() { + // need to delete manifests referenced in index.json, so that we can trigger error after first recurse + indexJSON, err := common.GetIndex(imgStore, repoName, log.Logger) + So(err, ShouldBeNil) + + indexJSON.Manifests = indexJSON.Manifests[4:5] + + indexContent, err := json.Marshal(indexJSON) + So(err, ShouldBeNil) + + err = os.WriteFile(path.Join(imgStore.RootDir(), repoName, "index.json"), + indexContent, storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + err = os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", digest.Encoded()), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", digest.Encoded()), 0o755) + So(err, ShouldBeNil) + }() + + err = common.GetReferencedBlobs(imgStore, repoName, map[string]bool{}, log.Logger) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/pkg/storage/constants/constants.go b/pkg/storage/constants/constants.go index 2a7df48a88..6bb7d119db 100644 --- a/pkg/storage/constants/constants.go +++ b/pkg/storage/constants/constants.go @@ -21,4 +21,5 @@ const ( DynamoDBDriverName = "dynamodb" DefaultGCDelay = 1 * time.Hour S3StorageDriverName = "s3" + LocalStorageDriverName = "filesystem" ) diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go new file mode 100644 index 0000000000..983f4ce08b --- /dev/null +++ b/pkg/storage/imagestore/imagestore.go @@ -0,0 +1,2235 @@ +package imagestore + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/docker/distribution/registry/storage/driver" + guuid "github.com/gofrs/uuid" + notreg "github.com/notaryproject/notation-go/registry" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" + "github.com/rs/zerolog" + "github.com/sigstore/cosign/v2/pkg/oci/remote" + + zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/extensions/monitoring" + syncConstants "zotregistry.io/zot/pkg/extensions/sync/constants" + zlog "zotregistry.io/zot/pkg/log" + zreg "zotregistry.io/zot/pkg/regexp" + "zotregistry.io/zot/pkg/scheduler" + "zotregistry.io/zot/pkg/storage/cache" + common "zotregistry.io/zot/pkg/storage/common" + storageConstants "zotregistry.io/zot/pkg/storage/constants" + storageTypes "zotregistry.io/zot/pkg/storage/types" + "zotregistry.io/zot/pkg/test/inject" +) + +// ImageStore provides the image storage operations. +type ImageStore struct { + rootDir string + storeDriver storageTypes.Driver + lock *sync.RWMutex + log zerolog.Logger + metrics monitoring.MetricServer + cache cache.Cache + dedupe bool + linter common.Lint + commit bool + gc bool + gcDelay time.Duration +} + +func (is *ImageStore) RootDir() string { + return is.rootDir +} + +func (is *ImageStore) DirExists(d string) bool { + return is.storeDriver.DirExists(d) +} + +// NewImageStore returns a new image store backed by cloud storages. +// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers +// Use the last argument to properly set a cache database, or it will default to boltDB local storage. +func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, + log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, + storeDriver storageTypes.Driver, cacheDriver cache.Cache, +) storageTypes.ImageStore { + if err := storeDriver.EnsureDir(rootDir); err != nil { + log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") + + return nil + } + + imgStore := &ImageStore{ + rootDir: rootDir, + storeDriver: storeDriver, + lock: &sync.RWMutex{}, + log: log.With().Caller().Logger(), + metrics: metrics, + dedupe: dedupe, + linter: linter, + commit: commit, + gc: gc, + gcDelay: gcDelay, + cache: cacheDriver, + } + + return imgStore +} + +// RLock read-lock. +func (is *ImageStore) RLock(lockStart *time.Time) { + *lockStart = time.Now() + + is.lock.RLock() +} + +// RUnlock read-unlock. +func (is *ImageStore) RUnlock(lockStart *time.Time) { + is.lock.RUnlock() + + lockEnd := time.Now() + // includes time spent in acquiring and holding a lock + latency := lockEnd.Sub(*lockStart) + monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram +} + +// Lock write-lock. +func (is *ImageStore) Lock(lockStart *time.Time) { + *lockStart = time.Now() + + is.lock.Lock() +} + +// Unlock write-unlock. +func (is *ImageStore) Unlock(lockStart *time.Time) { + is.lock.Unlock() + + lockEnd := time.Now() + // includes time spent in acquiring and holding a lock + latency := lockEnd.Sub(*lockStart) + monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram +} + +func (is *ImageStore) initRepo(name string) error { + repoDir := path.Join(is.rootDir, name) + + if !utf8.ValidString(name) { + is.log.Error().Msg("input is not valid UTF-8") + + return zerr.ErrInvalidRepositoryName + } + + if !zreg.FullNameRegexp.MatchString(name) { + is.log.Error().Str("repository", name).Msg("invalid repository name") + + return zerr.ErrInvalidRepositoryName + } + + // create "blobs" subdir + err := is.storeDriver.EnsureDir(path.Join(repoDir, "blobs")) + if err != nil { + is.log.Error().Err(err).Msg("error creating blobs subdir") + + return err + } + // create BlobUploadDir subdir + err = is.storeDriver.EnsureDir(path.Join(repoDir, storageConstants.BlobUploadDir)) + if err != nil { + is.log.Error().Err(err).Msg("error creating blob upload subdir") + + return err + } + + // "oci-layout" file - create if it doesn't exist + ilPath := path.Join(repoDir, ispec.ImageLayoutFile) + if _, err := is.storeDriver.Stat(ilPath); err != nil { + il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} + + buf, err := json.Marshal(il) + if err != nil { + is.log.Error().Err(err).Msg("unable to marshal JSON") + + return err + } + + if _, err := is.storeDriver.WriteFile(ilPath, buf); err != nil { + is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") + + return err + } + } + + // "index.json" file - create if it doesn't exist + indexPath := path.Join(repoDir, "index.json") + if _, err := is.storeDriver.Stat(indexPath); err != nil { + index := ispec.Index{} + index.SchemaVersion = 2 + + buf, err := json.Marshal(index) + if err != nil { + is.log.Error().Err(err).Msg("unable to marshal JSON") + + return err + } + + if _, err := is.storeDriver.WriteFile(indexPath, buf); err != nil { + is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") + + return err + } + } + + return nil +} + +// InitRepo creates an image repository under this store. +func (is *ImageStore) InitRepo(name string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + return is.initRepo(name) +} + +// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. +func (is *ImageStore) ValidateRepo(name string) (bool, error) { + if !zreg.FullNameRegexp.MatchString(name) { + return false, zerr.ErrInvalidRepositoryName + } + + // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content + // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] + // and an additional/optional BlobUploadDir in each image store + // for s3 we can not create empty dirs, so we check only against index.json and oci-layout + dir := path.Join(is.rootDir, name) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return false, zerr.ErrRepoNotFound + } + + files, err := is.storeDriver.List(dir) + if err != nil { + is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") + + return false, zerr.ErrRepoNotFound + } + + //nolint:gomnd + if len(files) < 2 { + return false, zerr.ErrRepoBadVersion + } + + found := map[string]bool{ + ispec.ImageLayoutFile: false, + "index.json": false, + } + + for _, file := range files { + fileInfo, err := is.storeDriver.Stat(file) + if err != nil { + return false, err + } + + filename, err := filepath.Rel(dir, file) + if err != nil { + return false, err + } + + if filename == "blobs" && !fileInfo.IsDir() { + return false, nil + } + + found[filename] = true + } + + // check blobs dir exists only for filesystem, in s3 we can't have empty dirs + if is.storeDriver.Name() == storageConstants.LocalStorageDriverName { + if !is.storeDriver.DirExists(path.Join(dir, "blobs")) { + return false, nil + } + } + + for k, v := range found { + if !v && k != storageConstants.BlobUploadDir { + return false, nil + } + } + + buf, err := is.storeDriver.ReadFile(path.Join(dir, ispec.ImageLayoutFile)) + if err != nil { + return false, err + } + + var il ispec.ImageLayout + if err := json.Unmarshal(buf, &il); err != nil { + return false, err + } + + if il.Version != ispec.ImageLayoutVersion { + return false, zerr.ErrRepoBadVersion + } + + return true, nil +} + +// GetRepositories returns a list of all the repositories under this store. +func (is *ImageStore) GetRepositories() ([]string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + stores := make([]string, 0) + + err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + if !fileInfo.IsDir() { + return nil + } + + rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) + if err != nil { + return nil //nolint:nilerr // ignore paths that are not under root dir + } + + if ok, err := is.ValidateRepo(rel); !ok || err != nil { + return nil //nolint:nilerr // ignore invalid repos + } + + stores = append(stores, rel) + + return nil + }) + + // if the root directory is not yet created then return an empty slice of repositories + var perr driver.PathNotFoundError + if errors.As(err, &perr) { + return stores, nil + } + + return stores, err +} + +// GetNextRepository returns next repository under this store. +func (is *ImageStore) GetNextRepository(repo string) (string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + _, err := is.storeDriver.List(dir) + if err != nil { + is.log.Error().Err(err).Msg("failure walking storage root-dir") + + return "", err + } + + found := false + store := "" + err = is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + if !fileInfo.IsDir() { + return nil + } + + rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) + if err != nil { + return nil //nolint:nilerr // ignore paths not relative to root dir + } + + ok, err := is.ValidateRepo(rel) + if !ok || err != nil { + return nil //nolint:nilerr // ignore invalid repos + } + + if repo == "" && ok && err == nil { + store = rel + + return io.EOF + } + + if found { + store = rel + + return io.EOF + } + + if rel == repo { + found = true + } + + return nil + }) + + return store, err +} + +// GetImageTags returns a list of image tags available in the specified repository. +func (is *ImageStore) GetImageTags(repo string) ([]string, error) { + var lockLatency time.Time + + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return nil, zerr.ErrRepoNotFound + } + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return nil, err + } + + return common.GetTagsByIndex(index), nil +} + +// GetImageManifest returns the image manifest of an image in the specific repository. +func (is *ImageStore) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return nil, "", "", zerr.ErrRepoNotFound + } + + var lockLatency time.Time + + var err error + + is.RLock(&lockLatency) + defer func() { + is.RUnlock(&lockLatency) + + if err == nil { + monitoring.IncDownloadCounter(is.metrics, repo) + } + }() + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return nil, "", "", err + } + + manifestDesc, found := common.GetManifestDescByReference(index, reference) + if !found { + return nil, "", "", zerr.ErrManifestNotFound + } + + buf, err := is.GetBlobContent(repo, manifestDesc.Digest) + if err != nil { + if errors.Is(err, zerr.ErrBlobNotFound) { + return nil, "", "", zerr.ErrManifestNotFound + } + + return nil, "", "", err + } + + var manifest ispec.Manifest + if err := json.Unmarshal(buf, &manifest); err != nil { + is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") + + return nil, "", "", err + } + + return buf, manifestDesc.Digest, manifestDesc.MediaType, nil +} + +// PutImageManifest adds an image manifest to the repository. +func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo + body []byte, +) (godigest.Digest, godigest.Digest, error) { + if err := is.InitRepo(repo); err != nil { + is.log.Debug().Err(err).Msg("init repo") + + return "", "", err + } + + var lockLatency time.Time + + var err error + + is.Lock(&lockLatency) + defer func() { + is.Unlock(&lockLatency) + + if err == nil { + monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) + monitoring.IncUploadCounter(is.metrics, repo) + } + }() + + refIsDigest := true + + mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) + if err != nil { + if errors.Is(err, zerr.ErrBadManifest) { + return mDigest, "", err + } + + refIsDigest = false + } + + dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) + if err != nil { + return dig, "", err + } + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return "", "", err + } + + // create a new descriptor + desc := ispec.Descriptor{ + MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, + } + + if !refIsDigest { + desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} + } + + var subjectDigest godigest.Digest + + artifactType := "" + + if mediaType == ispec.MediaTypeImageManifest { + var manifest ispec.Manifest + + err := json.Unmarshal(body, &manifest) + if err != nil { + return "", "", err + } + + if manifest.Subject != nil { + subjectDigest = manifest.Subject.Digest + } + + artifactType = zcommon.GetManifestArtifactType(manifest) + } else if mediaType == ispec.MediaTypeImageIndex { + var index ispec.Index + + err := json.Unmarshal(body, &index) + if err != nil { + return "", "", err + } + + if index.Subject != nil { + subjectDigest = index.Subject.Digest + } + + artifactType = zcommon.GetIndexArtifactType(index) + } + + updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) + if err != nil { + return "", "", err + } + + if !updateIndex { + return desc.Digest, subjectDigest, nil + } + + // write manifest to "blobs" + dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) + manifestPath := path.Join(dir, mDigest.Encoded()) + + if _, err = is.storeDriver.WriteFile(manifestPath, body); err != nil { + is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") + + return "", "", err + } + + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) + if err != nil { + return "", "", err + } + + // now update "index.json" + index.Manifests = append(index.Manifests, desc) + dir = path.Join(is.rootDir, repo) + indexPath := path.Join(dir, "index.json") + + buf, err := json.Marshal(index) + if err != nil { + is.log.Error().Err(err).Str("file", indexPath).Msg("unable to marshal JSON") + + return "", "", err + } + + // update the descriptors artifact type in order to check for signatures when applying the linter + desc.ArtifactType = artifactType + + // apply linter only on images, not signatures + pass, err := common.ApplyLinter(is, is.linter, repo, desc) + if !pass { + is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") + + return "", "", err + } + + if _, err = is.storeDriver.WriteFile(indexPath, buf); err != nil { + is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") + + return "", "", err + } + + if is.gc { + if err := is.garbageCollect(repo); err != nil { + return "", "", err + } + } + + return desc.Digest, subjectDigest, nil +} + +// DeleteImageManifest deletes the image manifest from the repository. +func (is *ImageStore) DeleteImageManifest(repo, reference string, detectCollisions bool) error { + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return zerr.ErrRepoNotFound + } + + var lockLatency time.Time + + var err error + + is.Lock(&lockLatency) + defer func() { + is.Unlock(&lockLatency) + + if err == nil { + monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) + } + }() + + err = is.deleteImageManifest(repo, reference, detectCollisions) + if err != nil { + return err + } + + if is.gc { + if err := is.garbageCollect(repo); err != nil { + return err + } + } + + return nil +} + +func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisions bool) error { + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return err + } + + manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollisions) + if err != nil { + return err + } + + /* check if manifest is referenced in image indexes, do not allow index images manipulations + (ie. remove manifest being part of an image index) */ + if manifestDesc.MediaType == ispec.MediaTypeImageManifest { + for _, mDesc := range index.Manifests { + if mDesc.MediaType == ispec.MediaTypeImageIndex { + if ok, _ := common.IsBlobReferencedInImageIndex(is, repo, manifestDesc.Digest, ispec.Index{ + Manifests: []ispec.Descriptor{mDesc}, + }, is.log); ok { + return zerr.ErrBlobReferenced + } + } + } + } + + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) + if err != nil { + return err + } + + // now update "index.json" + dir := path.Join(is.rootDir, repo) + file := path.Join(dir, "index.json") + + buf, err := json.Marshal(index) + if err != nil { + return err + } + + if _, err := is.storeDriver.WriteFile(file, buf); err != nil { + is.log.Debug().Str("deleting reference", reference).Msg("") + + return err + } + + // Delete blob only when blob digest not present in manifest entry. + // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. + toDelete := true + + for _, manifest := range index.Manifests { + if manifestDesc.Digest.String() == manifest.Digest.String() { + toDelete = false + + break + } + } + + if toDelete { + p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) + + err = is.storeDriver.Delete(p) + if err != nil { + return err + } + } + + return nil +} + +// BlobUploadPath returns the upload path for a blob in this store. +func (is *ImageStore) BlobUploadPath(repo, uuid string) string { + dir := path.Join(is.rootDir, repo) + blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) + + return blobUploadPath +} + +// NewBlobUpload returns the unique ID for an upload in progress. +func (is *ImageStore) NewBlobUpload(repo string) (string, error) { + if err := is.InitRepo(repo); err != nil { + is.log.Error().Err(err).Msg("error initializing repo") + + return "", err + } + + uuid, err := guuid.NewV4() + if err != nil { + return "", err + } + + uid := uuid.String() + + blobUploadPath := is.BlobUploadPath(repo, uid) + + // create multipart upload (append false) + writer, err := is.storeDriver.Writer(blobUploadPath, false) + if err != nil { + is.log.Debug().Err(err).Str("blob", blobUploadPath).Msg("failed to start multipart writer") + + return "", zerr.ErrRepoNotFound + } + + defer writer.Close() + + return uid, nil +} + +// GetBlobUpload returns the current size of a blob upload. +func (is *ImageStore) GetBlobUpload(repo, uuid string) (int64, error) { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + if !utf8.ValidString(blobUploadPath) { + is.log.Error().Msg("input is not valid UTF-8") + + return -1, zerr.ErrInvalidRepositoryName + } + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + return -1, err + } + + defer writer.Close() + + return writer.Size(), nil +} + +// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns +// the number of actual bytes to the blob. +func (is *ImageStore) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { + if err := is.InitRepo(repo); err != nil { + return -1, err + } + + blobUploadPath := is.BlobUploadPath(repo, uuid) + + file, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + is.log.Error().Err(err).Msg("failed to continue multipart upload") + + return -1, err + } + + var n int64 //nolint: varnamelen + + defer func() { + err = file.Close() + }() + + n, err = io.Copy(file, body) + + return n, err +} + +// PutBlobChunk writes another chunk of data to the specified blob. It returns +// the number of actual bytes to the blob. +func (is *ImageStore) PutBlobChunk(repo, uuid string, from, to int64, + body io.Reader, +) (int64, error) { + if err := is.InitRepo(repo); err != nil { + return -1, err + } + + blobUploadPath := is.BlobUploadPath(repo, uuid) + + file, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + is.log.Error().Err(err).Msg("failed to continue multipart upload") + + return -1, err + } + + defer file.Close() + + if from != file.Size() { + is.log.Error().Int64("expected", from).Int64("actual", file.Size()). + Msg("invalid range start for blob upload") + + return -1, zerr.ErrBadUploadRange + } + + n, err := io.Copy(file, body) + + return n, err +} + +// BlobUploadInfo returns the current blob size in bytes. +func (is *ImageStore) BlobUploadInfo(repo, uuid string) (int64, error) { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + return -1, err + } + + defer writer.Close() + + return writer.Size(), nil +} + +// FinishBlobUpload finalizes the blob upload and moves blob the repository. +func (is *ImageStore) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { + if err := dstDigest.Validate(); err != nil { + return err + } + + src := is.BlobUploadPath(repo, uuid) + + // complete multiUploadPart + fileWriter, err := is.storeDriver.Writer(src, true) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") + + return zerr.ErrUploadNotFound + } + + if err := fileWriter.Commit(); err != nil { + is.log.Error().Err(err).Msg("failed to commit file") + + return err + } + + if err := fileWriter.Close(); err != nil { + is.log.Error().Err(err).Msg("failed to close file") + + return err + } + + fileReader, err := is.storeDriver.Reader(src, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open file") + + return zerr.ErrUploadNotFound + } + + defer fileReader.Close() + + srcDigest, err := godigest.FromReader(fileReader) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") + + return zerr.ErrBadBlobDigest + } + + if srcDigest != dstDigest { + is.log.Error().Str("srcDigest", srcDigest.String()). + Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") + + return zerr.ErrBadBlobDigest + } + + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + + err = is.storeDriver.EnsureDir(dir) + if err != nil { + is.log.Error().Err(err).Msg("error creating blobs/sha256 dir") + + return err + } + + dst := is.BlobPath(repo, dstDigest) + + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + err = is.DedupeBlob(src, dstDigest, dst) + if err := inject.Error(err); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + + return err + } + } else { + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to finish blob") + + return err + } + } + + return nil +} + +// FullBlobUpload handles a full blob upload, and no partial session is created. +func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest) (string, int64, error) { + if err := dstDigest.Validate(); err != nil { + return "", -1, err + } + + if err := is.InitRepo(repo); err != nil { + return "", -1, err + } + + u, err := guuid.NewV4() + if err != nil { + return "", -1, err + } + + uuid := u.String() + src := is.BlobUploadPath(repo, uuid) + digester := sha256.New() + buf := new(bytes.Buffer) + + _, err = buf.ReadFrom(body) + if err != nil { + is.log.Error().Err(err).Msg("failed to read blob") + + return "", -1, err + } + + nbytes, err := is.storeDriver.WriteFile(src, buf.Bytes()) + if err != nil { + is.log.Error().Err(err).Msg("failed to write blob") + + return "", -1, err + } + + _, err = digester.Write(buf.Bytes()) + if err != nil { + is.log.Error().Err(err).Msg("digester failed to write") + + return "", -1, err + } + + srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) + if srcDigest != dstDigest { + is.log.Error().Str("srcDigest", srcDigest.String()). + Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") + + return "", -1, zerr.ErrBadBlobDigest + } + + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + _ = is.storeDriver.EnsureDir(dir) + + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + dst := is.BlobPath(repo, dstDigest) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + if err := is.DedupeBlob(src, dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + + return "", -1, err + } + } else { + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to finish blob") + + return "", -1, err + } + } + + return uuid, int64(nbytes), nil +} + +func (is *ImageStore) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { +retry: + is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") + + dstRecord, err := is.cache.GetBlob(dstDigest) + if err := inject.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") + + return err + } + + if dstRecord == "" { + // cache record doesn't exist, so first disk and cache entry for this digest + if err := is.cache.PutBlob(dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") + + return err + } + + // move the blob from uploads to final dest + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") + + return err + } + + is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") + } else { + // cache record exists, but due to GC and upgrades from older versions, + // disk content and cache records may go out of sync + + if is.cache.UsesRelativePaths() { + dstRecord = path.Join(is.rootDir, dstRecord) + } + + _, err := is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") + // the actual blob on disk may have been removed by GC, so sync the cache + err := is.cache.DeleteBlob(dstDigest, dstRecord) + if err = inject.Error(err); err != nil { + //nolint:lll + is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") + + return err + } + + goto retry + } + + // prevent overwrite original blob + if !is.storeDriver.SameFile(dst, dstRecord) { + if err := is.storeDriver.Link(dstRecord, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to link blobs") + + return err + } + + if err := is.cache.PutBlob(dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") + + return err + } + } + + // remove temp blobupload + if err := is.storeDriver.Delete(src); err != nil { + is.log.Error().Err(err).Str("src", src).Msg("dedupe: unable to remove blob") + + return err + } + + is.log.Debug().Str("src", src).Msg("dedupe: remove") + } + + return nil +} + +// DeleteBlobUpload deletes an existing blob upload that is currently in progress. +func (is *ImageStore) DeleteBlobUpload(repo, uuid string) error { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return zerr.ErrUploadNotFound + } + + return err + } + + defer writer.Close() + + if err := writer.Cancel(); err != nil { + is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") + + return err + } + + return nil +} + +// BlobPath returns the repository path of a blob. +func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string { + return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) +} + +/* + CheckBlob verifies a blob and returns true if the blob is correct + +If the blob is not found but it's found in cache then it will be copied over. +*/ +func (is *ImageStore) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return false, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + } else { + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + } + + binfo, err := is.storeDriver.Stat(blobPath) + if err == nil && binfo.Size() > 0 { + is.log.Debug().Str("blob path", blobPath).Msg("blob path found") + + return true, binfo.Size(), nil + } + // otherwise is a 'deduped' blob (empty file) + + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return false, -1, zerr.ErrBlobNotFound + } + + blobSize, err := is.copyBlob(repo, blobPath, dstRecord) + if err != nil { + return false, -1, zerr.ErrBlobNotFound + } + + // put deduped blob in cache + if err := is.cache.PutBlob(digest, blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") + + return false, -1, err + } + + return true, blobSize, nil +} + +// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. +func (is *ImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) { + if err := digest.Validate(); err != nil { + return false, -1, time.Time{}, err + } + + blobPath := is.BlobPath(repo, digest) + + binfo, err := is.storeDriver.Stat(blobPath) + if err == nil && binfo.Size() > 0 { + is.log.Debug().Str("blob path", blobPath).Msg("blob path found") + + return true, binfo.Size(), binfo.ModTime(), nil + } + + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + // then it's a 'deduped' blob + + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + binfo, err = is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + return true, binfo.Size(), binfo.ModTime(), nil +} + +func (is *ImageStore) checkCacheBlob(digest godigest.Digest) (string, error) { + if err := digest.Validate(); err != nil { + return "", err + } + + if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { + return "", zerr.ErrBlobNotFound + } + + dstRecord, err := is.cache.GetBlob(digest) + if err != nil { + return "", err + } + + if is.cache.UsesRelativePaths() { + dstRecord = path.Join(is.rootDir, dstRecord) + } + + if _, err := is.storeDriver.Stat(dstRecord); err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") + + // the actual blob on disk may have been removed by GC, so sync the cache + if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). + Msg("unable to remove blob path from cache") + + return "", err + } + + return "", zerr.ErrBlobNotFound + } + + is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") + + return dstRecord, nil +} + +func (is *ImageStore) copyBlob(repo string, blobPath, dstRecord string) (int64, error) { + if err := is.initRepo(repo); err != nil { + is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") + + return -1, err + } + + _ = is.storeDriver.EnsureDir(filepath.Dir(blobPath)) + + if err := is.storeDriver.Link(dstRecord, blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to hard link") + + return -1, zerr.ErrBlobNotFound + } + + // return original blob with content instead of the deduped one (blobPath) + binfo, err := is.storeDriver.Stat(dstRecord) + if err == nil { + return binfo.Size(), nil + } + + return -1, zerr.ErrBlobNotFound +} + +// GetBlobPartial returns a partial stream to read the blob. +// blob selector instead of directly downloading the blob. +func (is *ImageStore) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, +) (io.ReadCloser, int64, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return nil, -1, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + + // is a deduped blob + if binfo.Size() == 0 { + // Check blobs in cache + blobPath, err = is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + + binfo, err = is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + } + + end := to + + if to < 0 || to >= binfo.Size() { + end = binfo.Size() - 1 + } + + blobHandle, err := is.storeDriver.Reader(blobPath, from) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, -1, -1, err + } + + blobReadCloser, err := newBlobStream(blobHandle, from, end) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") + + return nil, -1, -1, err + } + + // The caller function is responsible for calling Close() + return blobReadCloser, end - from + 1, binfo.Size(), nil +} + +// GetBlob returns a stream to read the blob. +// blob selector instead of directly downloading the blob. +func (is *ImageStore) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return nil, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, zerr.ErrBlobNotFound + } + + blobReadCloser, err := is.storeDriver.Reader(blobPath, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, -1, err + } + + // is a 'deduped' blob? + if binfo.Size() == 0 { + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, -1, zerr.ErrBlobNotFound + } + + binfo, err := is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") + + return nil, -1, zerr.ErrBlobNotFound + } + + blobReadCloser, err := is.storeDriver.Reader(dstRecord, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") + + return nil, -1, err + } + + return blobReadCloser, binfo.Size(), nil + } + + // The caller function is responsible for calling Close() + return blobReadCloser, binfo.Size(), nil +} + +// GetBlobContent returns blob contents, the caller function SHOULD lock from outside. +func (is *ImageStore) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { + if err := digest.Validate(); err != nil { + return []byte{}, err + } + + blobPath := is.BlobPath(repo, digest) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return []byte{}, zerr.ErrBlobNotFound + } + + blobBuf, err := is.storeDriver.ReadFile(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, err + } + + // is a 'deduped' blob? + if binfo.Size() == 0 { + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, zerr.ErrBlobNotFound + } + + blobBuf, err := is.storeDriver.ReadFile(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") + + return nil, err + } + + return blobBuf, nil + } + + return blobBuf, nil +} + +func (is *ImageStore) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, +) (ispec.Index, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) +} + +func (is *ImageStore) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, +) ([]artifactspec.Descriptor, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) +} + +// GetIndexContent returns index.json contents, the caller function SHOULD lock from outside. +func (is *ImageStore) GetIndexContent(repo string) ([]byte, error) { + dir := path.Join(is.rootDir, repo) + + buf, err := is.storeDriver.ReadFile(path.Join(dir, "index.json")) + if err != nil { + if errors.Is(err, driver.PathNotFoundError{}) { + is.log.Error().Err(err).Str("dir", dir).Msg("index.json doesn't exist") + + return []byte{}, zerr.ErrRepoNotFound + } + + is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") + + return []byte{}, err + } + + return buf, nil +} + +// DeleteBlob removes the blob from the repository. +func (is *ImageStore) DeleteBlob(repo string, digest godigest.Digest) error { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return err + } + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + return is.deleteBlob(repo, digest) +} + +func (is *ImageStore) deleteBlob(repo string, digest godigest.Digest) error { + blobPath := is.BlobPath(repo, digest) + + _, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return zerr.ErrBlobNotFound + } + + // first check if this blob is not currently in use + if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { + return zerr.ErrBlobReferenced + } + + if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + dstRecord, err := is.cache.GetBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") + + return err + } + + // remove cache entry and move blob contents to the next candidate if there is any + if ok := is.cache.HasBlob(digest, blobPath); ok { + if err := is.cache.DeleteBlob(digest, blobPath); err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). + Msg("unable to remove blob path from cache") + + return err + } + } + + // if the deleted blob is one with content + if dstRecord == blobPath { + // get next candidate + dstRecord, err := is.cache.GetBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") + + return err + } + + // if we have a new candidate move the blob content to it + if dstRecord != "" { + if err := is.storeDriver.Move(blobPath, dstRecord); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") + + return err + } + + return nil + } + } + } + + if err := is.storeDriver.Delete(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") + + return err + } + + return nil +} + +type extendedManifest struct { + ispec.Manifest + + Digest godigest.Digest +} + +func (is *ImageStore) garbageCollect(repo string) error { + // gc untagged manifests and signatures + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return err + } + + referencedByImageIndex := []string{} + cosignDescriptors := []ispec.Descriptor{} + notationManifests := []extendedManifest{} + + /* gather manifests references by multiarch images (to skip gc) + gather cosign and notation signatures descriptors */ + for _, desc := range index.Manifests { + switch desc.MediaType { + case ispec.MediaTypeImageIndex: + indexImage, err := common.GetImageIndex(is, repo, desc.Digest, is.log) + if err != nil { + is.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read multiarch(index) image") + + return err + } + + for _, indexDesc := range indexImage.Manifests { + referencedByImageIndex = append(referencedByImageIndex, indexDesc.Digest.String()) + } + case ispec.MediaTypeImageManifest: + tag, ok := desc.Annotations[ispec.AnnotationRefName] + if ok { + // gather cosign references + if strings.HasPrefix(tag, "sha256-") && (strings.HasSuffix(tag, remote.SignatureTagSuffix) || + strings.HasSuffix(tag, remote.SBOMTagSuffix)) { + cosignDescriptors = append(cosignDescriptors, desc) + + continue + } + } + + manifestContent, err := common.GetImageManifest(is, repo, desc.Digest, is.log) + if err != nil { + is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation { + notationManifests = append(notationManifests, extendedManifest{ + Digest: desc.Digest, + Manifest: manifestContent, + }) + + continue + } + } + } + + is.log.Info().Msg("gc: untagged manifests") + + if err := gcUntaggedManifests(is, &index, repo, referencedByImageIndex); err != nil { + return err + } + + is.log.Info().Msg("gc: cosign references") + + if err := gcCosignReferences(is, &index, repo, cosignDescriptors); err != nil { + return err + } + + is.log.Info().Msg("gc: notation signatures") + + if err := gcNotationSignatures(is, &index, repo, notationManifests); err != nil { + return err + } + + is.log.Info().Msg("gc: blobs") + + if err := is.garbageCollectBlobs(is, repo, is.gcDelay, is.log); err != nil { + return err + } + + return nil +} + +func gcUntaggedManifests(imgStore *ImageStore, index *ispec.Index, repo string, + referencedByImageIndex []string, +) error { + for _, desc := range index.Manifests { + // skip manifests referenced in image indexex + if zcommon.Contains(referencedByImageIndex, desc.Digest.String()) { + continue + } + + // remove untagged images + if desc.MediaType == ispec.MediaTypeImageManifest { + _, ok := desc.Annotations[ispec.AnnotationRefName] + if !ok { + // check if is indeed an image and not an artifact by checking it's config blob + buf, err := imgStore.GetBlobContent(repo, desc.Digest) + if err != nil { + imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read image manifest") + + return err + } + + manifest := ispec.Manifest{} + + err = json.Unmarshal(buf, &manifest) + if err != nil { + return err + } + + // skip manifests which are not of type image + if manifest.Config.MediaType != ispec.MediaTypeImageConfig { + imgStore.log.Info().Str("config mediaType", manifest.Config.MediaType). + Msgf("skipping gc untagged manifest, because config blob is not %s", ispec.MediaTypeImageConfig) + + continue + } + + // remove manifest if it's older than gc.delay + canGC, err := isBlobOlderThan(imgStore, repo, desc.Digest, imgStore.gcDelay, imgStore.log) + if err != nil { + imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Str("delay", imgStore.gcDelay.String()).Msg("gc: failed to check if blob is older than delay") + + return err + } + + if canGC { + imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("gc: removing manifest without tag") + + if err := imgStore.deleteImageManifest(repo, desc.Digest.String(), true); err != nil { + if errors.Is(err, zerr.ErrManifestConflict) { + imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("gc: skipping removing manifest due to conflict") + + continue + } + + return err + } + } + } + } + } + + return nil +} + +func gcCosignReferences(imgStore *ImageStore, index *ispec.Index, repo string, + cosignDescriptors []ispec.Descriptor, +) error { + for _, cosignDesc := range cosignDescriptors { + foundSubject := false + // check if we can find the manifest which the reference points to + for _, desc := range index.Manifests { + // signature + subject := fmt.Sprintf("sha256-%s.%s", desc.Digest.Encoded(), remote.SignatureTagSuffix) + if subject == cosignDesc.Annotations[ispec.AnnotationRefName] { + foundSubject = true + } + + // sbom + subject = fmt.Sprintf("sha256-%s.%s", desc.Digest.Encoded(), remote.SBOMTagSuffix) + if subject == cosignDesc.Annotations[ispec.AnnotationRefName] { + foundSubject = true + } + } + + if !foundSubject { + // remove manifest + imgStore.log.Info().Str("repository", repo).Str("digest", cosignDesc.Digest.String()). + Msg("gc: removing cosign reference without subject") + + // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have + if err := imgStore.deleteImageManifest(repo, cosignDesc.Digest.String(), false); err != nil { + return err + } + } + } + + return nil +} + +func gcNotationSignatures(imgStore *ImageStore, index *ispec.Index, repo string, + notationManifests []extendedManifest, +) error { + for _, notationManifest := range notationManifests { + foundSubject := false + + for _, desc := range index.Manifests { + if desc.Digest == notationManifest.Subject.Digest { + foundSubject = true + } + } + + if !foundSubject { + // remove manifest + imgStore.log.Info().Str("repository", repo).Str("digest", notationManifest.Digest.String()). + Msg("gc: removing notation signature without subject") + + // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have + if err := imgStore.deleteImageManifest(repo, notationManifest.Digest.String(), false); err != nil { + return err + } + } + } + + return nil +} + +func isBlobOlderThan(imgStore storageTypes.ImageStore, repo string, + digest godigest.Digest, delay time.Duration, log zerolog.Logger, +) (bool, error) { + _, _, modtime, err := imgStore.StatBlob(repo, digest) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", digest.String()). + Msg("gc: failed to stat blob") + + return false, err + } + + if modtime.Add(delay).After(time.Now()) { + return false, nil + } + + log.Info().Str("repository", repo).Str("digest", digest.String()).Msg("perform GC on blob") + + return true, nil +} + +func (is *ImageStore) garbageCollectBlobs(imgStore *ImageStore, repo string, + delay time.Duration, log zerolog.Logger, +) error { + refBlobs := map[string]bool{} + + err := common.GetReferencedBlobs(imgStore, repo, refBlobs, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Msg("unable to get referenced blobs in repo") + + return err + } + + allBlobs, err := imgStore.GetAllBlobs(repo) + if err != nil { + log.Error().Err(err).Str("repository", repo).Msg("unable to get all blobs") + + return err + } + + reaped := 0 + + for _, blob := range allBlobs { + digest := godigest.NewDigestFromEncoded(godigest.SHA256, blob) + if err = digest.Validate(); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to parse digest") + + return err + } + + if _, ok := refBlobs[digest.String()]; !ok { + ok, err := isBlobOlderThan(imgStore, repo, digest, delay, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to determine GC delay") + + return err + } + + if !ok { + continue + } + + if err := imgStore.deleteBlob(repo, digest); err != nil { + if errors.Is(err, zerr.ErrBlobReferenced) { + if err := imgStore.deleteImageManifest(repo, digest.String(), true); err != nil { + if errors.Is(err, zerr.ErrManifestConflict) { + continue + } + + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to delete blob") + + return err + } + } else { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to delete blob") + + return err + } + } + + log.Info().Str("repository", repo).Str("digest", blob).Msg("garbage collected blob") + + reaped++ + } + } + + // if we cleaned all blobs let's also remove the repo so that it won't be returned by catalog + if reaped == len(allBlobs) { + if err := is.storeDriver.Delete(path.Join(is.rootDir, repo)); err != nil { + log.Error().Err(err).Str("repository", repo).Msg("unable to delete repo") + + return err + } + } + + log.Info().Str("repository", repo).Int("count", reaped).Msg("garbage collected blobs") + + return nil +} + +func (is *ImageStore) gcRepo(repo string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + err := is.garbageCollect(repo) + is.Unlock(&lockLatency) + + if err != nil { + return err + } + + return nil +} + +func (is *ImageStore) GetAllBlobs(repo string) ([]string, error) { + dir := path.Join(is.rootDir, repo, "blobs", "sha256") + + files, err := is.storeDriver.List(dir) + if err != nil { + return []string{}, err + } + + ret := []string{} + + for _, file := range files { + ret = append(ret, filepath.Base(file)) + } + + return ret, nil +} + +func (is *ImageStore) RunGCRepo(repo string) error { + is.log.Info().Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(is.RootDir(), repo))) + + if err := is.gcRepo(repo); err != nil { + errMessage := fmt.Sprintf("error while running GC for %s", path.Join(is.RootDir(), repo)) + is.log.Error().Err(err).Msg(errMessage) + is.log.Info().Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(is.RootDir(), repo))) + + return err + } + + is.log.Info().Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(is.RootDir(), repo))) + + return nil +} + +func (is *ImageStore) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { + generator := &taskGenerator{ + imgStore: is, + } + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) +} + +type taskGenerator struct { + imgStore *ImageStore + lastRepo string + done bool +} + +func (gen *taskGenerator) Next() (scheduler.Task, error) { + repo, err := gen.imgStore.GetNextRepository(gen.lastRepo) + + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + + if repo == "" { + gen.done = true + + return nil, nil + } + + gen.lastRepo = repo + + return newGCTask(gen.imgStore, repo), nil +} + +func (gen *taskGenerator) IsDone() bool { + return gen.done +} + +func (gen *taskGenerator) Reset() { + gen.lastRepo = "" + gen.done = false +} + +type gcTask struct { + imgStore *ImageStore + repo string +} + +func newGCTask(imgStore *ImageStore, repo string) *gcTask { + return &gcTask{imgStore, repo} +} + +func (gcT *gcTask) DoWork() error { + return gcT.imgStore.RunGCRepo(gcT.repo) +} + +func (is *ImageStore) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + var duplicateBlobs []string + + var digest godigest.Digest + + err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + // skip blobs under .sync + if strings.HasSuffix(fileInfo.Path(), syncConstants.SyncBlobUploadDir) { + return driver.ErrSkipDir + } + + if fileInfo.IsDir() { + return nil + } + + blobDigest := godigest.NewDigestFromEncoded("sha256", path.Base(fileInfo.Path())) + if err := blobDigest.Validate(); err != nil { + return nil //nolint:nilerr // ignore files which are not blobs + } + + if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { + digest = blobDigest + } + + if blobDigest == digest { + duplicateBlobs = append(duplicateBlobs, fileInfo.Path()) + } + + return nil + }) + + // if the root directory is not yet created + var perr driver.PathNotFoundError + + if errors.As(err, &perr) { + return digest, duplicateBlobs, nil + } + + return digest, duplicateBlobs, err +} + +func (is *ImageStore) getOriginalBlobFromDisk(duplicateBlobs []string) (string, error) { + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return "", zerr.ErrBlobNotFound + } + + if binfo.Size() > 0 { + return blobPath, nil + } + } + + return "", zerr.ErrBlobNotFound +} + +func (is *ImageStore) getOriginalBlob(digest godigest.Digest, duplicateBlobs []string) (string, error) { + var originalBlob string + + var err error + + originalBlob, err = is.checkCacheBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrBlobNotFound) && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find blob in cache") + + return originalBlob, err + } + + // if we still don't have, search it + if originalBlob == "" { + is.log.Warn().Msg("rebuild dedupe: failed to find blob in cache, searching it in s3...") + // a rebuild dedupe was attempted in the past + // get original blob, should be found otherwise exit with error + + originalBlob, err = is.getOriginalBlobFromDisk(duplicateBlobs) + if err != nil { + return originalBlob, err + } + } + + is.log.Info().Str("originalBlob", originalBlob).Msg("rebuild dedupe: found original blob") + + return originalBlob, nil +} + +func (is *ImageStore) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { + if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { + is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") + + return zerr.ErrDedupeRebuild + } + + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") + + var originalBlob string + + // rebuild from dedupe false to true + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return err + } + + if binfo.Size() == 0 { + is.log.Warn().Msg("rebuild dedupe: found file without content, trying to find the original blob") + // a rebuild dedupe was attempted in the past + // get original blob, should be found otherwise exit with error + if originalBlob == "" { + originalBlob, err = is.getOriginalBlob(digest, duplicateBlobs) + if err != nil { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") + + return zerr.ErrDedupeRebuild + } + + // cache original blob + if ok := is.cache.HasBlob(digest, originalBlob); !ok { + if err := is.cache.PutBlob(digest, originalBlob); err != nil { + return err + } + } + } + + // cache dedupe blob + if ok := is.cache.HasBlob(digest, blobPath); !ok { + if err := is.cache.PutBlob(digest, blobPath); err != nil { + return err + } + } + } else { + // if we have an original blob cached then we can safely dedupe the rest of them + if originalBlob != "" { + if err := is.storeDriver.Link(originalBlob, blobPath); err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: unable to dedupe blob") + + return err + } + } + + // cache it + if ok := is.cache.HasBlob(digest, blobPath); !ok { + if err := is.cache.PutBlob(digest, blobPath); err != nil { + return err + } + } + + // mark blob as preserved + originalBlob = blobPath + } + } + + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") + + return nil +} + +func (is *ImageStore) restoreDedupedBlobs(digest godigest.Digest, duplicateBlobs []string) error { + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: restoring deduped blobs for digest") + + // first we need to find the original blob, either in cache or by checking each blob size + originalBlob, err := is.getOriginalBlob(digest, duplicateBlobs) + if err != nil { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") + + return zerr.ErrDedupeRebuild + } + + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return err + } + + // if we find a deduped blob, then copy original blob content to deduped one + if binfo.Size() == 0 { + // move content from original blob to deduped one + buf, err := is.storeDriver.ReadFile(originalBlob) + if err != nil { + is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to get original blob content") + + return err + } + + _, err = is.storeDriver.WriteFile(blobPath, buf) + if err != nil { + return err + } + } + } + + is.log.Info().Str("digest", digest.String()). + Msg("rebuild dedupe: restoring deduped blobs for digest finished successfully") + + return nil +} + +func (is *ImageStore) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + if dedupe { + return is.dedupeBlobs(digest, duplicateBlobs) + } + + return is.restoreDedupedBlobs(digest, duplicateBlobs) +} + +func (is *ImageStore) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { + generator := &common.DedupeTaskGenerator{ + ImgStore: is, + Dedupe: is.dedupe, + Log: is.log, + } + + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) +} + +type blobStream struct { + reader io.Reader + closer io.Closer +} + +func newBlobStream(readCloser io.ReadCloser, from, to int64) (io.ReadCloser, error) { + if from < 0 || to < from { + return nil, zerr.ErrBadRange + } + + return &blobStream{reader: io.LimitReader(readCloser, to-from+1), closer: readCloser}, nil +} + +func (bs *blobStream) Read(buf []byte) (int, error) { + return bs.reader.Read(buf) +} + +func (bs *blobStream) Close() error { + return bs.closer.Close() +} diff --git a/pkg/storage/local/driver.go b/pkg/storage/local/driver.go new file mode 100644 index 0000000000..87429657e9 --- /dev/null +++ b/pkg/storage/local/driver.go @@ -0,0 +1,456 @@ +package local + +import ( + "bufio" + "bytes" + "errors" + "io" + "io/fs" + "os" + "path" + "sort" + "syscall" + "time" + "unicode/utf8" + + storagedriver "github.com/docker/distribution/registry/storage/driver" + "github.com/sirupsen/logrus" + + zerr "zotregistry.io/zot/errors" + storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/test/inject" +) + +type Driver struct { + commit bool +} + +func New(commit bool) *Driver { + return &Driver{commit: commit} +} + +func (driver *Driver) Name() string { + return storageConstants.LocalStorageDriverName +} + +func (driver *Driver) EnsureDir(path string) error { + return os.MkdirAll(path, storageConstants.DefaultDirPerms) +} + +func (driver *Driver) DirExists(path string) bool { + if !utf8.ValidString(path) { + return false + } + + fileInfo, err := os.Stat(path) + if err != nil { + if e, ok := err.(*fs.PathError); ok && errors.Is(e.Err, syscall.ENAMETOOLONG) || //nolint: errorlint + errors.Is(e.Err, syscall.EINVAL) { + return false + } + } + + if err != nil && os.IsNotExist(err) { + return false + } + + if !fileInfo.IsDir() { + return false + } + + return true +} + +func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) { + file, err := os.OpenFile(path, os.O_RDONLY, storageConstants.DefaultFilePerms) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: path} + } + + return nil, err + } + + seekPos, err := file.Seek(offset, io.SeekStart) + if err != nil { + file.Close() + + return nil, err + } else if seekPos < offset { + file.Close() + + return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset} + } + + return file, nil +} + +func (driver *Driver) ReadFile(path string) ([]byte, error) { + reader, err := driver.Reader(path, 0) + if err != nil { + return nil, err + } + + defer reader.Close() + + buf, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (driver *Driver) Delete(path string) error { + _, err := os.Stat(path) + if err != nil && !os.IsNotExist(err) { + return err + } else if err != nil { + return storagedriver.PathNotFoundError{Path: path} + } + + return os.RemoveAll(path) +} + +func (driver *Driver) Stat(path string) (storagedriver.FileInfo, error) { + fi, err := os.Stat(path) //nolint: varnamelen + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: path} + } + + return nil, err + } + + return fileInfo{ + path: path, + FileInfo: fi, + }, nil +} + +func (driver *Driver) Writer(filepath string, append bool) (storagedriver.FileWriter, error) { //nolint:predeclared + if append { + _, err := os.Stat(filepath) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: filepath} + } + + return nil, err + } + } + + parentDir := path.Dir(filepath) + if err := os.MkdirAll(parentDir, storageConstants.DefaultDirPerms); err != nil { + return nil, err + } + + file, err := os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) + if err != nil { + return nil, err + } + + var offset int64 + + if !append { + err := file.Truncate(0) + if err != nil { + file.Close() + + return nil, err + } + } else { + n, err := file.Seek(0, io.SeekEnd) //nolint: varnamelen + if err != nil { + file.Close() + + return nil, err + } + offset = n + } + + return newFileWriter(file, offset, driver.commit), nil +} + +func (driver *Driver) WriteFile(filepath string, contents []byte) (int, error) { + writer, err := driver.Writer(filepath, false) + if err != nil { + return -1, err + } + + nbytes, err := io.Copy(writer, bytes.NewReader(contents)) + if err != nil { + _ = writer.Cancel() + + return -1, err + } + + return int(nbytes), writer.Close() +} + +func (driver *Driver) Walk(path string, walkFn storagedriver.WalkFn) error { + children, err := driver.List(path) + if err != nil { + return err + } + + sort.Stable(sort.StringSlice(children)) + + for _, child := range children { + // Calling driver.Stat for every entry is quite + // expensive when running against backends with a slow Stat + // implementation, such as s3. This is very likely a serious + // performance bottleneck. + fileInfo, err := driver.Stat(child) + if err != nil { + switch errors.As(err, &storagedriver.PathNotFoundError{}) { + case true: + // repository was removed in between listing and enumeration. Ignore it. + logrus.WithField("path", child).Infof("ignoring deleted path") + + continue + default: + return err + } + } + err = walkFn(fileInfo) + //nolint: gocritic + if err == nil && fileInfo.IsDir() { + if err := driver.Walk(child, walkFn); err != nil { + return err + } + } else if errors.Is(err, storagedriver.ErrSkipDir) { + // Stop iteration if it's a file, otherwise noop if it's a directory + if !fileInfo.IsDir() { + return nil + } + } else if err != nil { + return err + } + } + + return nil +} + +func (driver *Driver) List(fullpath string) ([]string, error) { + dir, err := os.Open(fullpath) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: fullpath} + } + + return nil, err + } + + defer dir.Close() + + fileNames, err := dir.Readdirnames(0) + if err != nil { + return nil, err + } + + keys := make([]string, 0, len(fileNames)) + for _, fileName := range fileNames { + keys = append(keys, path.Join(fullpath, fileName)) + } + + return keys, nil +} + +func (driver *Driver) Move(sourcePath string, destPath string) error { + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + return storagedriver.PathNotFoundError{Path: sourcePath} + } + + if err := os.MkdirAll(path.Dir(destPath), storageConstants.DefaultDirPerms); err != nil { + return err + } + + return os.Rename(sourcePath, destPath) +} + +func (driver *Driver) SameFile(path1, path2 string) bool { + file1, err := os.Stat(path1) + if err != nil { + return false + } + + file2, err := os.Stat(path2) + if err != nil { + return false + } + + return os.SameFile(file1, file2) +} + +func (driver *Driver) Link(src, dest string) error { + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return err + } + + return os.Link(src, dest) +} + +type fileInfo struct { + os.FileInfo + path string +} + +// asserts fileInfo implements storagedriver.FileInfo. +var _ storagedriver.FileInfo = fileInfo{} + +// Path provides the full path of the target of this file info. +func (fi fileInfo) Path() string { + return fi.path +} + +// Size returns current length in bytes of the file. The return value can +// be used to write to the end of the file at path. The value is +// meaningless if IsDir returns true. +func (fi fileInfo) Size() int64 { + if fi.IsDir() { + return 0 + } + + return fi.FileInfo.Size() +} + +// ModTime returns the modification time for the file. For backends that +// don't have a modification time, the creation time should be returned. +func (fi fileInfo) ModTime() time.Time { + return fi.FileInfo.ModTime() +} + +// IsDir returns true if the path is a directory. +func (fi fileInfo) IsDir() bool { + return fi.FileInfo.IsDir() +} + +type fileWriter struct { + file *os.File + size int64 + bw *bufio.Writer + closed bool + committed bool + cancelled bool + commit bool +} + +func newFileWriter(file *os.File, size int64, commit bool) *fileWriter { + return &fileWriter{ + file: file, + size: size, + commit: commit, + bw: bufio.NewWriter(file), + } +} + +func (fw *fileWriter) Write(buf []byte) (int, error) { + //nolint: gocritic + if fw.closed { + return 0, zerr.ErrFileAlreadyClosed + } else if fw.committed { + return 0, zerr.ErrFileAlreadyCommitted + } else if fw.cancelled { + return 0, zerr.ErrFileAlreadyCancelled + } + + n, err := fw.bw.Write(buf) + fw.size += int64(n) + + return n, err +} + +func (fw *fileWriter) Size() int64 { + return fw.size +} + +func (fw *fileWriter) Close() error { + if fw.closed { + return zerr.ErrFileAlreadyClosed + } + + if err := fw.bw.Flush(); err != nil { + return err + } + + if fw.commit { + if err := inject.Error(fw.file.Sync()); err != nil { + return err + } + } + + if err := inject.Error(fw.file.Close()); err != nil { + return err + } + + fw.closed = true + + return nil +} + +func (fw *fileWriter) Cancel() error { + if fw.closed { + return zerr.ErrFileAlreadyClosed + } + + fw.cancelled = true + fw.file.Close() + + return os.Remove(fw.file.Name()) +} + +func (fw *fileWriter) Commit() error { + //nolint: gocritic + if fw.closed { + return zerr.ErrFileAlreadyClosed + } else if fw.committed { + return zerr.ErrFileAlreadyCommitted + } else if fw.cancelled { + return zerr.ErrFileAlreadyCancelled + } + + if err := fw.bw.Flush(); err != nil { + return err + } + + if fw.commit { + if err := fw.file.Sync(); err != nil { + return err + } + } + + fw.committed = true + + return nil +} + +func ValidateHardLink(rootDir string) error { + if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { + return err + } + + err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), + []byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms) + if err != nil { + return err + } + + err = os.Link(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt")) + if err != nil { + // Remove hardlinkcheck.txt if hardlink fails + zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) + if zerr != nil { + return zerr + } + + return err + } + + err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) + if err != nil { + return err + } + + return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt")) +} diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index 74707952b3..2d333f5f9a 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -1,2028 +1,32 @@ package local import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - "sync" "time" - "unicode/utf8" - apexlog "github.com/apex/log" - guuid "github.com/gofrs/uuid" - "github.com/minio/sha256-simd" - notreg "github.com/notaryproject/notation-go/registry" - godigest "github.com/opencontainers/go-digest" - imeta "github.com/opencontainers/image-spec/specs-go" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/opencontainers/umoci" - "github.com/opencontainers/umoci/oci/casext" - oras "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" - "github.com/sigstore/cosign/v2/pkg/oci/remote" - - zerr "zotregistry.io/zot/errors" - zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" - zreg "zotregistry.io/zot/pkg/regexp" - "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage/cache" common "zotregistry.io/zot/pkg/storage/common" - storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" storageTypes "zotregistry.io/zot/pkg/storage/types" - "zotregistry.io/zot/pkg/test/inject" ) -// ImageStoreLocal provides the image storage operations. -type ImageStoreLocal struct { - rootDir string - lock *sync.RWMutex - cache cache.Cache - gc bool - dedupe bool - commit bool - gcDelay time.Duration - log zerolog.Logger - metrics monitoring.MetricServer - linter common.Lint -} - -func (is *ImageStoreLocal) RootDir() string { - return is.rootDir -} - -func (is *ImageStoreLocal) DirExists(d string) bool { - return zcommon.DirExists(d) -} - // NewImageStore returns a new image store backed by a file storage. // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache, ) storageTypes.ImageStore { - if _, err := os.Stat(rootDir); os.IsNotExist(err) { - if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { - log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") - - return nil - } - } - - imgStore := &ImageStoreLocal{ - rootDir: rootDir, - lock: &sync.RWMutex{}, - gc: gc, - gcDelay: gcDelay, - dedupe: dedupe, - commit: commit, - log: log.With().Caller().Logger(), - metrics: metrics, - linter: linter, - } - - imgStore.cache = cacheDriver - - if gc { - // we use umoci GC to perform garbage-collection, but it uses its own logger - // - so capture those logs, could be useful - apexlog.SetLevel(apexlog.DebugLevel) - apexlog.SetHandler(apexlog.HandlerFunc(func(entry *apexlog.Entry) error { - e := log.Debug() - for k, v := range entry.Fields { - e = e.Interface(k, v) - } - e.Msg(entry.Message) - - return nil - })) - } - - return imgStore -} - -// RLock read-lock. -func (is *ImageStoreLocal) RLock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.RLock() -} - -// RUnlock read-unlock. -func (is *ImageStoreLocal) RUnlock(lockStart *time.Time) { - is.lock.RUnlock() - - lockEnd := time.Now() - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram -} - -// Lock write-lock. -func (is *ImageStoreLocal) Lock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.Lock() -} - -// Unlock write-unlock. -func (is *ImageStoreLocal) Unlock(lockStart *time.Time) { - is.lock.Unlock() - - lockEnd := time.Now() - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram -} - -func (is *ImageStoreLocal) initRepo(name string) error { - repoDir := path.Join(is.rootDir, name) - - if !utf8.ValidString(name) { - is.log.Error().Msg("input is not valid UTF-8") - - return zerr.ErrInvalidRepositoryName - } - - if !zreg.FullNameRegexp.MatchString(name) { - is.log.Error().Str("repository", name).Msg("invalid repository name") - - return zerr.ErrInvalidRepositoryName - } - - // create "blobs" subdir - err := ensureDir(path.Join(repoDir, "blobs"), is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blobs subdir") - - return err - } - // create BlobUploadDir subdir - err = ensureDir(path.Join(repoDir, storageConstants.BlobUploadDir), is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blob upload subdir") - - return err - } - - // "oci-layout" file - create if it doesn't exist - ilPath := path.Join(repoDir, ispec.ImageLayoutFile) - if _, err := os.Stat(ilPath); err != nil { - il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} - - buf, err := json.Marshal(il) - if err != nil { - is.log.Panic().Err(err).Msg("unable to marshal JSON") - } - - if err := is.writeFile(ilPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - // "index.json" file - create if it doesn't exist - indexPath := path.Join(repoDir, "index.json") - if _, err := os.Stat(indexPath); err != nil { - index := ispec.Index{Versioned: imeta.Versioned{SchemaVersion: storageConstants.SchemaVersion}} - - buf, err := json.Marshal(index) - if err != nil { - is.log.Panic().Err(err).Msg("unable to marshal JSON") - } - - if err := is.writeFile(indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", indexPath).Msg("unable to write file") - - return err - } - } - - return nil -} - -// InitRepo creates an image repository under this store. -func (is *ImageStoreLocal) InitRepo(name string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - return is.initRepo(name) -} - -// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. -func (is *ImageStoreLocal) ValidateRepo(name string) (bool, error) { - // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content - // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] - // and an additional/optional BlobUploadDir in each image store - if !zreg.FullNameRegexp.MatchString(name) { - return false, zerr.ErrInvalidRepositoryName - } - - dir := path.Join(is.rootDir, name) - if !is.DirExists(dir) { - return false, zerr.ErrRepoNotFound - } - - files, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") - - return false, zerr.ErrRepoNotFound - } - - if len(files) < 3 { //nolint:gomnd - return false, zerr.ErrRepoBadVersion - } - - found := map[string]bool{ - "blobs": false, - ispec.ImageLayoutFile: false, - "index.json": false, - } - - for _, file := range files { - if file.Name() == "blobs" && !file.IsDir() { - return false, nil - } - - found[file.Name()] = true - } - - for k, v := range found { - if !v && k != storageConstants.BlobUploadDir { - return false, nil - } - } - - buf, err := os.ReadFile(path.Join(dir, ispec.ImageLayoutFile)) - if err != nil { - return false, err - } - - var il ispec.ImageLayout - if err := json.Unmarshal(buf, &il); err != nil { - return false, err - } - - if il.Version != ispec.ImageLayoutVersion { - return false, zerr.ErrRepoBadVersion - } - - return true, nil -} - -// GetRepositories returns a list of all the repositories under this store. -func (is *ImageStoreLocal) GetRepositories() ([]string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - _, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Msg("failure walking storage root-dir") - - return nil, err - } - - stores := make([]string, 0) - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, path) - if err != nil { - return nil //nolint:nilerr // ignore paths not relative to root dir - } - - if ok, err := is.ValidateRepo(rel); !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - // is.log.Debug().Str("dir", path).Str("name", info.Name()).Msg("found image store") - stores = append(stores, rel) - - return nil - }) - - return stores, err -} - -// GetNextRepository returns next repository under this store. -func (is *ImageStoreLocal) GetNextRepository(repo string) (string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - _, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Msg("failure walking storage root-dir") - - return "", err - } - - found := false - store := "" - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, path) - if err != nil { - return nil //nolint:nilerr // ignore paths not relative to root dir - } - - ok, err := is.ValidateRepo(rel) - if !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - if repo == "" && ok && err == nil { - store = rel - - return io.EOF - } - - if found { - store = rel - - return io.EOF - } - - if rel == repo { - found = true - } - - return nil - }) - - return store, err -} - -// GetImageTags returns a list of image tags available in the specified repository. -func (is *ImageStoreLocal) GetImageTags(repo string) ([]string, error) { - var lockLatency time.Time - - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return nil, zerr.ErrRepoNotFound - } - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, err - } - - return common.GetTagsByIndex(index), nil -} - -// GetImageManifest returns the image manifest of an image in the specific repository. -func (is *ImageStoreLocal) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return nil, "", "", zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.RLock(&lockLatency) - defer func() { - is.RUnlock(&lockLatency) - - if err == nil { - monitoring.IncDownloadCounter(is.metrics, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, "", "", err - } - - manifestDesc, found := common.GetManifestDescByReference(index, reference) - if !found { - return nil, "", "", zerr.ErrManifestNotFound - } - - buf, err := is.GetBlobContent(repo, manifestDesc.Digest) - if err != nil { - if errors.Is(err, zerr.ErrBlobNotFound) { - return nil, "", "", zerr.ErrManifestNotFound - } - - return nil, "", "", err - } - - var manifest ispec.Manifest - if err := json.Unmarshal(buf, &manifest); err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") - - return nil, "", "", err - } - - return buf, manifestDesc.Digest, manifestDesc.MediaType, nil -} - -// PutImageManifest adds an image manifest to the repository. -func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo - body []byte, -) (godigest.Digest, godigest.Digest, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Debug().Err(err).Msg("init repo") - - return "", "", err - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - monitoring.IncUploadCounter(is.metrics, repo) - } - }() - - refIsDigest := true - - mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) - if err != nil { - if errors.Is(err, zerr.ErrBadManifest) { - return mDigest, "", err - } - - refIsDigest = false - } - - digest, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) - if err != nil { - return digest, "", err - } - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return "", "", err - } - - // create a new descriptor - desc := ispec.Descriptor{ - MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, - } - - if !refIsDigest { - desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} - } - - var subjectDigest godigest.Digest - - artifactType := "" - - if mediaType == ispec.MediaTypeImageManifest { - var manifest ispec.Manifest - - err := json.Unmarshal(body, &manifest) - if err != nil { - return "", "", err - } - - if manifest.Subject != nil { - subjectDigest = manifest.Subject.Digest - } - - artifactType = zcommon.GetManifestArtifactType(manifest) - } else if mediaType == ispec.MediaTypeImageIndex { - var index ispec.Index - - err := json.Unmarshal(body, &index) - if err != nil { - return "", "", err - } - - if index.Subject != nil { - subjectDigest = index.Subject.Digest - } - - artifactType = zcommon.GetIndexArtifactType(index) - } - - updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) - if err != nil { - return "", "", err - } - - if !updateIndex { - return desc.Digest, subjectDigest, nil - } - - // write manifest to "blobs" - dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) - _ = ensureDir(dir, is.log) - file := path.Join(dir, mDigest.Encoded()) - - // in case the linter will not pass, it will be garbage collected - if err := is.writeFile(file, body); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to write") - - return "", "", err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) - if err != nil { - return "", "", err - } - - // now update "index.json" - index.Manifests = append(index.Manifests, desc) - dir = path.Join(is.rootDir, repo) - file = path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to marshal JSON") - - return "", "", err - } - - // update the descriptors artifact type in order to check for signatures when applying the linter - desc.ArtifactType = artifactType - - // apply linter only on images, not signatures or indexes - pass, err := common.ApplyLinter(is, is.linter, repo, desc) - if !pass { - is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") - - return "", "", err - } - - err = is.writeFile(file, buf) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to write") - - return "", "", err - } - - if is.gc { - if err := is.garbageCollect(dir, repo); err != nil { - return "", "", err - } - } - - return desc.Digest, subjectDigest, nil -} - -// DeleteImageManifest deletes the image manifest from the repository. -func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string, detectCollision bool) error { - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return err - } - - manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollision) - if err != nil { - return err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) - if err != nil { - return err - } - - // now update "index.json" - dir = path.Join(is.rootDir, repo) - file := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - return err - } - - if err := is.writeFile(file, buf); err != nil { - return err - } - - if is.gc { - if err := is.garbageCollect(dir, repo); err != nil { - return err - } - } - - // Delete blob only when blob digest not present in manifest entry. - // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. - toDelete := true - - for _, manifest := range index.Manifests { - if manifestDesc.Digest.String() == manifest.Digest.String() { - toDelete = false - - break - } - } - - if toDelete { - p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) - - _ = os.Remove(p) - } - - return nil -} - -// BlobUploadPath returns the upload path for a blob in this store. -func (is *ImageStoreLocal) BlobUploadPath(repo, uuid string) string { - dir := path.Join(is.rootDir, repo) - blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) - - return blobUploadPath -} - -// NewBlobUpload returns the unique ID for an upload in progress. -func (is *ImageStoreLocal) NewBlobUpload(repo string) (string, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Error().Err(err).Msg("error initializing repo") - - return "", err - } - - uuid, err := guuid.NewV4() - if err != nil { - return "", err - } - - uid := uuid.String() - - blobUploadPath := is.BlobUploadPath(repo, uid) - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - return "", zerr.ErrRepoNotFound - } - - defer file.Close() - - return uid, nil -} - -// GetBlobUpload returns the current size of a blob upload. -func (is *ImageStoreLocal) GetBlobUpload(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - if !utf8.ValidString(blobUploadPath) { - is.log.Error().Msg("input is not valid UTF-8") - - return -1, zerr.ErrInvalidRepositoryName - } - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - if os.IsNotExist(err) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return binfo.Size(), nil -} - -// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ImageStoreLocal) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - _, err := os.Stat(blobUploadPath) - if err != nil { - return -1, zerr.ErrUploadNotFound - } - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - is.log.Error().Err(err).Msg("failed to open file") - - return -1, err - } - - defer func() { - if is.commit { - _ = file.Sync() - } - - _ = file.Close() - }() - - if _, err := file.Seek(0, io.SeekEnd); err != nil { - is.log.Error().Err(err).Msg("failed to seek file") - - return -1, err - } - - n, err := io.Copy(file, body) - - return n, err -} - -// PutBlobChunk writes another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ImageStoreLocal) PutBlobChunk(repo, uuid string, from, to int64, - body io.Reader, -) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - return -1, zerr.ErrUploadNotFound - } - - if from != binfo.Size() { - is.log.Error().Int64("expected", from).Int64("actual", binfo.Size()). - Msg("invalid range start for blob upload") - - return -1, zerr.ErrBadUploadRange - } - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - is.log.Error().Err(err).Msg("failed to open file") - - return -1, err - } - - defer func() { - if is.commit { - _ = file.Sync() - } - - _ = file.Close() - }() - - if _, err := file.Seek(from, io.SeekStart); err != nil { - is.log.Error().Err(err).Msg("failed to seek file") - - return -1, err - } - - n, err := io.Copy(file, body) - - return n, err -} - -// BlobUploadInfo returns the current blob size in bytes. -func (is *ImageStoreLocal) BlobUploadInfo(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobUploadPath).Msg("failed to stat blob") - - return -1, err - } - - size := binfo.Size() - - return size, nil -} - -// FinishBlobUpload finalizes the blob upload and moves blob the repository. -func (is *ImageStoreLocal) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { - if err := dstDigest.Validate(); err != nil { - return err - } - - src := is.BlobUploadPath(repo, uuid) - - _, err := os.Stat(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to stat blob") - - return zerr.ErrUploadNotFound - } - - blobFile, err := os.Open(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrUploadNotFound - } - - defer blobFile.Close() - - digester := sha256.New() - - _, err = io.Copy(digester, blobFile) - if err != nil { - is.log.Error().Err(err).Str("repository", repo).Str("blob", src).Str("digest", dstDigest.String()). - Msg("unable to compute hash") - - return err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return zerr.ErrBadBlobDigest - } - - dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - err = ensureDir(dir, is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blobs/sha256 dir") - - return err - } - - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - err = is.DedupeBlob(src, dstDigest, dst) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return err - } - } else { - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return err - } - } - - return nil -} - -// FullBlobUpload handles a full blob upload, and no partial session is created. -func (is *ImageStoreLocal) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest, -) (string, int64, error) { - if err := dstDigest.Validate(); err != nil { - return "", -1, err - } - - if err := is.InitRepo(repo); err != nil { - return "", -1, err - } - - u, err := guuid.NewV4() - if err != nil { - return "", -1, err - } - - uuid := u.String() - - src := is.BlobUploadPath(repo, uuid) - - blobFile, err := os.Create(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return "", -1, zerr.ErrUploadNotFound - } - - defer func() { - if is.commit { - _ = blobFile.Sync() - } - - _ = blobFile.Close() - }() - - digester := sha256.New() - mw := io.MultiWriter(blobFile, digester) - - nbytes, err := io.Copy(mw, body) - if err != nil { - return "", -1, err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return "", -1, zerr.ErrBadBlobDigest - } - - dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _ = ensureDir(dir, is.log) - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return "", -1, err - } - } else { - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return "", -1, err - } - } - - return uuid, nbytes, nil -} - -func (is *ImageStoreLocal) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { -retry: - is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") - - dstRecord, err := is.cache.GetBlob(dstDigest) - - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") - - return err - } - - if dstRecord == "" { - // cache record doesn't exist, so first disk and cache entry for this diges - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - // move the blob from uploads to final dest - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") - - return err - } - - is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") - } else { - // cache record exists, but due to GC and upgrades from older versions, - // disk content and cache records may go out of sync - dstRecord = path.Join(is.rootDir, dstRecord) - - dstRecordFi, err := os.Stat(dstRecord) - if err != nil { - is.log.Warn().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat cache record, removing it") - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(dstDigest, dstRecord); err != nil { - //nolint:lll // gofumpt conflicts with lll - is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") - - return err - } - - goto retry - } - - dstFi, err := os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - - return err - } - - if !os.SameFile(dstFi, dstRecordFi) { - // blob lookup cache out of sync with actual disk contents - if err := os.Remove(dst); err != nil && !os.IsNotExist(err) { - is.log.Error().Err(err).Str("dst", dst).Msg("dedupe: unable to remove blob") - - return err - } - - is.log.Debug().Str("blobPath", dst).Str("dstRecord", dstRecord).Msg("dedupe: creating hard link") - - if err := os.Link(dstRecord, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("dedupe: unable to hard link") - - return err - } - } - - // also put dedupe blob in cache - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - if err := os.Remove(src); err != nil { - is.log.Error().Err(err).Str("src", src).Msg("dedupe: uname to remove blob") - - return err - } - - is.log.Debug().Str("src", src).Msg("dedupe: remove") - } - - return nil -} - -// DeleteBlobUpload deletes an existing blob upload that is currently in progress. -func (is *ImageStoreLocal) DeleteBlobUpload(repo, uuid string) error { - blobUploadPath := is.BlobUploadPath(repo, uuid) - if err := os.Remove(blobUploadPath); err != nil { - is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") - - return err - } - - return nil -} - -// BlobPath returns the repository path of a blob. -func (is *ImageStoreLocal) BlobPath(repo string, digest godigest.Digest) string { - return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) -} - -/* - CheckBlob verifies a blob and returns true if the blob is correct - -If the blob is not found but it's found in cache then it will be copied over. -*/ -func (is *ImageStoreLocal) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return false, -1, err - } - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - } else { - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - } - - if ok, size, err := is.StatBlob(repo, digest); err == nil || ok { - return true, size, nil - } - - blobPath := is.BlobPath(repo, digest) - - is.log.Debug().Str("blob", blobPath).Msg("failed to find blob, searching it in cache") - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - // If found copy to location - blobSize, err := is.copyBlob(repo, blobPath, dstRecord) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - if err := is.cache.PutBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") - - return false, -1, err - } - - return true, blobSize, nil -} - -// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. -func (is *ImageStoreLocal) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Str("blob path", blobPath).Msg("failed to find blob") - - return false, -1, zerr.ErrBlobNotFound - } - - return true, binfo.Size(), nil -} - -func (is *ImageStoreLocal) checkCacheBlob(digest godigest.Digest) (string, error) { - if err := digest.Validate(); err != nil { - return "", err - } - - if !is.dedupe || fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - return "", zerr.ErrBlobNotFound - } - - dstRecord, err := is.cache.GetBlob(digest) - if err != nil { - if errors.Is(err, zerr.ErrCacheMiss) { - is.log.Debug().Err(err).Str("digest", string(digest)).Msg("unable to find blob in cache") - } else { - is.log.Error().Err(err).Str("digest", string(digest)).Msg("unable to search blob in cache") - } - - return "", err - } - - dstRecord = path.Join(is.rootDir, dstRecord) - - if _, err := os.Stat(dstRecord); err != nil { - is.log.Warn().Err(err).Str("blob", dstRecord).Msg("unable to stat cache record, removing it") - - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). - Msg("unable to remove blob path from cache") - - return "", err - } - - return "", zerr.ErrBlobNotFound - } - - is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") - - return dstRecord, nil -} - -func (is *ImageStoreLocal) copyBlob(repo, blobPath, dstRecord string) (int64, error) { - if err := is.initRepo(repo); err != nil { - is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") - - return -1, err - } - - _ = ensureDir(filepath.Dir(blobPath), is.log) - - if err := os.Link(dstRecord, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to hard link") - - return -1, zerr.ErrBlobNotFound - } - - binfo, err := os.Stat(blobPath) - if err == nil { - return binfo.Size(), nil - } - - return -1, zerr.ErrBlobNotFound -} - -// blobStream is using to serve blob range requests. -type blobStream struct { - reader io.Reader - closer io.Closer -} - -func newBlobStream(blobPath string, from, to int64) (io.ReadCloser, error) { - blobFile, err := os.Open(blobPath) - if err != nil { - return nil, err - } - - if from > 0 { - _, err = blobFile.Seek(from, io.SeekStart) - if err != nil { - return nil, err - } - } - - if from < 0 || to < from { - return nil, zerr.ErrBadRange - } - - blobstrm := blobStream{reader: blobFile, closer: blobFile} - - blobstrm.reader = io.LimitReader(blobFile, to-from+1) - - return &blobstrm, nil -} - -func (bs *blobStream) Read(buf []byte) (int, error) { - return bs.reader.Read(buf) -} - -func (bs *blobStream) Close() error { - return bs.closer.Close() -} - -// GetBlobPartial returns a partial stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ImageStoreLocal) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, -) (io.ReadCloser, int64, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - if to < 0 || to >= binfo.Size() { - to = binfo.Size() - 1 - } - - blobReadCloser, err := newBlobStream(blobPath, from, to) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, -1, err - } - - // The caller function is responsible for calling Close() - return blobReadCloser, to - from + 1, binfo.Size(), nil -} - -// GetBlob returns a stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ImageStoreLocal) GetBlob(repo string, digest godigest.Digest, mediaType string, -) (io.ReadCloser, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := os.Open(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, err - } - - // The caller function is responsible for calling Close() - return blobReadCloser, binfo.Size(), nil -} - -// GetBlobContent returns blob contents, SHOULD lock from outside. -func (is *ImageStoreLocal) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { - if err := digest.Validate(); err != nil { - return []byte{}, err - } - - blobPath := is.BlobPath(repo, digest) - - blob, err := os.ReadFile(blobPath) - if err != nil { - if os.IsNotExist(err) { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("blob doesn't exist") - - return []byte{}, zerr.ErrBlobNotFound - } - - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to read blob") - - return []byte{}, err - } - - return blob, nil -} - -// GetIndexContent returns index.json contents, SHOULD lock from outside. -func (is *ImageStoreLocal) GetIndexContent(repo string) ([]byte, error) { - dir := path.Join(is.rootDir, repo) - - buf, err := os.ReadFile(path.Join(dir, "index.json")) - if err != nil { - if os.IsNotExist(err) { - is.log.Debug().Err(err).Str("dir", dir).Msg("index.json doesn't exist") - - return []byte{}, zerr.ErrRepoNotFound - } - - is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") - - return []byte{}, err - } - - return buf, nil -} - -// DeleteBlob removes the blob from the repository. -func (is *ImageStoreLocal) DeleteBlob(repo string, digest godigest.Digest) error { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return err - } - - blobPath := is.BlobPath(repo, digest) - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return zerr.ErrBlobNotFound - } - - // first check if this blob is not currently in use - if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { - return zerr.ErrBlobReferenced - } - - if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.cache.DeleteBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("unable to remove blob path from cache") - - return err - } - } - - if err := os.Remove(blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil -} - -func (is *ImageStoreLocal) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, -) (ispec.Index, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) -} - -func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, -) ([]oras.Descriptor, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) -} - -func (is *ImageStoreLocal) writeFile(filename string, data []byte) error { - if !is.commit { - return os.WriteFile(filename, data, storageConstants.DefaultFilePerms) - } - - fhandle, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, storageConstants.DefaultFilePerms) - if err != nil { - return err - } - - _, err = fhandle.Write(data) - - if err1 := inject.Error(fhandle.Sync()); err1 != nil && err == nil { - err = err1 - is.log.Error().Err(err).Str("filename", filename).Msg("unable to sync file") - } - - if err1 := inject.Error(fhandle.Close()); err1 != nil && err == nil { - err = err1 - } - - return err -} - -func ValidateHardLink(rootDir string) error { - if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { - return err - } - - err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), - []byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms) - if err != nil { - return err - } - - err = os.Link(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt")) - if err != nil { - // Remove hardlinkcheck.txt if hardlink fails - zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) - if zerr != nil { - return zerr - } - - return err - } - - err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) - if err != nil { - return err - } - - return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt")) -} - -// utility routines. -func ensureDir(dir string, log zerolog.Logger) error { - if err := os.MkdirAll(dir, storageConstants.DefaultDirPerms); err != nil { - log.Error().Err(err).Str("dir", dir).Msg("unable to create dir") - - return err - } - - return nil -} - -type extendedManifest struct { - ispec.Manifest - - Digest godigest.Digest -} - -func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { - oci, err := umoci.OpenLayout(dir) - if err := inject.Error(err); err != nil { - return err - } - defer oci.Close() - - // gc untagged manifests and signatures - index, err := oci.GetIndex(context.Background()) - if err != nil { - return err - } - - referencedByImageIndex := []string{} - cosignDescriptors := []ispec.Descriptor{} - notationManifests := []extendedManifest{} - - /* gather manifests references by multiarch images (to skip gc) - gather cosign and notation signatures descriptors */ - for _, desc := range index.Manifests { - switch desc.MediaType { - case ispec.MediaTypeImageIndex: - indexImage, err := common.GetImageIndex(is, repo, desc.Digest, is.log) - if err != nil { - is.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read multiarch(index) image") - - return err - } - - for _, indexDesc := range indexImage.Manifests { - referencedByImageIndex = append(referencedByImageIndex, indexDesc.Digest.String()) - } - case ispec.MediaTypeImageManifest: - tag, ok := desc.Annotations[ispec.AnnotationRefName] - if ok { - // gather cosign references - if strings.HasPrefix(tag, "sha256-") && (strings.HasSuffix(tag, remote.SignatureTagSuffix) || - strings.HasSuffix(tag, remote.SBOMTagSuffix)) { - cosignDescriptors = append(cosignDescriptors, desc) - - continue - } - } - - manifestContent, err := common.GetImageManifest(is, repo, desc.Digest, is.log) - if err != nil { - is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read manifest image") - - return err - } - - if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation { - notationManifests = append(notationManifests, extendedManifest{ - Digest: desc.Digest, - Manifest: manifestContent, - }) - - continue - } - } - } - - is.log.Info().Msg("gc: untagged manifests") - - if err := gcUntaggedManifests(is, oci, &index, repo, referencedByImageIndex); err != nil { - return err - } - - is.log.Info().Msg("gc: cosign references") - - if err := gcCosignReferences(is, oci, &index, repo, cosignDescriptors); err != nil { - return err - } - - is.log.Info().Msg("gc: notation signatures") - - if err := gcNotationSignatures(is, oci, &index, repo, notationManifests); err != nil { - return err - } - - is.log.Info().Msg("gc: blobs") - - err = oci.GC(context.Background(), ifOlderThan(is, repo, is.gcDelay)) - if err := inject.Error(err); err != nil { - return err - } - - return nil -} - -func gcUntaggedManifests(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - referencedByImageIndex []string, -) error { - for _, desc := range index.Manifests { - // skip manifests referenced in image indexex - if zcommon.Contains(referencedByImageIndex, desc.Digest.String()) { - continue - } - - // remove untagged images - if desc.MediaType == ispec.MediaTypeImageManifest { - _, ok := desc.Annotations[ispec.AnnotationRefName] - if !ok { - // check if is indeed an image and not an artifact by checking it's config blob - buf, err := imgStore.GetBlobContent(repo, desc.Digest) - if err != nil { - imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read image manifest") - - return err - } - - manifest := ispec.Manifest{} - - err = json.Unmarshal(buf, &manifest) - if err != nil { - return err - } - - // skip manifests which are not of type image - if manifest.Config.MediaType != ispec.MediaTypeImageConfig { - imgStore.log.Info().Str("config mediaType", manifest.Config.MediaType). - Msg("skipping gc untagged manifest, because config blob is not application/vnd.oci.image.config.v1+json") - - continue - } - - // remove manifest if it's older than gc.delay - canGC, err := isBlobOlderThan(imgStore, repo, desc.Digest, imgStore.gcDelay) - if err != nil { - imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Str("delay", imgStore.gcDelay.String()).Msg("gc: failed to check if blob is older than delay") - - return err - } - - if canGC { - imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: removing manifest without tag") - - _, err = common.RemoveManifestDescByReference(index, desc.Digest.String(), true) - if errors.Is(err, zerr.ErrManifestConflict) { - imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: skipping removing manifest due to conflict") - - continue - } - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - } - } - - return nil -} - -func gcCosignReferences(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - cosignDescriptors []ispec.Descriptor, -) error { - for _, cosignDesc := range cosignDescriptors { - foundSubject := false - // check if we can find the manifest which the reference points to - for _, desc := range index.Manifests { - // signature - subject := fmt.Sprintf("sha256-%s.%s", desc.Digest.Encoded(), remote.SignatureTagSuffix) - if subject == cosignDesc.Annotations[ispec.AnnotationRefName] { - foundSubject = true - } - - // sbom - subject = fmt.Sprintf("sha256-%s.%s", desc.Digest.Encoded(), remote.SBOMTagSuffix) - if subject == cosignDesc.Annotations[ispec.AnnotationRefName] { - foundSubject = true - } - } - - if !foundSubject { - // remove manifest - imgStore.log.Info().Str("repository", repo).Str("digest", cosignDesc.Digest.String()). - Msg("gc: removing cosign reference without subject") - - // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = common.RemoveManifestDescByReference(index, cosignDesc.Digest.String(), false) - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - - return nil -} - -func gcNotationSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - notationManifests []extendedManifest, -) error { - for _, notationManifest := range notationManifests { - foundSubject := false - - for _, desc := range index.Manifests { - if desc.Digest == notationManifest.Subject.Digest { - foundSubject = true - } - } - - if !foundSubject { - // remove manifest - imgStore.log.Info().Str("repository", repo).Str("digest", notationManifest.Digest.String()). - Msg("gc: removing notation signature without subject") - - // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = common.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false) - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - - return nil -} - -func ifOlderThan(imgStore *ImageStoreLocal, repo string, delay time.Duration) casext.GCPolicy { - return func(ctx context.Context, digest godigest.Digest) (bool, error) { - return isBlobOlderThan(imgStore, repo, digest, delay) - } -} - -func isBlobOlderThan(imgStore *ImageStoreLocal, repo string, digest godigest.Digest, delay time.Duration, -) (bool, error) { - blobPath := imgStore.BlobPath(repo, digest) - - fileInfo, err := os.Stat(blobPath) - if err != nil { - imgStore.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("gc: failed to stat blob") - - return false, err - } - - if fileInfo.ModTime().Add(delay).After(time.Now()) { - return false, nil - } - - imgStore.log.Info().Str("digest", digest.String()).Str("blobPath", blobPath).Msg("perform GC on blob") - - return true, nil -} - -func (is *ImageStoreLocal) gcRepo(repo string) error { - dir := path.Join(is.RootDir(), repo) - - var lockLatency time.Time - - is.Lock(&lockLatency) - err := is.garbageCollect(dir, repo) - is.Unlock(&lockLatency) - - if err != nil { - return err - } - - return nil -} - -func (is *ImageStoreLocal) RunGCRepo(repo string) error { - is.log.Info().Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(is.RootDir(), repo))) - - if err := is.gcRepo(repo); err != nil { - errMessage := fmt.Sprintf("error while running GC for %s", path.Join(is.RootDir(), repo)) - is.log.Error().Err(err).Msg(errMessage) - is.log.Info().Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(is.RootDir(), repo))) - - return err - } - - is.log.Info().Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(is.RootDir(), repo))) - - return nil -} - -func (is *ImageStoreLocal) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { - generator := &taskGenerator{ - imgStore: is, - } - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) -} - -type taskGenerator struct { - imgStore *ImageStoreLocal - lastRepo string - done bool -} - -func (gen *taskGenerator) Next() (scheduler.Task, error) { - repo, err := gen.imgStore.GetNextRepository(gen.lastRepo) - - if err != nil && !errors.Is(err, io.EOF) { - return nil, err - } - - if repo == "" { - gen.done = true - - return nil, nil - } - - gen.lastRepo = repo - - return newGCTask(gen.imgStore, repo), nil -} - -func (gen *taskGenerator) IsDone() bool { - return gen.done -} - -func (gen *taskGenerator) Reset() { - gen.lastRepo = "" - gen.done = false -} - -type gcTask struct { - imgStore *ImageStoreLocal - repo string -} - -func newGCTask(imgStore *ImageStoreLocal, repo string) *gcTask { - return &gcTask{imgStore, repo} -} - -func (gcT *gcTask) DoWork() error { - return gcT.imgStore.RunGCRepo(gcT.repo) -} - -func (is *ImageStoreLocal) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest, -) (godigest.Digest, []string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - var duplicateBlobs []string - - var digest godigest.Digest - - err := filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { - if err != nil { - is.log.Warn().Err(err).Msg("unable to walk dir, skipping it") - // skip files/dirs which can't be walked - return filepath.SkipDir - } - - if info.IsDir() { - return nil - } - - blobDigest := godigest.NewDigestFromEncoded("sha256", info.Name()) - if err := blobDigest.Validate(); err != nil { - return nil //nolint:nilerr // ignore files which are not blobs - } - - if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { - digest = blobDigest - } - - if blobDigest == digest { - duplicateBlobs = append(duplicateBlobs, path) - } - - return nil - }) - - return digest, duplicateBlobs, err -} - -func (is *ImageStoreLocal) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") - - return zerr.ErrDedupeRebuild - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") - - var originalBlob string - - var originalBlobFi fs.FileInfo - - var err error - // rebuild from dedupe false to true - for _, blobPath := range duplicateBlobs { - /* for local storage, because we use hard links, we can assume that any blob can be original - so we skip the first one and hard link the rest of them with the first*/ - if originalBlob == "" { - originalBlob = blobPath - - originalBlobFi, err = os.Stat(originalBlob) - if err != nil { - is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - - continue - } - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // dedupe blob - if !os.SameFile(originalBlobFi, binfo) { - // we should link to a temp file instead of removing blob and then linking - // to make this more atomic - uuid, err := guuid.NewV4() - if err != nil { - return err - } - - // put temp blob in /.uploads dir - tempLinkBlobDir := path.Join(strings.Replace(blobPath, path.Join("blobs/sha256", binfo.Name()), "", 1), - storageConstants.BlobUploadDir) - - if err := os.MkdirAll(tempLinkBlobDir, storageConstants.DefaultDirPerms); err != nil { - is.log.Error().Err(err).Str("dir", tempLinkBlobDir).Msg("rebuild dedupe: unable to mkdir") - - return err - } - - tempLinkBlobPath := path.Join(tempLinkBlobDir, uuid.String()) - - if err := os.Link(originalBlob, tempLinkBlobPath); err != nil { - is.log.Error().Err(err).Str("src", originalBlob). - Str("dst", tempLinkBlobPath).Msg("rebuild dedupe: unable to hard link") - - return err - } - - if err := os.Rename(tempLinkBlobPath, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("rebuild dedupe: unable to rename temp link") - - return err - } - - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - } - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") - - return nil -} - -func (is *ImageStoreLocal) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if dedupe { - return is.dedupeBlobs(digest, duplicateBlobs) - } - - // otherwise noop - return nil -} - -func (is *ImageStoreLocal) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { - // for local storage no need to undedupe blobs - if is.dedupe { - generator := &common.DedupeTaskGenerator{ - ImgStore: is, - Dedupe: is.dedupe, - Log: is.log, - } - - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) - } + return imagestore.NewImageStore( + rootDir, + rootDir, + gc, + gcDelay, + dedupe, + commit, + log, + metrics, + linter, + New(commit), + cacheDriver, + ) } diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index 4b4b3e03c3..e7e040f657 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -37,7 +37,6 @@ import ( "zotregistry.io/zot/pkg/storage/local" storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" - "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" ) @@ -68,6 +67,7 @@ func TestStorageFSAPIs(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -205,6 +205,7 @@ func TestGetOrasReferrers(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) Convey("Get referrers", t, func(c C) { @@ -260,6 +261,7 @@ func FuzzNewBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -764,6 +766,7 @@ func TestStorageCacheErrors(t *testing.T) { cblob, cdigest := test.GetRandomImageConfig() getBlobPath := "" + imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, &mocks.CacheMock{ PutBlobFn: func(digest godigest.Digest, path string) error { @@ -787,7 +790,7 @@ func TestStorageCacheErrors(t *testing.T) { _, _, err = imgStore.FullBlobUpload(originRepo, bytes.NewReader(cblob), cdigest) So(err, ShouldBeNil) - getBlobPath = strings.ReplaceAll(imgStore.BlobPath(originRepo, cdigest), imgStore.RootDir(), "") + getBlobPath = imgStore.BlobPath(originRepo, cdigest) _, _, err = imgStore.FullBlobUpload(dedupedRepo, bytes.NewReader(cblob), cdigest) So(err, ShouldNotBeNil) }) @@ -1080,8 +1083,9 @@ func FuzzGetOrasReferrers(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, + true, *log, metrics, nil, cacheDriver) err := test.CopyFiles("../../../test/data/zot-test", path.Join(dir, "zot-test")) if err != nil { @@ -1465,7 +1469,6 @@ func TestDedupeLinks(t *testing.T) { fi2, err := os.Stat(path.Join(dir, "dedupe2", "blobs", "sha256", blobDigest2)) So(err, ShouldBeNil) - // deduped happened, but didn't cached So(os.SameFile(fi1, fi2), ShouldEqual, true) }) } @@ -1535,6 +1538,7 @@ func TestDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + il := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) So(il.DedupeBlob("", "", ""), ShouldNotBeNil) @@ -1554,6 +1558,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + So(local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver), ShouldNotBeNil) if os.Geteuid() != 0 { @@ -1577,6 +1582,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -1627,6 +1633,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -1715,9 +1722,8 @@ func TestNegativeCases(t *testing.T) { panic(err) } - if os.Geteuid() != 0 { - So(func() { _, _ = imgStore.ValidateRepo("test") }, ShouldPanic) - } + _, err = imgStore.GetRepositories() + So(err, ShouldNotBeNil) err = os.Chmod(dir, 0o755) // add perms if err != nil { @@ -1728,16 +1734,9 @@ func TestNegativeCases(t *testing.T) { if err != nil { panic(err) } - - _, err = imgStore.GetRepositories() - So(err, ShouldNotBeNil) }) Convey("Invalid get image tags", t, func(c C) { - var ilfs local.ImageStoreLocal - _, err := ilfs.GetImageTags("test") - So(err, ShouldNotBeNil) - dir := t.TempDir() log := log.Logger{Logger: zerolog.New(os.Stdout)} @@ -1747,13 +1746,14 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil) - _, err = imgStore.GetImageTags("test") + _, err := imgStore.GetImageTags("test") So(err, ShouldNotBeNil) So(os.RemoveAll(path.Join(dir, "test")), ShouldBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1763,10 +1763,6 @@ func TestNegativeCases(t *testing.T) { }) Convey("Invalid get image manifest", t, func(c C) { - var ilfs local.ImageStoreLocal - _, _, _, err := ilfs.GetImageManifest("test", "") - So(err, ShouldNotBeNil) - dir := t.TempDir() log := log.Logger{Logger: zerolog.New(os.Stdout)} @@ -1776,13 +1772,14 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) - err = os.Chmod(path.Join(dir, "test", "index.json"), 0o000) + err := os.Chmod(path.Join(dir, "test", "index.json"), 0o000) if err != nil { panic(err) } @@ -1823,6 +1820,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -1984,42 +1982,6 @@ func TestHardLink(t *testing.T) { } func TestInjectWriteFile(t *testing.T) { - Convey("writeFile with commit", t, func() { - dir := t.TempDir() - - log := log.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) - - Convey("Failure path1", func() { - injected := inject.InjectFailure(0) - - err := imgStore.InitRepo("repo1") - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - }) - - Convey("Failure path2", func() { - injected := inject.InjectFailure(1) - - err := imgStore.InitRepo("repo2") - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - }) - }) - Convey("writeFile without commit", t, func() { dir := t.TempDir() @@ -2030,6 +1992,7 @@ func TestInjectWriteFile(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, false, log, metrics, nil, cacheDriver) @@ -2040,391 +2003,6 @@ func TestInjectWriteFile(t *testing.T) { }) } -func TestGarbageCollect(t *testing.T) { - Convey("Repo layout", t, func(c C) { - dir := t.TempDir() - - log := log.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) - - Convey("Garbage collect with default/long delay", func() { - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) - repoName := "gc-long" - - upload, err := imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content := []byte("test-data1") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - bdigest := godigest.FromBytes(content) - - blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - err = imgStore.DeleteImageManifest(repoName, digest.String(), false) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - }) - - Convey("Garbage collect with short delay", func() { - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) - repoName := "gc-short" - - // upload orphan blob - upload, err := imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content := []byte("test-data1") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - odigest := godigest.FromBytes(content) - - blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) - So(err, ShouldBeNil) - - // sleep so orphan blob can be GC'ed - time.Sleep(5 * time.Second) - - // upload blob - upload, err = imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content = []byte("test-data2") - buf = bytes.NewBuffer(content) - buflen = buf.Len() - bdigest := godigest.FromBytes(content) - - blob, err = imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - - hasBlob, _, err = imgStore.StatBlob(repoName, odigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - hasBlob, _, err = imgStore.StatBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // sleep so orphan blob can be GC'ed - time.Sleep(5 * time.Second) - - err = imgStore.DeleteImageManifest(repoName, digest.String(), false) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - }) - - Convey("Garbage collect with dedupe", func() { - // garbage-collect is repo-local and dedupe is global and they can interact in strange ways - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics, nil, cacheDriver) - - // first upload an image to the first repo and wait for GC timeout - - repo1Name := "gc1" - - // upload blob - upload, err := imgStore.NewBlobUpload(repo1Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content := []byte("test-data") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - bdigest := godigest.FromBytes(content) - tdigest := bdigest - - blob, err := imgStore.PutBlobChunk(repo1Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo1Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repo1Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repo1Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - _, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // sleep so past GC timeout - time.Sleep(10 * time.Second) - - hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // upload another image into a second repo with the same blob contents so dedupe is triggered - - repo2Name := "gc2" - - upload, err = imgStore.NewBlobUpload(repo2Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - buf = bytes.NewBuffer(content) - buflen = buf.Len() - - blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap = make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest = test.GetRandomImageConfig() - _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest = ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - - _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // immediately upload any other image to second repo which should invoke GC inline, but expect layers to persist - - upload, err = imgStore.NewBlobUpload(repo2Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content = []byte("test-data-more") - buf = bytes.NewBuffer(content) - buflen = buf.Len() - bdigest = godigest.FromBytes(content) - - blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap = make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest = test.GetRandomImageConfig() - _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest = ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - - _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - // original blob should exist - - hasBlob, _, err = imgStore.CheckBlob(repo2Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - _, _, _, err = imgStore.GetImageManifest(repo2Name, digest.String()) - So(err, ShouldBeNil) - }) - }) -} - func TestGarbageCollectForImageStore(t *testing.T) { Convey("Garbage collect for a specific repo from an ImageStore", t, func(c C) { dir := t.TempDir() @@ -2441,6 +2019,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) repoName := "gc-all-repos-short" @@ -2476,6 +2055,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) repoName := "gc-all-repos-short" @@ -2508,6 +2088,7 @@ func TestGarbageCollectErrors(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, 500*time.Millisecond, true, true, log, metrics, nil, cacheDriver) repoName := "gc-index" @@ -2743,6 +2324,7 @@ func TestInitRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -2765,6 +2347,7 @@ func TestValidateRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -2785,6 +2368,7 @@ func TestValidateRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -2830,6 +2414,7 @@ func TestGetRepositories(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver, ) @@ -3018,6 +2603,7 @@ func TestGetNextRepository(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver, ) @@ -3063,6 +2649,7 @@ func TestPutBlobChunkStreamed(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) @@ -3092,6 +2679,7 @@ func TestPullRange(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) repoName := "pull-range" @@ -3130,6 +2718,100 @@ func TestPullRange(t *testing.T) { }) } +func TestStorageDriverErr(t *testing.T) { + dir := t.TempDir() + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, + true, log, metrics, nil, cacheDriver) + + Convey("Init repo", t, func() { + err := imgStore.InitRepo(repoName) + So(err, ShouldBeNil) + + Convey("New blob upload error", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, storageConstants.BlobUploadDir), 0o000) + So(err, ShouldBeNil) + + _, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(imgStore.RootDir(), repoName, storageConstants.BlobUploadDir), + storageConstants.DefaultDirPerms) + So(err, ShouldBeNil) + + uuid, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + size, err := imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + So(size, ShouldEqual, 0) + + content := []byte("test-blob") + buf := bytes.NewBuffer(content) + bufLen := buf.Len() + digest := godigest.FromBytes(content) + + size, err = imgStore.PutBlobChunkStreamed(repoName, uuid, buf) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + size, err = imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + err = imgStore.DeleteBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + + err = imgStore.DeleteBlobUpload(repoName, uuid) + So(err, ShouldNotBeNil) + + _, err = imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldNotBeNil) + + // push again + buf = bytes.NewBuffer(content) + + uuid, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + size, err = imgStore.PutBlobChunkStreamed(repoName, uuid, buf) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + // finish blob upload + err = os.Chmod(path.Join(imgStore.BlobUploadPath(repoName, uuid)), 0o000) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(imgStore.BlobUploadPath(repoName, uuid)), storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldNotBeNil) + + // delete blob + err = imgStore.DeleteBlob(repoName, digest) + So(err, ShouldBeNil) + + err = imgStore.DeleteBlob(repoName, digest) + So(err, ShouldNotBeNil) + }) + }) +} + func NewRandomImgManifest(data []byte, cdigest, ldigest godigest.Digest, cblob, lblob []byte) (*ispec.Manifest, error) { annotationsMap := make(map[string]string) diff --git a/pkg/storage/s3/driver.go b/pkg/storage/s3/driver.go new file mode 100644 index 0000000000..5f3aa8f24a --- /dev/null +++ b/pkg/storage/s3/driver.go @@ -0,0 +1,115 @@ +package s3 + +import ( + "context" + "io" + + // Add s3 support. + "github.com/docker/distribution/registry/storage/driver" + _ "github.com/docker/distribution/registry/storage/driver/s3-aws" + + storageConstants "zotregistry.io/zot/pkg/storage/constants" +) + +type Driver struct { + store driver.StorageDriver +} + +func New(storeDriver driver.StorageDriver) *Driver { + return &Driver{store: storeDriver} +} + +func (driver *Driver) Name() string { + return storageConstants.S3StorageDriverName +} + +func (driver *Driver) EnsureDir(path string) error { + return nil +} + +func (driver *Driver) DirExists(path string) bool { + if fi, err := driver.store.Stat(context.Background(), path); err == nil && fi.IsDir() { + return true + } + + return false +} + +func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) { + return driver.store.Reader(context.Background(), path, offset) +} + +func (driver *Driver) ReadFile(path string) ([]byte, error) { + return driver.store.GetContent(context.Background(), path) +} + +func (driver *Driver) Delete(path string) error { + return driver.store.Delete(context.Background(), path) +} + +func (driver *Driver) Stat(path string) (driver.FileInfo, error) { + return driver.store.Stat(context.Background(), path) +} + +func (driver *Driver) Writer(filepath string, append bool) (driver.FileWriter, error) { //nolint:predeclared + return driver.store.Writer(context.Background(), filepath, append) +} + +func (driver *Driver) WriteFile(filepath string, contents []byte) (int, error) { + var n int + + if stwr, err := driver.store.Writer(context.Background(), filepath, false); err == nil { + defer stwr.Close() + + if n, err = stwr.Write(contents); err != nil { + return -1, err + } + + if err := stwr.Commit(); err != nil { + return -1, err + } + } else { + return -1, err + } + + return n, nil +} + +func (driver *Driver) Walk(path string, f driver.WalkFn) error { + return driver.store.Walk(context.Background(), path, f) +} + +func (driver *Driver) List(fullpath string) ([]string, error) { + return driver.store.List(context.Background(), fullpath) +} + +func (driver *Driver) Move(sourcePath string, destPath string) error { + return driver.store.Move(context.Background(), sourcePath, destPath) +} + +func (driver *Driver) SameFile(path1, path2 string) bool { + fi1, _ := driver.store.Stat(context.Background(), path1) + + fi2, _ := driver.store.Stat(context.Background(), path2) + + if fi1 != nil && fi2 != nil { + if fi1.IsDir() == fi2.IsDir() && + fi1.ModTime() == fi2.ModTime() && + fi1.Path() == fi2.Path() && + fi1.Size() == fi2.Size() { + return true + } + } + + return false +} + +/* + Link put an empty file that will act like a link between the original file and deduped one + +because s3 doesn't support symlinks, wherever the storage will encounter an empty file, it will get the original one +from cache. +*/ +func (driver *Driver) Link(src, dest string) error { + return driver.store.PutContent(context.Background(), dest, []byte{}) +} diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index b02e5d2dcb..968e2496f2 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -1,69 +1,21 @@ package s3 import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "path" - "path/filepath" - "sync" "time" // Add s3 support. "github.com/docker/distribution/registry/storage/driver" // Load s3 driver. _ "github.com/docker/distribution/registry/storage/driver/s3-aws" - guuid "github.com/gofrs/uuid" - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" - zerr "zotregistry.io/zot/errors" - zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" - zreg "zotregistry.io/zot/pkg/regexp" - "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage/cache" common "zotregistry.io/zot/pkg/storage/common" - storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" storageTypes "zotregistry.io/zot/pkg/storage/types" - "zotregistry.io/zot/pkg/test/inject" ) -const ( - CacheDBName = "s3_cache" -) - -// ObjectStorage provides the image storage operations. -type ObjectStorage struct { - rootDir string - store driver.StorageDriver - lock *sync.RWMutex - log zerolog.Logger - metrics monitoring.MetricServer - cache cache.Cache - dedupe bool - linter common.Lint -} - -func (is *ObjectStorage) RootDir() string { - return is.rootDir -} - -func (is *ObjectStorage) DirExists(d string) bool { - if fi, err := is.store.Stat(context.Background(), d); err == nil && fi.IsDir() { - return true - } - - return false -} - // NewObjectStorage returns a new image store backed by cloud storages. // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers // Use the last argument to properly set a cache database, or it will default to boltDB local storage. @@ -71,1659 +23,17 @@ func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Durati log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, cacheDriver cache.Cache, ) storageTypes.ImageStore { - imgStore := &ObjectStorage{ - rootDir: rootDir, - store: store, - lock: &sync.RWMutex{}, - log: log.With().Caller().Logger(), - metrics: metrics, - dedupe: dedupe, - linter: linter, - } - - imgStore.cache = cacheDriver - - return imgStore -} - -// RLock read-lock. -func (is *ObjectStorage) RLock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.RLock() -} - -// RUnlock read-unlock. -func (is *ObjectStorage) RUnlock(lockStart *time.Time) { - is.lock.RUnlock() - - lockEnd := time.Now() - // includes time spent in acquiring and holding a lock - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram -} - -// Lock write-lock. -func (is *ObjectStorage) Lock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.Lock() -} - -// Unlock write-unlock. -func (is *ObjectStorage) Unlock(lockStart *time.Time) { - is.lock.Unlock() - - lockEnd := time.Now() - // includes time spent in acquiring and holding a lock - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram -} - -func (is *ObjectStorage) initRepo(name string) error { - repoDir := path.Join(is.rootDir, name) - - if !zreg.FullNameRegexp.MatchString(name) { - is.log.Error().Str("repository", name).Msg("invalid repository name") - - return zerr.ErrInvalidRepositoryName - } - - // "oci-layout" file - create if it doesn't exist - ilPath := path.Join(repoDir, ispec.ImageLayoutFile) - if _, err := is.store.Stat(context.Background(), ilPath); err != nil { - il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} - - buf, err := json.Marshal(il) - if err != nil { - is.log.Error().Err(err).Msg("unable to marshal JSON") - - return err - } - - if _, err := writeFile(is.store, ilPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - // "index.json" file - create if it doesn't exist - indexPath := path.Join(repoDir, "index.json") - if _, err := is.store.Stat(context.Background(), indexPath); err != nil { - index := ispec.Index{} - index.SchemaVersion = 2 - - buf, err := json.Marshal(index) - if err != nil { - is.log.Error().Err(err).Msg("unable to marshal JSON") - - return err - } - - if _, err := writeFile(is.store, indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - return nil -} - -// InitRepo creates an image repository under this store. -func (is *ObjectStorage) InitRepo(name string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - return is.initRepo(name) -} - -// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. -func (is *ObjectStorage) ValidateRepo(name string) (bool, error) { - if !zreg.FullNameRegexp.MatchString(name) { - return false, zerr.ErrInvalidRepositoryName - } - - // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content - // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] - // and an additional/optional BlobUploadDir in each image store - // for objects storage we can not create empty dirs, so we check only against index.json and oci-layout - dir := path.Join(is.rootDir, name) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return false, zerr.ErrRepoNotFound - } - - files, err := is.store.List(context.Background(), dir) - if err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") - - return false, zerr.ErrRepoNotFound - } - - //nolint:gomnd - if len(files) < 2 { - return false, zerr.ErrRepoBadVersion - } - - found := map[string]bool{ - ispec.ImageLayoutFile: false, - "index.json": false, - } - - for _, file := range files { - _, err := is.store.Stat(context.Background(), file) - if err != nil { - return false, err - } - - filename, err := filepath.Rel(dir, file) - if err != nil { - return false, err - } - - found[filename] = true - } - - for k, v := range found { - if !v && k != storageConstants.BlobUploadDir { - return false, nil - } - } - - buf, err := is.store.GetContent(context.Background(), path.Join(dir, ispec.ImageLayoutFile)) - if err != nil { - return false, err - } - - var il ispec.ImageLayout - if err := json.Unmarshal(buf, &il); err != nil { - return false, err - } - - if il.Version != ispec.ImageLayoutVersion { - return false, zerr.ErrRepoBadVersion - } - - return true, nil -} - -// GetRepositories returns a list of all the repositories under this store. -func (is *ObjectStorage) GetRepositories() ([]string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - stores := make([]string, 0) - err := is.store.Walk(context.Background(), dir, func(fileInfo driver.FileInfo) error { - if !fileInfo.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) - if err != nil { - return nil //nolint:nilerr // ignore paths that are not under root dir - } - - if ok, err := is.ValidateRepo(rel); !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - stores = append(stores, rel) - - return nil - }) - - // if the root directory is not yet created then return an empty slice of repositories - var perr driver.PathNotFoundError - if errors.As(err, &perr) { - return stores, nil - } - - return stores, err -} - -// GetNextRepository returns next repository under this store. -func (is *ObjectStorage) GetNextRepository(repo string) (string, error) { - return "", nil -} - -// GetImageTags returns a list of image tags available in the specified repository. -func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) { - var lockLatency time.Time - - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return nil, zerr.ErrRepoNotFound - } - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, err - } - - return common.GetTagsByIndex(index), nil -} - -// GetImageManifest returns the image manifest of an image in the specific repository. -func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return nil, "", "", zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.RLock(&lockLatency) - defer func() { - is.RUnlock(&lockLatency) - - if err == nil { - monitoring.IncDownloadCounter(is.metrics, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, "", "", zerr.ErrRepoNotFound - } - - manifestDesc, found := common.GetManifestDescByReference(index, reference) - if !found { - return nil, "", "", zerr.ErrManifestNotFound - } - - buf, err := is.GetBlobContent(repo, manifestDesc.Digest) - if err != nil { - if errors.Is(err, zerr.ErrBlobNotFound) { - return nil, "", "", zerr.ErrManifestNotFound - } - - return nil, "", "", err - } - - var manifest ispec.Manifest - if err := json.Unmarshal(buf, &manifest); err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") - - return nil, "", "", err - } - - return buf, manifestDesc.Digest, manifestDesc.MediaType, nil -} - -// PutImageManifest adds an image manifest to the repository. -func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo - body []byte, -) (godigest.Digest, godigest.Digest, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Debug().Err(err).Msg("init repo") - - return "", "", err - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - monitoring.IncUploadCounter(is.metrics, repo) - } - }() - - refIsDigest := true - - mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) - if err != nil { - if errors.Is(err, zerr.ErrBadManifest) { - return mDigest, "", err - } - - refIsDigest = false - } - - dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) - if err != nil { - return dig, "", err - } - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return "", "", err - } - - // create a new descriptor - desc := ispec.Descriptor{ - MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, - } - - if !refIsDigest { - desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} - } - - var subjectDigest godigest.Digest - - artifactType := "" - - if mediaType == ispec.MediaTypeImageManifest { - var manifest ispec.Manifest - - err := json.Unmarshal(body, &manifest) - if err != nil { - return "", "", err - } - - if manifest.Subject != nil { - subjectDigest = manifest.Subject.Digest - } - - artifactType = zcommon.GetManifestArtifactType(manifest) - } else if mediaType == ispec.MediaTypeImageIndex { - var index ispec.Index - - err := json.Unmarshal(body, &index) - if err != nil { - return "", "", err - } - - if index.Subject != nil { - subjectDigest = index.Subject.Digest - } - - artifactType = zcommon.GetIndexArtifactType(index) - } - - updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) - if err != nil { - return "", "", err - } - - if !updateIndex { - return desc.Digest, subjectDigest, nil - } - - // write manifest to "blobs" - dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) - manifestPath := path.Join(dir, mDigest.Encoded()) - - if err = is.store.PutContent(context.Background(), manifestPath, body); err != nil { - is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") - - return "", "", err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) - if err != nil { - return "", "", err - } - - // now update "index.json" - index.Manifests = append(index.Manifests, desc) - dir = path.Join(is.rootDir, repo) - indexPath := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - is.log.Error().Err(err).Str("file", indexPath).Msg("unable to marshal JSON") - - return "", "", err - } - - // update the descriptors artifact type in order to check for signatures when applying the linter - desc.ArtifactType = artifactType - - // apply linter only on images, not signatures - pass, err := common.ApplyLinter(is, is.linter, repo, desc) - if !pass { - is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") - - return "", "", err - } - - if err = is.store.PutContent(context.Background(), indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") - - return "", "", err - } - - return desc.Digest, subjectDigest, nil -} - -// DeleteImageManifest deletes the image manifest from the repository. -func (is *ObjectStorage) DeleteImageManifest(repo, reference string, detectCollisions bool) error { - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return err - } - - manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollisions) - if err != nil { - return err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) - if err != nil { - return err - } - - // now update "index.json" - dir = path.Join(is.rootDir, repo) - file := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - return err - } - - if _, err := writeFile(is.store, file, buf); err != nil { - is.log.Debug().Str("deleting reference", reference).Msg("") - - return err - } - - // Delete blob only when blob digest not present in manifest entry. - // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. - toDelete := true - - for _, manifest := range index.Manifests { - if manifestDesc.Digest.String() == manifest.Digest.String() { - toDelete = false - - break - } - } - - if toDelete { - p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) - - err = is.store.Delete(context.Background(), p) - if err != nil { - return err - } - } - - return nil -} - -// BlobUploadPath returns the upload path for a blob in this store. -func (is *ObjectStorage) BlobUploadPath(repo, uuid string) string { - dir := path.Join(is.rootDir, repo) - blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) - - return blobUploadPath -} - -// NewBlobUpload returns the unique ID for an upload in progress. -func (is *ObjectStorage) NewBlobUpload(repo string) (string, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Error().Err(err).Msg("error initializing repo") - - return "", err - } - - uuid, err := guuid.NewV4() - if err != nil { - return "", err - } - - uid := uuid.String() - - blobUploadPath := is.BlobUploadPath(repo, uid) - - // create multipart upload (append false) - _, err = is.store.Writer(context.Background(), blobUploadPath, false) - if err != nil { - return "", err - } - - return uid, nil -} - -// GetBlobUpload returns the current size of a blob upload. -func (is *ObjectStorage) GetBlobUpload(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return writer.Size(), nil -} - -// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ObjectStorage) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - file, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - is.log.Error().Err(err).Msg("failed to continue multipart upload") - - return -1, err - } - - defer file.Close() - - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return -1, err - } - - nbytes, err := file.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to append to file") - - return -1, err - } - - return int64(nbytes), err -} - -// PutBlobChunk writes another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ObjectStorage) PutBlobChunk(repo, uuid string, from, to int64, - body io.Reader, -) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - file, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - is.log.Error().Err(err).Msg("failed to continue multipart upload") - - return -1, err - } - - defer file.Close() - - if from != file.Size() { - is.log.Error().Int64("expected", from).Int64("actual", file.Size()). - Msg("invalid range start for blob upload") - - return -1, zerr.ErrBadUploadRange - } - - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return -1, err - } - - nbytes, err := file.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to append to file") - - return -1, err - } - - return int64(nbytes), err -} - -// BlobUploadInfo returns the current blob size in bytes. -func (is *ObjectStorage) BlobUploadInfo(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return writer.Size(), nil -} - -// FinishBlobUpload finalizes the blob upload and moves blob the repository. -func (is *ObjectStorage) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { - if err := dstDigest.Validate(); err != nil { - return err - } - - src := is.BlobUploadPath(repo, uuid) - - // complete multiUploadPart - fileWriter, err := is.store.Writer(context.Background(), src, true) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrBadBlobDigest - } - - if err := fileWriter.Commit(); err != nil { - is.log.Error().Err(err).Msg("failed to commit file") - - return err - } - - if err := fileWriter.Close(); err != nil { - is.log.Error().Err(err).Msg("failed to close file") - - return err - } - - fileReader, err := is.store.Reader(context.Background(), src, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open file") - - return zerr.ErrUploadNotFound - } - - defer fileReader.Close() - - srcDigest, err := godigest.FromReader(fileReader) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrBadBlobDigest - } - - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return zerr.ErrBadBlobDigest - } - - dst := is.BlobPath(repo, dstDigest) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return err - } - } else { - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return err - } - } - - return nil -} - -// FullBlobUpload handles a full blob upload, and no partial session is created. -func (is *ObjectStorage) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest) (string, int64, error) { - if err := dstDigest.Validate(); err != nil { - return "", -1, err - } - - if err := is.InitRepo(repo); err != nil { - return "", -1, err - } - - u, err := guuid.NewV4() - if err != nil { - return "", -1, err - } - - uuid := u.String() - src := is.BlobUploadPath(repo, uuid) - digester := sha256.New() - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return "", -1, err - } - - nbytes, err := writeFile(is.store, src, buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to write blob") - - return "", -1, err - } - - _, err = digester.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("digester failed to write") - - return "", -1, err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return "", -1, zerr.ErrBadBlobDigest - } - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return "", -1, err - } - } else { - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return "", -1, err - } - } - - return uuid, int64(nbytes), nil -} - -func (is *ObjectStorage) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { -retry: - is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") - - dstRecord, err := is.cache.GetBlob(dstDigest) - if err := inject.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") - - return err - } - - if dstRecord == "" { - // cache record doesn't exist, so first disk and cache entry for this digest - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - // move the blob from uploads to final dest - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") - - return err - } - - is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") - } else { - // cache record exists, but due to GC and upgrades from older versions, - // disk content and cache records may go out of sync - _, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - // the actual blob on disk may have been removed by GC, so sync the cache - err := is.cache.DeleteBlob(dstDigest, dstRecord) - if err = inject.Error(err); err != nil { - //nolint:lll - is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") - - return err - } - - goto retry - } - - fileInfo, err := is.store.Stat(context.Background(), dst) - if err != nil && !errors.As(err, &driver.PathNotFoundError{}) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - - return err - } - - // prevent overwrite original blob - if fileInfo == nil && dstRecord != dst { - // put empty file so that we are compliant with oci layout, this will act as a deduped blob - err = is.store.PutContent(context.Background(), dst, []byte{}) - if err != nil { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to write empty file") - - return err - } - - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - } - - // remove temp blobupload - if err := is.store.Delete(context.Background(), src); err != nil { - is.log.Error().Err(err).Str("src", src).Msg("dedupe: unable to remove blob") - - return err - } - - is.log.Debug().Str("src", src).Msg("dedupe: remove") - } - - return nil -} - -func (is *ObjectStorage) RunGCRepo(repo string) error { - return nil -} - -func (is *ObjectStorage) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { -} - -// DeleteBlobUpload deletes an existing blob upload that is currently in progress. -func (is *ObjectStorage) DeleteBlobUpload(repo, uuid string) error { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return zerr.ErrUploadNotFound - } - - return err - } - - defer writer.Close() - - if err := writer.Cancel(); err != nil { - is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") - - return err - } - - return nil -} - -// BlobPath returns the repository path of a blob. -func (is *ObjectStorage) BlobPath(repo string, digest godigest.Digest) string { - return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) -} - -/* - CheckBlob verifies a blob and returns true if the blob is correct - -If the blob is not found but it's found in cache then it will be copied over. -*/ -func (is *ObjectStorage) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - } else { - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - } - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err == nil && binfo.Size() > 0 { - is.log.Debug().Str("blob path", blobPath).Msg("blob path found") - - return true, binfo.Size(), nil - } - // otherwise is a 'deduped' blob (empty file) - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return false, -1, zerr.ErrBlobNotFound - } - - blobSize, err := is.copyBlob(repo, blobPath, dstRecord) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - // put deduped blob in cache - if err := is.cache.PutBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") - - return false, -1, err - } - - return true, blobSize, nil -} - -// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. -func (is *ObjectStorage) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err == nil && binfo.Size() > 0 { - is.log.Debug().Str("blob path", blobPath).Msg("blob path found") - - return true, binfo.Size(), nil - } - - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return false, -1, zerr.ErrBlobNotFound - } - - // then it's a 'deduped' blob - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return false, -1, zerr.ErrBlobNotFound - } - - binfo, err = is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return false, -1, zerr.ErrBlobNotFound - } - - return true, binfo.Size(), nil -} - -func (is *ObjectStorage) checkCacheBlob(digest godigest.Digest) (string, error) { - if err := digest.Validate(); err != nil { - return "", err - } - - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - return "", zerr.ErrBlobNotFound - } - - dstRecord, err := is.cache.GetBlob(digest) - if err != nil { - return "", err - } - - if _, err := is.store.Stat(context.Background(), dstRecord); err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). - Msg("unable to remove blob path from cache") - - return "", err - } - - return "", zerr.ErrBlobNotFound - } - - is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") - - return dstRecord, nil -} - -func (is *ObjectStorage) copyBlob(repo string, blobPath, dstRecord string) (int64, error) { - if err := is.initRepo(repo); err != nil { - is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") - - return -1, err - } - - if err := is.store.PutContent(context.Background(), blobPath, []byte{}); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to link") - - return -1, zerr.ErrBlobNotFound - } - - // return original blob with content instead of the deduped one (blobPath) - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err == nil { - return binfo.Size(), nil - } - - return -1, zerr.ErrBlobNotFound -} - -// blobStream is using to serve blob range requests. -type blobStream struct { - reader io.Reader - closer io.Closer -} - -func NewBlobStream(readCloser io.ReadCloser, from, to int64) (io.ReadCloser, error) { - return &blobStream{reader: io.LimitReader(readCloser, to-from+1), closer: readCloser}, nil -} - -func (bs *blobStream) Read(buf []byte) (int, error) { - return bs.reader.Read(buf) -} - -func (bs *blobStream) Close() error { - return bs.closer.Close() -} - -// GetBlobPartial returns a partial stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ObjectStorage) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, -) (io.ReadCloser, int64, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - end := to - - if to < 0 || to >= binfo.Size() { - end = binfo.Size() - 1 - } - - blobHandle, err := is.store.Reader(context.Background(), blobPath, from) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, -1, err - } - - blobReadCloser, err := NewBlobStream(blobHandle, from, end) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") - - return nil, -1, -1, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - defer blobReadCloser.Close() - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - end := to - - if to < 0 || to >= binfo.Size() { - end = binfo.Size() - 1 - } - - blobHandle, err := is.store.Reader(context.Background(), dstRecord, from) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, -1, -1, err - } - - blobReadCloser, err := NewBlobStream(blobHandle, from, end) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") - - return nil, -1, -1, err - } - - return blobReadCloser, end - from + 1, binfo.Size(), nil - } - - // The caller function is responsible for calling Close() - return blobReadCloser, end - from + 1, binfo.Size(), nil -} - -// GetBlob returns a stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ObjectStorage) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := is.store.Reader(context.Background(), blobPath, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, -1, zerr.ErrBlobNotFound - } - - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := is.store.Reader(context.Background(), dstRecord, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, -1, err - } - - return blobReadCloser, binfo.Size(), nil - } - - // The caller function is responsible for calling Close() - return blobReadCloser, binfo.Size(), nil -} - -// GetBlobContent returns blob contents, the caller function SHOULD lock from outside. -func (is *ObjectStorage) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { - if err := digest.Validate(); err != nil { - return []byte{}, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return []byte{}, zerr.ErrBlobNotFound - } - - blobBuf, err := is.store.GetContent(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, zerr.ErrBlobNotFound - } - - blobBuf, err := is.store.GetContent(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, err - } - - return blobBuf, nil - } - - return blobBuf, nil -} - -func (is *ObjectStorage) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, -) (ispec.Index, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) -} - -func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, -) ([]artifactspec.Descriptor, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) -} - -// GetIndexContent returns index.json contents, the caller function SHOULD lock from outside. -func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) { - dir := path.Join(is.rootDir, repo) - - buf, err := is.store.GetContent(context.Background(), path.Join(dir, "index.json")) - if err != nil { - if errors.Is(err, driver.PathNotFoundError{}) { - is.log.Error().Err(err).Str("dir", dir).Msg("index.json doesn't exist") - - return []byte{}, zerr.ErrRepoNotFound - } - - is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") - - return []byte{}, err - } - - return buf, nil -} - -// DeleteBlob removes the blob from the repository. -func (is *ObjectStorage) DeleteBlob(repo string, digest godigest.Digest) error { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return err - } - - blobPath := is.BlobPath(repo, digest) - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return zerr.ErrBlobNotFound - } - - // first check if this blob is not currently in use - if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { - return zerr.ErrBlobReferenced - } - - if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - dstRecord, err := is.cache.GetBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") - - return err - } - - // remove cache entry and move blob contents to the next candidate if there is any - if ok := is.cache.HasBlob(digest, blobPath); ok { - if err := is.cache.DeleteBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("unable to remove blob path from cache") - - return err - } - } - - // if the deleted blob is one with content - if dstRecord == blobPath { - // get next candidate - dstRecord, err := is.cache.GetBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") - - return err - } - - // if we have a new candidate move the blob content to it - if dstRecord != "" { - if err := is.store.Move(context.Background(), blobPath, dstRecord); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil - } - } - } - - if err := is.store.Delete(context.Background(), blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil -} - -// Do not use for multipart upload, buf must not be empty. -// If you want to create an empty file use is.store.PutContent(). -func writeFile(store driver.StorageDriver, filepath string, buf []byte) (int, error) { - var n int - - if stwr, err := store.Writer(context.Background(), filepath, false); err == nil { - defer stwr.Close() - - if n, err = stwr.Write(buf); err != nil { - return -1, err - } - - if err := stwr.Commit(); err != nil { - return -1, err - } - } else { - return -1, err - } - - return n, nil -} - -func (is *ObjectStorage) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - var duplicateBlobs []string - - var digest godigest.Digest - - err := is.store.Walk(context.Background(), dir, func(fileInfo driver.FileInfo) error { - if fileInfo.IsDir() { - return nil - } - - blobDigest := godigest.NewDigestFromEncoded("sha256", path.Base(fileInfo.Path())) - if err := blobDigest.Validate(); err != nil { - return nil //nolint:nilerr // ignore files which are not blobs - } - - if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { - digest = blobDigest - } - - if blobDigest == digest { - duplicateBlobs = append(duplicateBlobs, fileInfo.Path()) - } - - return nil - }) - - // if the root directory is not yet created - var perr driver.PathNotFoundError - - if errors.As(err, &perr) { - return digest, duplicateBlobs, nil - } - - return digest, duplicateBlobs, err -} - -func (is *ObjectStorage) getOriginalBlobFromDisk(duplicateBlobs []string) (string, error) { - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return "", zerr.ErrBlobNotFound - } - - if binfo.Size() > 0 { - return blobPath, nil - } - } - - return "", zerr.ErrBlobNotFound -} - -func (is *ObjectStorage) getOriginalBlob(digest godigest.Digest, duplicateBlobs []string) (string, error) { - var originalBlob string - - var err error - - originalBlob, err = is.checkCacheBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrBlobNotFound) && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find blob in cache") - - return originalBlob, err - } - - // if we still don't have, search it - if originalBlob == "" { - is.log.Warn().Msg("rebuild dedupe: failed to find blob in cache, searching it in s3...") - // a rebuild dedupe was attempted in the past - // get original blob, should be found otherwise exit with error - - originalBlob, err = is.getOriginalBlobFromDisk(duplicateBlobs) - if err != nil { - return originalBlob, err - } - } - - is.log.Info().Str("originalBlob", originalBlob).Msg("rebuild dedupe: found original blob") - - return originalBlob, nil -} - -func (is *ObjectStorage) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") - - return zerr.ErrDedupeRebuild - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") - - var originalBlob string - - // rebuild from dedupe false to true - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - if binfo.Size() == 0 { - is.log.Warn().Msg("rebuild dedupe: found file without content, trying to find the original blob") - // a rebuild dedupe was attempted in the past - // get original blob, should be found otherwise exit with error - if originalBlob == "" { - originalBlob, err = is.getOriginalBlob(digest, duplicateBlobs) - if err != nil { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") - - return zerr.ErrDedupeRebuild - } - - // cache original blob - if ok := is.cache.HasBlob(digest, originalBlob); !ok { - if err := is.cache.PutBlob(digest, originalBlob); err != nil { - return err - } - } - } - - // cache dedupe blob - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - } else { - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - - // if we have an original blob cached then we can safely dedupe the rest of them - if originalBlob != "" { - if err := is.store.PutContent(context.Background(), blobPath, []byte{}); err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: unable to dedupe blob") - - return err - } - } - - // mark blob as preserved - originalBlob = blobPath - } - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") - - return nil -} - -func (is *ObjectStorage) restoreDedupedBlobs(digest godigest.Digest, duplicateBlobs []string) error { - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: restoring deduped blobs for digest") - - // first we need to find the original blob, either in cache or by checking each blob size - originalBlob, err := is.getOriginalBlob(digest, duplicateBlobs) - if err != nil { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") - - return zerr.ErrDedupeRebuild - } - - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // if we find a deduped blob, then copy original blob content to deduped one - if binfo.Size() == 0 { - // move content from original blob to deduped one - buf, err := is.store.GetContent(context.Background(), originalBlob) - if err != nil { - is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to get original blob content") - - return err - } - - _, err = writeFile(is.store, blobPath, buf) - if err != nil { - return err - } - } - } - - is.log.Info().Str("digest", digest.String()). - Msg("rebuild dedupe: restoring deduped blobs for digest finished successfully") - - return nil -} - -func (is *ObjectStorage) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if dedupe { - return is.dedupeBlobs(digest, duplicateBlobs) - } - - return is.restoreDedupedBlobs(digest, duplicateBlobs) -} - -func (is *ObjectStorage) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { - generator := &common.DedupeTaskGenerator{ - ImgStore: is, - Dedupe: is.dedupe, - Log: is.log, - } - - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) + return imagestore.NewImageStore( + rootDir, + cacheDir, + gc, + gcDelay, + dedupe, + commit, + log, + metrics, + linter, + New(store), + cacheDriver, + ) } diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 9430b2cc62..2d098e2989 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -77,13 +77,15 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive var cacheDriver cache.Cache // from pkg/cli/root.go/applyDefaultValues, s3 magic - if _, err := os.Stat(path.Join(cacheDir, "s3_cache.db")); dedupe || (!dedupe && err == nil) { + if _, err := os.Stat(path.Join(cacheDir, + storageConstants.BoltdbName+storageConstants.DBExtensionName)); dedupe || (!dedupe && err == nil) { cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) } + il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, dedupe, false, log, metrics, nil, store, cacheDriver, ) @@ -150,11 +152,11 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( var err error // from pkg/cli/root.go/applyDefaultValues, s3 magic - s3CacheDBPath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + s3CacheDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err = os.Stat(s3CacheDBPath); dedupe || (!dedupe && err == nil) { cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) } @@ -893,7 +895,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { _, _, err = imgStore.CheckBlob(testImage, digest) So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob(testImage, digest) + _, _, _, err = imgStore.StatBlob(testImage, digest) So(err, ShouldNotBeNil) }) @@ -1050,7 +1052,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { return &FileWriterMock{WriteFn: func(b []byte) (int, error) { return 0, errS3 - }}, nil + }}, errS3 }, }) _, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", io.NopCloser(strings.NewReader(""))) @@ -1091,7 +1093,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { WriteFn: func(b []byte) (int, error) { return 0, errS3 }, - }, nil + }, errS3 }, }) _, err := imgStore.PutBlobChunk(testImage, "uuid", 12, 100, io.NopCloser(strings.NewReader(""))) @@ -1280,7 +1282,7 @@ func TestS3Dedupe(t *testing.T) { So(checkBlobSize1, ShouldBeGreaterThan, 0) So(err, ShouldBeNil) - ok, checkBlobSize1, err = imgStore.StatBlob("dedupe1", digest) + ok, checkBlobSize1, _, err = imgStore.StatBlob("dedupe1", digest) So(ok, ShouldBeTrue) So(checkBlobSize1, ShouldBeGreaterThan, 0) So(err, ShouldBeNil) @@ -1466,12 +1468,12 @@ func TestS3Dedupe(t *testing.T) { Convey("Check backward compatibility - switch dedupe to false", func() { /* copy cache to the new storage with dedupe false (doing this because we already have a cache object holding the lock on cache db file) */ - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, false) @@ -3306,6 +3308,7 @@ func TestS3ManifestImageIndex(t *testing.T) { err = imgStore.DeleteImageManifest("index", "test:index1", false) So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") So(err, ShouldNotBeNil) @@ -3599,7 +3602,7 @@ func TestS3DedupeErr(t *testing.T) { imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{}) - err = os.Remove(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + err = os.Remove(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest") // trigger unable to insert blob record @@ -3640,8 +3643,9 @@ func TestS3DedupeErr(t *testing.T) { err := imgStore.DedupeBlob("", digest, "dst") So(err, ShouldBeNil) + // error will be triggered in driver.SameFile() err = imgStore.DedupeBlob("", digest, "dst2") - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) }) Convey("Test DedupeBlob - error on store.PutContent()", t, func(c C) { @@ -3776,12 +3780,12 @@ func TestS3DedupeErr(t *testing.T) { So(err, ShouldBeNil) // copy cache db to the new imagestore - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ @@ -3797,12 +3801,14 @@ func TestS3DedupeErr(t *testing.T) { _, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldNotBeNil) + // now it should move content from /repo1/dst1 to /repo2/dst2 _, err = imgStore.GetBlobContent("repo2", digest) - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) - _, _, err = imgStore.StatBlob("repo2", digest) - So(err, ShouldNotBeNil) + _, _, _, err = imgStore.StatBlob("repo2", digest) + So(err, ShouldBeNil) + // it errors out because of bad range, as mock store returns a driver.FileInfo with 0 size _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) So(err, ShouldNotBeNil) }) @@ -3822,12 +3828,12 @@ func TestS3DedupeErr(t *testing.T) { So(err, ShouldBeNil) // copy cache db to the new imagestore - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ @@ -3887,7 +3893,7 @@ func TestS3DedupeErr(t *testing.T) { _, err = imgStore.GetBlobContent("repo2", digest) So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob("repo2", digest) + _, _, _, err = imgStore.StatBlob("repo2", digest) So(err, ShouldNotBeNil) _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 7aab199d7b..6c11ecefb0 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -51,7 +51,8 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer if config.Storage.StorageDriver == nil { // false positive lint - linter does not implement Lint method //nolint:typecheck,contextcheck - defaultStore = local.NewImageStore(config.Storage.RootDirectory, + rootDir := config.Storage.RootDirectory + defaultStore = local.NewImageStore(rootDir, config.Storage.GC, config.Storage.GCDelay, config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, CreateCacheDatabaseDriver(config.Storage.StorageConfig, log), @@ -152,9 +153,12 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // add it to uniqueSubFiles // Create a new image store and assign it to imgStoreMap if isUnique { - imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(storageConfig.RootDirectory, + rootDir := storageConfig.RootDirectory + imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(rootDir, storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, - storageConfig.Commit, log, metrics, linter, CreateCacheDatabaseDriver(storageConfig, log)) + storageConfig.Commit, log, metrics, linter, + CreateCacheDatabaseDriver(storageConfig, log), + ) subImageStore[route] = imgStoreMap[storageConfig.RootDirectory] } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 53ba4b0d1e..485ef093b2 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -33,6 +33,7 @@ import ( "zotregistry.io/zot/pkg/storage/cache" storageCommon "zotregistry.io/zot/pkg/storage/common" storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/storage/s3" storageTypes "zotregistry.io/zot/pkg/storage/types" @@ -52,7 +53,9 @@ func skipIt(t *testing.T) { } } -func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, storageTypes.ImageStore, error) { +func createObjectsStore(rootDir string, cacheDir string, gcDelay time.Duration) ( + driver.StorageDriver, storageTypes.ImageStore, error, +) { bucket := "zot-storage-test" endpoint := os.Getenv("S3MOCK_ENDPOINT") storageDriverParams := map[string]interface{}{ @@ -85,11 +88,11 @@ func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, + il := s3.NewImageStore(rootDir, cacheDir, true, gcDelay, true, false, log, metrics, nil, store, cacheDriver, ) @@ -103,11 +106,11 @@ var testCases = []struct { }{ { testCaseName: "S3APIs", - storageType: "s3", + storageType: storageConstants.S3StorageDriverName, }, { testCaseName: "FileSystemAPIs", - storageType: "fs", + storageType: storageConstants.LocalStorageDriverName, }, } @@ -116,7 +119,7 @@ func TestStorageAPIs(t *testing.T) { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { var imgStore storageTypes.ImageStore - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -128,7 +131,7 @@ func TestStorageAPIs(t *testing.T) { tdir := t.TempDir() var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay) defer cleanupStorage(store, testDir) } else { dir := t.TempDir() @@ -140,8 +143,11 @@ func TestStorageAPIs(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, storageConstants.DefaultGCDelay, true, + true, log, metrics, nil, driver, cacheDriver) } Convey("Repo layout", t, func(c C) { @@ -166,15 +172,15 @@ func TestStorageAPIs(t *testing.T) { }) Convey("Validate repo", func() { - v, err := imgStore.ValidateRepo(repoName) + repos, err := imgStore.ValidateRepo(repoName) So(err, ShouldBeNil) - So(v, ShouldEqual, true) + So(repos, ShouldEqual, true) }) Convey("Get repos", func() { - v, err := imgStore.GetRepositories() + repos, err := imgStore.GetRepositories() So(err, ShouldBeNil) - So(v, ShouldNotBeEmpty) + So(repos, ShouldNotBeEmpty) }) Convey("Get image tags", func() { @@ -261,7 +267,7 @@ func TestStorageAPIs(t *testing.T) { _, _, err = imgStore.CheckBlob("test", digest) So(err, ShouldBeNil) - ok, _, err := imgStore.StatBlob("test", digest) + ok, _, _, err := imgStore.StatBlob("test", digest) So(ok, ShouldBeTrue) So(err, ShouldBeNil) @@ -385,6 +391,11 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldBeNil) So(len(tags), ShouldEqual, 2) + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos[0], ShouldEqual, "test") + // We deleted only one tag, make sure blob should not be removed. hasBlob, _, err = imgStore.CheckBlob("test", digest) So(err, ShouldBeNil) @@ -407,7 +418,7 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldNotBeNil) So(hasBlob, ShouldEqual, false) - hasBlob, _, err = imgStore.StatBlob("test", digest) + hasBlob, _, _, err = imgStore.StatBlob("test", digest) So(err, ShouldNotBeNil) So(hasBlob, ShouldEqual, false) @@ -464,6 +475,10 @@ func TestStorageAPIs(t *testing.T) { err = imgStore.FinishBlobUpload("test", "inexistent", buf, digest) So(err, ShouldNotBeNil) + // invalid digest + err = imgStore.FinishBlobUpload("test", "inexistent", buf, "sha256:invalid") + So(err, ShouldNotBeNil) + err = imgStore.FinishBlobUpload("test", bupload, buf, digest) So(err, ShouldBeNil) @@ -471,7 +486,7 @@ func TestStorageAPIs(t *testing.T) { So(ok, ShouldBeTrue) So(err, ShouldBeNil) - ok, _, err = imgStore.StatBlob("test", digest) + ok, _, _, err = imgStore.StatBlob("test", digest) So(ok, ShouldBeTrue) So(err, ShouldBeNil) @@ -502,7 +517,7 @@ func TestStorageAPIs(t *testing.T) { _, _, err = imgStore.CheckBlob("test", "inexistent") So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob("test", "inexistent") + _, _, _, err = imgStore.StatBlob("test", "inexistent") So(err, ShouldNotBeNil) }) @@ -564,7 +579,7 @@ func TestStorageAPIs(t *testing.T) { indexContent, err := imgStore.GetIndexContent("test") So(err, ShouldBeNil) - if testcase.storageType == "fs" { + if testcase.storageType == "filesystem" { err = os.Chmod(path.Join(imgStore.RootDir(), "test", "index.json"), 0o000) So(err, ShouldBeNil) _, err = imgStore.GetIndexContent("test") @@ -737,7 +752,7 @@ func TestMandatoryAnnotations(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -748,13 +763,14 @@ func TestMandatoryAnnotations(t *testing.T) { testDir = path.Join("/oci-repo-test", uuid.String()) tdir = t.TempDir() - store, _, _ = createObjectsStore(testDir, tdir) - imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, + store, _, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay) + driver := s3.New(store) + imgStore = imagestore.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, store, nil) + }, driver, nil) defer cleanupStorage(store, testDir) } else { @@ -764,12 +780,13 @@ func TestMandatoryAnnotations(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, cacheDriver) + }, driver, cacheDriver) } Convey("Setup manifest", t, func() { @@ -815,27 +832,29 @@ func TestMandatoryAnnotations(t *testing.T) { }) Convey("Error on mandatory annotations", func() { - if testcase.storageType == "s3" { - imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, + if testcase.storageType == storageConstants.S3StorageDriverName { + driver := s3.New(store) + imgStore = imagestore.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, - }, store, nil) + }, driver, nil) } else { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: tdir, Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, - }, cacheDriver) + }, driver, cacheDriver) } _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) @@ -857,7 +876,7 @@ func TestDeleteBlobsInUse(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -868,7 +887,7 @@ func TestDeleteBlobsInUse(t *testing.T) { testDir = path.Join("/oci-repo-test", uuid.String()) tdir = t.TempDir() - store, imgStore, _ = createObjectsStore(testDir, tdir) + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay) defer cleanupStorage(store, testDir) } else { @@ -878,8 +897,9 @@ func TestDeleteBlobsInUse(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, storageConstants.DefaultGCDelay, true, + true, log, metrics, nil, driver, cacheDriver) } Convey("Setup manifest", t, func() { @@ -961,7 +981,7 @@ func TestDeleteBlobsInUse(t *testing.T) { So(err, ShouldBeNil) }) - if testcase.storageType != "s3" { + if testcase.storageType != storageConstants.S3StorageDriverName { Convey("get image manifest error", func() { err := os.Chmod(path.Join(imgStore.RootDir(), "repo", "blobs", "sha256", manifestDigest.Encoded()), 0o000) So(err, ShouldBeNil) @@ -1011,6 +1031,7 @@ func TestDeleteBlobsInUse(t *testing.T) { var cdigest godigest.Digest var cblob []byte + //nolint: dupl for i := 0; i < 4; i++ { // upload image config blob upload, err = imgStore.NewBlobUpload(repoName) @@ -1067,6 +1088,12 @@ func TestDeleteBlobsInUse(t *testing.T) { indexManifestDigest, _, err := imgStore.PutImageManifest(repoName, "index", ispec.MediaTypeImageIndex, indexContent) So(err, ShouldBeNil) + Convey("Try to delete manifest being referenced by image index", func() { + // modifying multi arch images should not be allowed + err := imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldEqual, zerr.ErrBlobReferenced) + }) + Convey("Try to delete blob currently in use", func() { // layer blob err := imgStore.DeleteBlob("test", bdgst1) @@ -1103,7 +1130,7 @@ func TestDeleteBlobsInUse(t *testing.T) { So(err, ShouldBeNil) }) - if testcase.storageType != "s3" { + if testcase.storageType != storageConstants.S3StorageDriverName { Convey("repo not found", func() { // delete repo err := os.RemoveAll(path.Join(imgStore.RootDir(), repoName)) @@ -1148,22 +1175,25 @@ func TestStorageHandler(t *testing.T) { var secondRootDir string var thirdRootDir string - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) var firstStorageDriver driver.StorageDriver var secondStorageDriver driver.StorageDriver var thirdStorageDriver driver.StorageDriver firstRootDir = "/util_test1" - firstStorageDriver, firstStore, _ = createObjectsStore(firstRootDir, t.TempDir()) + firstStorageDriver, firstStore, _ = createObjectsStore(firstRootDir, t.TempDir(), + storageConstants.DefaultGCDelay) defer cleanupStorage(firstStorageDriver, firstRootDir) secondRootDir = "/util_test2" - secondStorageDriver, secondStore, _ = createObjectsStore(secondRootDir, t.TempDir()) + secondStorageDriver, secondStore, _ = createObjectsStore(secondRootDir, t.TempDir(), + storageConstants.DefaultGCDelay) defer cleanupStorage(secondStorageDriver, secondRootDir) thirdRootDir = "/util_test3" - thirdStorageDriver, thirdStore, _ = createObjectsStore(thirdRootDir, t.TempDir()) + thirdStorageDriver, thirdStore, _ = createObjectsStore(thirdRootDir, t.TempDir(), + storageConstants.DefaultGCDelay) defer cleanupStorage(thirdStorageDriver, thirdRootDir) } else { // Create temporary directory @@ -1175,15 +1205,17 @@ func TestStorageHandler(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) + driver := local.New(true) + // Create ImageStore - firstStore = local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + firstStore = imagestore.NewImageStore(firstRootDir, firstRootDir, false, storageConstants.DefaultGCDelay, + false, false, log, metrics, nil, driver, nil) - secondStore = local.NewImageStore(secondRootDir, false, - storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + secondStore = imagestore.NewImageStore(secondRootDir, secondRootDir, false, + storageConstants.DefaultGCDelay, false, false, log, metrics, nil, driver, nil) - thirdStore = local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + thirdStore = imagestore.NewImageStore(thirdRootDir, thirdRootDir, false, storageConstants.DefaultGCDelay, + false, false, log, metrics, nil, driver, nil) } Convey("Test storage handler", t, func() { @@ -1226,3 +1258,987 @@ func TestRoutePrefix(t *testing.T) { So(routePrefix, ShouldEqual, "/a") }) } + +func TestGarbageCollectImageManifest(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + Convey("Repo layout", t, func(c C) { + Convey("Garbage collect with default/long delay", func() { + var imgStore storageTypes.ImageStore + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, storageConstants.DefaultGCDelay, true, + true, log, metrics, nil, driver, cacheDriver) + } + + repoName := "gc-long" + + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + bdigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: digest, + Size: int64(len(manifestBuf)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + err = imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + }) + + Convey("Garbage collect with short delay", func() { + var imgStore storageTypes.ImageStore + + gcDelay := 1 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, gcDelay, true, + true, log, metrics, nil, driver, cacheDriver) + } + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + // sleep so orphan blob can be GC'ed + time.Sleep(5 * time.Second) + + // upload blob + upload, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data2") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + bdigest := godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: digest, + Size: int64(len(manifestBuf)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // sleep so orphan blob can be GC'ed + time.Sleep(5 * time.Second) + + Convey("Garbage collect blobs after manifest is removed", func() { + err = imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // delete artifact + err = imgStore.DeleteImageManifest(repoName, artifactDigest.String(), false) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + + Convey("Garbage collect - don't gc manifests/blobs which are referenced by another image", func() { + // upload same image with another tag + _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, tag, false) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + }) + }) + + Convey("Garbage collect with dedupe", func() { + // garbage-collect is repo-local and dedupe is global and they can interact in strange ways + var imgStore storageTypes.ImageStore + + gcDelay := 5 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, gcDelay, true, + true, log, metrics, nil, driver, cacheDriver) + } + + // first upload an image to the first repo and wait for GC timeout + + repo1Name := "gc1" + + // upload blob + upload, err := imgStore.NewBlobUpload(repo1Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + bdigest := godigest.FromBytes(content) + tdigest := bdigest + + blob, err := imgStore.PutBlobChunk(repo1Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo1Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repo1Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repo1Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // sleep so past GC timeout + time.Sleep(10 * time.Second) + + hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // upload another image into a second repo with the same blob contents so dedupe is triggered + + repo2Name := "gc2" + + upload, err = imgStore.NewBlobUpload(repo2Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + buf = bytes.NewBuffer(content) + buflen = buf.Len() + + blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap = make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // immediately upload any other image to second repo which should invoke GC inline, but expect layers to persist + + upload, err = imgStore.NewBlobUpload(repo2Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data-more") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + bdigest = godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap = make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // original blob should exist + + hasBlob, _, err = imgStore.CheckBlob(repo2Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + _, _, _, err = imgStore.GetImageManifest(repo2Name, digest.String()) + So(err, ShouldBeNil) + }) + }) + }) + } +} + +func TestGarbageCollectImageIndex(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + Convey("Repo layout", t, func(c C) { + Convey("Garbage collect with default/long delay", func() { + var imgStore storageTypes.ImageStore + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, storageConstants.DefaultGCDelay, true, + true, log, metrics, nil, driver, cacheDriver) + } + + repoName := "gc-long" + + content := []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + for i := 0; i < 4; i++ { //nolint: dupl + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: indexDigest, + Size: int64(len(indexContent)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + Convey("delete index manifest, layers should be persisted", func() { + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + }) + }) + + Convey("Garbage collect with short delay", func() { + var imgStore storageTypes.ImageStore + + gcDelay := 5 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, gcDelay, true, + true, log, metrics, nil, driver, cacheDriver) + } + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + content = []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + for i := 0; i < 4; i++ { //nolint: dupl + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: indexDigest, + Size: int64(len(indexContent)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // sleep so orphan blob can be GC'ed + time.Sleep(5 * time.Second) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + Convey("delete index manifest, layers should not be persisted", func() { + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // delete artifact + err = imgStore.DeleteImageManifest(repoName, artifactDigest.String(), false) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + }) + }) + }) + } +} diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 1d66d7d369..346aad60ba 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -4,6 +4,7 @@ import ( "io" "time" + storagedriver "github.com/docker/distribution/registry/storage/driver" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" @@ -38,7 +39,7 @@ type ImageStore interface { //nolint:interfacebloat DeleteBlobUpload(repo, uuid string) error BlobPath(repo string, digest godigest.Digest) string CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) - StatBlob(repo string, digest godigest.Digest) (bool, int64, error) + StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, ) (io.ReadCloser, int64, int64, error) @@ -52,4 +53,22 @@ type ImageStore interface { //nolint:interfacebloat RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) + GetAllBlobs(repo string) ([]string, error) +} + +type Driver interface { //nolint:interfacebloat + Name() string + EnsureDir(path string) error + DirExists(path string) bool + Reader(path string, offset int64) (io.ReadCloser, error) + ReadFile(path string) ([]byte, error) + Delete(path string) error + Stat(path string) (storagedriver.FileInfo, error) + Writer(filepath string, append bool) (storagedriver.FileWriter, error) //nolint: predeclared + WriteFile(filepath string, contents []byte) (int, error) + Walk(path string, f storagedriver.WalkFn) error + List(fullpath string) ([]string, error) + Move(sourcePath string, destPath string) error + SameFile(path1, path2 string) bool + Link(src, dest string) error } diff --git a/pkg/test/mocks/cache_mock.go b/pkg/test/mocks/cache_mock.go index 1d4e95e8f8..ef8dbf16b6 100644 --- a/pkg/test/mocks/cache_mock.go +++ b/pkg/test/mocks/cache_mock.go @@ -17,6 +17,16 @@ type CacheMock struct { // Delete a blob from the cachedb. DeleteBlobFn func(digest godigest.Digest, path string) error + + UsesRelativePathsFn func() bool +} + +func (cacheMock CacheMock) UsesRelativePaths() bool { + if cacheMock.UsesRelativePathsFn != nil { + return cacheMock.UsesRelativePaths() + } + + return false } func (cacheMock CacheMock) Name() string { diff --git a/pkg/test/mocks/image_store_mock.go b/pkg/test/mocks/image_store_mock.go index 63d05e85eb..7736d9464a 100644 --- a/pkg/test/mocks/image_store_mock.go +++ b/pkg/test/mocks/image_store_mock.go @@ -35,7 +35,7 @@ type MockedImageStore struct { DeleteBlobUploadFn func(repo string, uuid string) error BlobPathFn func(repo string, digest godigest.Digest) string CheckBlobFn func(repo string, digest godigest.Digest) (bool, int64, error) - StatBlobFn func(repo string, digest godigest.Digest) (bool, int64, error) + StatBlobFn func(repo string, digest godigest.Digest) (bool, int64, time.Time, error) GetBlobPartialFn func(repo string, digest godigest.Digest, mediaType string, from, to int64, ) (io.ReadCloser, int64, int64, error) GetBlobFn func(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) @@ -51,6 +51,7 @@ type MockedImageStore struct { RunDedupeBlobsFn func(interval time.Duration, sch *scheduler.Scheduler) RunDedupeForDigestFn func(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPathsFn func(lastDigests []godigest.Digest) (godigest.Digest, []string, error) + GetAllBlobsFn func(repo string) ([]string, error) } func (is MockedImageStore) Lock(t *time.Time) { @@ -142,6 +143,14 @@ func (is MockedImageStore) GetImageTags(name string) ([]string, error) { return []string{}, nil } +func (is MockedImageStore) GetAllBlobs(repo string) ([]string, error) { + if is.GetAllBlobsFn != nil { + return is.GetAllBlobsFn(repo) + } + + return []string{}, nil +} + func (is MockedImageStore) DeleteImageManifest(name string, reference string, detectCollision bool) error { if is.DeleteImageManifestFn != nil { return is.DeleteImageManifestFn(name, reference, detectCollision) @@ -252,12 +261,12 @@ func (is MockedImageStore) CheckBlob(repo string, digest godigest.Digest) (bool, return true, 0, nil } -func (is MockedImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { +func (is MockedImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) { if is.StatBlobFn != nil { return is.StatBlobFn(repo, digest) } - return true, 0, nil + return true, 0, time.Time{}, nil } func (is MockedImageStore) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, diff --git a/test/blackbox/scrub.bats b/test/blackbox/scrub.bats index 01f8f1113b..c3b6943b62 100644 --- a/test/blackbox/scrub.bats +++ b/test/blackbox/scrub.bats @@ -61,7 +61,7 @@ function teardown() { wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog" # wait for scrub to be done and logs to get populated - run sleep 10s + run sleep 20s run not_affected [ "$status" -eq 0 ] [ $(echo "${lines[0]}" ) = 'true' ] @@ -76,7 +76,7 @@ function teardown() { wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog" # wait for scrub to be done and logs to get populated - run sleep 10s + run sleep 20s run affected [ "$status" -eq 0 ] [ $(echo "${lines[0]}" ) = 'true' ] diff --git a/test/blackbox/sync_replica_cluster.bats b/test/blackbox/sync_replica_cluster.bats index 6b4e2eb3d3..4ea0ac8409 100644 --- a/test/blackbox/sync_replica_cluster.bats +++ b/test/blackbox/sync_replica_cluster.bats @@ -100,8 +100,8 @@ EOF } function teardown_file() { - local zot_sync_one_root_dir=${BATS_FILE_TMPDIR}/zot-per - local zot_sync_two_root_dir=${BATS_FILE_TMPDIR}/zot-ondemand + local zot_sync_one_root_dir=${BATS_FILE_TMPDIR}/zot-one + local zot_sync_two_root_dir=${BATS_FILE_TMPDIR}/zot-two teardown_zot_file_level rm -rf ${zot_sync_one_root_dir} rm -rf ${zot_sync_two_root_dir} @@ -120,7 +120,7 @@ function teardown_file() { [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ] - run sleep 30s + run sleep 40s run curl http://127.0.0.1:8082/v2/_catalog [ "$status" -eq 0 ] @@ -143,7 +143,7 @@ function teardown_file() { [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ] - run sleep 30s + run sleep 40s run curl http://127.0.0.1:8081/v2/_catalog [ "$status" -eq 0 ]