From 5bb6c1bf603d278442e5b3538791717a891e64cf Mon Sep 17 00:00:00 2001 From: Petu Eusebiu Date: Wed, 19 Jul 2023 18:14:09 +0300 Subject: [PATCH] refactor(storage): refactor storage into a single ImageStore unified both local and s3 ImageStore logic into a single ImageStore added a new driver interface for common file/dirs manipulations to be implemented by different storage types refactor(gc): drop umoci dependency, implemented internal gc added retentionDelay config option that specifies the garbage collect delay for images without tags this will also clean manifests which are part of an index image (multiarch) that no longer exist. fix(dedupe): skip blobs under .sync/ directory if startup dedupe is running while also syncing is running ignore blobs under sync's temporary storage fix(storage): do not allow image indexes modifications when deleting a manifest verify that it is not part of a multiarch image and throw a MethodNotAllowed error to the client if it is. we don't want to modify multiarch images Signed-off-by: Petu Eusebiu --- .github/workflows/ecosystem-tools.yaml | 6 +- .github/workflows/gc-stress-test.yaml | 46 +- .github/workflows/nightly.yaml | 42 + Makefile | 8 + errors/errors.go | 4 + examples/config-gc.json | 5 +- go.mod | 4 +- go.sum | 5 - pkg/api/config/config.go | 25 +- pkg/api/controller_test.go | 152 +- pkg/api/routes.go | 5 + pkg/cli/cve_cmd_test.go | 2 +- pkg/cli/root.go | 21 +- pkg/cli/root_test.go | 5 +- pkg/extensions/extension_image_trust_test.go | 10 +- pkg/extensions/extension_scrub.go | 27 +- pkg/extensions/lint/lint_test.go | 14 +- pkg/extensions/scrub/scrub_test.go | 8 +- pkg/extensions/search/cve/cve_test.go | 4 +- .../search/cve/trivy/scanner_internal_test.go | 24 +- .../search/cve/trivy/scanner_test.go | 2 +- pkg/extensions/search/search_test.go | 16 +- pkg/extensions/search/userprefs_test.go | 2 +- pkg/extensions/sync/constants/consts.go | 7 +- pkg/extensions/sync/local.go | 5 +- pkg/extensions/sync/oci_layout.go | 3 +- pkg/extensions/sync/sync_internal_test.go | 12 +- pkg/extensions/sync/sync_test.go | 15 +- pkg/extensions/sync/utils.go | 4 - pkg/meta/hooks_test.go | 8 +- pkg/meta/parse_test.go | 6 +- pkg/storage/cache/boltdb.go | 4 + pkg/storage/cache/cacheinterface.go | 3 + pkg/storage/cache/dynamodb.go | 4 + pkg/storage/cache_test.go | 2 + pkg/storage/common/common.go | 187 +- pkg/storage/common/common_test.go | 262 +- pkg/storage/constants/constants.go | 34 +- pkg/storage/imagestore/imagestore.go | 2252 +++++++++++++++++ pkg/storage/local/driver.go | 481 ++++ pkg/storage/local/local.go | 1975 +-------------- pkg/storage/local/local_elevated_test.go | 4 +- pkg/storage/local/local_test.go | 1078 +++----- pkg/storage/s3/driver.go | 115 + pkg/storage/s3/s3.go | 1726 +------------ pkg/storage/s3/s3_test.go | 62 +- pkg/storage/scrub_test.go | 10 +- pkg/storage/storage.go | 23 +- pkg/storage/storage_test.go | 1844 +++++++++++++- pkg/storage/types/types.go | 21 +- pkg/test/common.go | 2 +- pkg/test/mocks/cache_mock.go | 10 + pkg/test/mocks/image_store_mock.go | 15 +- pkg/test/oci-layout/oci_layout_test.go | 6 +- test/blackbox/garbage_collect.bats | 159 ++ test/blackbox/scrub.bats | 4 +- .../gc-stress/config-gc-bench-local.json | 6 +- test/gc-stress/config-gc-bench-s3.json | 34 + 58 files changed, 6065 insertions(+), 4755 deletions(-) create mode 100644 pkg/storage/imagestore/imagestore.go create mode 100644 pkg/storage/local/driver.go create mode 100644 pkg/storage/s3/driver.go create mode 100644 test/blackbox/garbage_collect.bats rename examples/config-gc-bench.json => test/gc-stress/config-gc-bench-local.json (64%) create mode 100644 test/gc-stress/config-gc-bench-s3.json diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml index 4963336daa..27a810542d 100644 --- a/.github/workflows/ecosystem-tools.yaml +++ b/.github/workflows/ecosystem-tools.yaml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/clean-runner - uses: actions/setup-go@v4 with: cache: false @@ -98,6 +99,9 @@ jobs: - name: Run annotations tests run: | make test-annotations + - name: Run garbage collect tests + run: | + make test-garbage-collect - name: Install localstack run: | pip install --upgrade pyopenssl @@ -129,4 +133,4 @@ jobs: sudo du -sh /var/ sudo du -sh /var/lib/docker/ du -sh /home/runner/work/ - set +x \ No newline at end of file + set +x diff --git a/.github/workflows/gc-stress-test.yaml b/.github/workflows/gc-stress-test.yaml index fe80e987c2..4c385d1491 100644 --- a/.github/workflows/gc-stress-test.yaml +++ b/.github/workflows/gc-stress-test.yaml @@ -12,8 +12,8 @@ on: permissions: read-all jobs: - client-tools: - name: GC with short interval + gc-stress-local: + name: GC on filesystem with short interval runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -27,7 +27,7 @@ jobs: run: | make binary make bench - ./bin/zot-linux-amd64 serve examples/config-gc-bench.json & + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-local.json & sleep 10 bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 @@ -35,3 +35,43 @@ jobs: # clean zot storage sudo rm -rf /tmp/zot + gc-stress-s3: + name: GC on S3 with short interval + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Setup localstack service + run: | + pip install localstack # Install LocalStack cli + docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + + aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" + aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + - name: Run zb + run: | + make binary + make bench + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json & + sleep 10 + bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 + + killall -r zot-* + + # clean zot storage + sudo rm -rf /tmp/zot + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 55458ac267..68b6e77e55 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -74,3 +74,45 @@ jobs: - name: Run sync harness run: | make test-sync-harness + gc-stress-s3: + name: GC on S3 with short interval + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Setup localstack service + run: | + pip install localstack # Install LocalStack cli + docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + + aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" + aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + - name: Run zb + run: | + make binary + make bench + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json & + sleep 10 + bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 + + killall -r zot-* + + # clean zot storage + sudo rm -rf /tmp/zot + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + + diff --git a/Makefile b/Makefile index a9e185811c..fe998a8054 100644 --- a/Makefile +++ b/Makefile @@ -357,6 +357,14 @@ test-push-pull: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(H test-push-pull-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL) $(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats +.PHONY: test-garbage-collect +test-garbage-collect: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) + $(BATS) --trace --print-output-on-failure test/blackbox/garbage_collect.bats + +.PHONY: test-garbage-collect-verbose +test-garbage-collect-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) + $(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/garbage_collect.bats + .PHONY: test-push-pull-running-dedupe test-push-pull-running-dedupe: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats diff --git a/errors/errors.go b/errors/errors.go index 41e19590ba..4798560476 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -58,6 +58,7 @@ var ( ErrBadBlob = errors.New("blob: bad blob") ErrBadBlobDigest = errors.New("blob: bad blob digest") ErrBlobReferenced = errors.New("blob: referenced by manifest") + ErrManifestReferenced = errors.New("manifest: referenced by index image") ErrUnknownCode = errors.New("error: unknown error code") ErrBadCACert = errors.New("tls: invalid ca cert") ErrBadUser = errors.New("auth: non-existent user") @@ -155,4 +156,7 @@ var ( ErrGQLEndpointNotFound = errors.New("cli: the server doesn't have a gql endpoint") ErrGQLQueryNotSupported = errors.New("cli: query is not supported or has different arguments") ErrBadHTTPStatusCode = errors.New("cli: the response doesn't contain the expected status code") + ErrFileAlreadyCancelled = errors.New("storageDriver: file already cancelled") + ErrFileAlreadyClosed = errors.New("storageDriver: file already closed") + ErrFileAlreadyCommitted = errors.New("storageDriver: file already committed") ) diff --git a/examples/config-gc.json b/examples/config-gc.json index c4923946e2..cad60105fd 100644 --- a/examples/config-gc.json +++ b/examples/config-gc.json @@ -3,7 +3,10 @@ "storage": { "rootDirectory": "/tmp/zot", "gc": true, - "gcDelay": "1s" + "gcReferrers": true, + "gcDelay": "2h", + "untaggedImageRetentionDelay": "4h", + "gcInterval": "1h" }, "http": { "address": "127.0.0.1", diff --git a/go.mod b/go.mod index b433614b61..890c9f56a8 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/99designs/gqlgen v0.17.35 github.com/Masterminds/semver v1.5.0 - github.com/apex/log v1.9.0 + github.com/apex/log v1.9.0 // indirect github.com/aquasecurity/trivy-db v0.0.0-20230703082116-dc52e83376ce github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/briandowns/spinner v1.23.0 @@ -23,7 +23,6 @@ require ( github.com/gorilla/mux v1.8.0 github.com/hashicorp/golang-lru/v2 v2.0.5 github.com/json-iterator/go v1.1.12 - github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/mapstructure v1.5.0 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba github.com/olekukonko/tablewriter v0.0.5 @@ -367,7 +366,6 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422 // indirect diff --git a/go.sum b/go.sum index 652b1c8eb1..8981a68047 100644 --- a/go.sum +++ b/go.sum @@ -1129,8 +1129,6 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 h1:BcxbplxjtczA1a6d3wYoa7a0WL3rq9DKBMGHeKyjEF0= github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -1257,8 +1255,6 @@ github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/migueleliasweb/go-github-mock v0.0.19 h1:z/88f6wPqZVFnE7s9DbwXMhCtmV/0FofNxc4M7FuSdU= github.com/migueleliasweb/go-github-mock v0.0.19/go.mod h1:dBoCB3W9NjzyABhoGkfI0iSlFpzulAXhI7M+9A4ONYI= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -2063,7 +2059,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index acf9925df8..b22eadaccf 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -23,15 +23,17 @@ var ( ) type StorageConfig struct { - RootDirectory string - Dedupe bool - RemoteCache bool - GC bool - Commit bool - GCDelay time.Duration - GCInterval time.Duration - StorageDriver map[string]interface{} `mapstructure:",omitempty"` - CacheDriver map[string]interface{} `mapstructure:",omitempty"` + RootDirectory string + Dedupe bool + RemoteCache bool + GC bool + Commit bool + GCDelay time.Duration + GCInterval time.Duration + GCReferrers bool + UntaggedImageRetentionDelay time.Duration + StorageDriver map[string]interface{} `mapstructure:",omitempty"` + CacheDriver map[string]interface{} `mapstructure:",omitempty"` } type TLSConfig struct { @@ -188,8 +190,9 @@ func New() *Config { BinaryType: BinaryType, Storage: GlobalStorageConfig{ StorageConfig: StorageConfig{ - GC: true, GCDelay: storageConstants.DefaultGCDelay, - GCInterval: storageConstants.DefaultGCInterval, Dedupe: true, + GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay, + UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + GCInterval: storageConstants.DefaultGCInterval, Dedupe: true, }, }, HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}}, diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index f71ce85709..c717759c65 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -971,7 +971,7 @@ func TestInterruptedBlobUpload(t *testing.T) { defer cm.StopServer() client := resty.New() - blob := make([]byte, 50*1024*1024) + blob := make([]byte, 200*1024*1024) digest := godigest.FromBytes(blob).String() //nolint: dupl @@ -1024,6 +1024,7 @@ func TestInterruptedBlobUpload(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) }) + //nolint: dupl Convey("Test negative interrupt PATCH blob upload", func() { resp, err := client.R().Post(baseURL + "/v2/" + AuthorizedNamespace + "/blobs/uploads/") So(err, ShouldBeNil) @@ -1126,6 +1127,7 @@ func TestInterruptedBlobUpload(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) }) + //nolint: dupl Convey("Test negative interrupt PUT blob upload", func() { resp, err := client.R().Post(baseURL + "/v2/" + AuthorizedNamespace + "/blobs/uploads/") So(err, ShouldBeNil) @@ -6746,6 +6748,12 @@ func TestManifestImageIndex(t *testing.T) { So(digestHdr, ShouldEqual, digest.String()) }) + Convey("Deleting manifest contained by a multiarch image should not be allowed", func() { + resp, err = resty.R().Delete(baseURL + fmt.Sprintf("/v2/index/manifests/%s", m2dgst.String())) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + }) + Convey("Deleting an image index", func() { // delete manifest by tag should pass resp, err = resty.R().Delete(baseURL + "/v2/index/manifests/test:index3") @@ -7296,7 +7304,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { So(digest, ShouldNotBeNil) // monolithic blob upload - injected := inject.InjectFailure(0) + injected := inject.InjectFailure(2) if injected { request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc, bytes.NewReader(content)) tokens := strings.Split(loc, "/") @@ -7369,7 +7377,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { // Testing router path: @Router /v2/{name}/manifests/{reference} [put] //nolint:lll // gofumpt conflicts with lll Convey("Uploading an image manifest blob (when injected simulates that PutImageManifest failed due to 'too many open files' error)", func() { - injected := inject.InjectFailure(1) + injected := inject.InjectFailure(2) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) @@ -7430,6 +7438,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { So(resp.StatusCode, ShouldEqual, http.StatusCreated) } }) + Convey("when index.json is not in json format", func() { resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). SetBody(content).Put(baseURL + "/v2/repotest/manifests/v1.0") @@ -7456,21 +7465,22 @@ func TestInjectTooManyOpenFiles(t *testing.T) { func TestGCSignaturesAndUntaggedManifests(t *testing.T) { Convey("Make controller", t, func() { - repoName := "testrepo" //nolint:goconst - tag := "0.0.1" + Convey("Garbage collect signatures without subject and manifests without tags", func(c C) { + repoName := "testrepo" //nolint:goconst + tag := "0.0.1" - port := test.GetFreePort() - baseURL := test.GetBaseURL(port) - conf := config.New() - conf.HTTP.Port = port + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port - ctlr := makeController(conf, t.TempDir()) + ctlr := makeController(conf, t.TempDir()) - Convey("Garbage collect signatures without subject and manifests without tags", func(c C) { dir := t.TempDir() ctlr.Config.Storage.RootDirectory = dir ctlr.Config.Storage.GC = true ctlr.Config.Storage.GCDelay = 1 * time.Millisecond + ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Millisecond ctlr.Config.Storage.Dedupe = false @@ -7582,75 +7592,88 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) { So(err, ShouldBeNil) }) - // push an image without tag - cfg, layers, manifest, err := test.GetImageComponents(2) //nolint:staticcheck - So(err, ShouldBeNil) + Convey("Overwrite original image, signatures should be garbage-collected", func() { + // push an image without tag + cfg, layers, manifest, err := test.GetImageComponents(2) //nolint:staticcheck + So(err, ShouldBeNil) - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - untaggedManifestDigest := godigest.FromBytes(manifestBuf) + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + untaggedManifestDigest := godigest.FromBytes(manifestBuf) - err = test.UploadImage( - test.Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, untaggedManifestDigest.String()) - So(err, ShouldBeNil) + err = test.UploadImage( + test.Image{ + Config: cfg, + Layers: layers, + Manifest: manifest, + }, baseURL, repoName, untaggedManifestDigest.String()) + So(err, ShouldBeNil) - // overwrite image so that signatures will get invalidated and gc'ed - cfg, layers, manifest, err = test.GetImageComponents(3) //nolint:staticcheck - So(err, ShouldBeNil) + // overwrite image so that signatures will get invalidated and gc'ed + cfg, layers, manifest, err = test.GetImageComponents(3) //nolint:staticcheck + So(err, ShouldBeNil) - err = test.UploadImage( - test.Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, tag) - So(err, ShouldBeNil) + err = test.UploadImage( + test.Image{ + Config: cfg, + Layers: layers, + Manifest: manifest, + }, baseURL, repoName, tag) + So(err, ShouldBeNil) - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - newManifestDigest := godigest.FromBytes(manifestBuf) + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + newManifestDigest := godigest.FromBytes(manifestBuf) - err = ctlr.StoreController.DefaultStore.RunGCRepo(repoName) - So(err, ShouldBeNil) + err = ctlr.StoreController.DefaultStore.RunGCRepo(repoName) + So(err, ShouldBeNil) - // both signatures should be gc'ed - resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, cosignTag)) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + // both signatures should be gc'ed + resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, cosignTag)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - resp, err = resty.R().SetQueryParam("artifactType", notreg.ArtifactTypeNotation).Get( - fmt.Sprintf("%s/v2/%s/referrers/%s", baseURL, repoName, digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) + resp, err = resty.R().SetQueryParam("artifactType", notreg.ArtifactTypeNotation).Get( + fmt.Sprintf("%s/v2/%s/referrers/%s", baseURL, repoName, digest.String())) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) - err = json.Unmarshal(resp.Body(), &index) - So(err, ShouldBeNil) - So(len(index.Manifests), ShouldEqual, 0) + err = json.Unmarshal(resp.Body(), &index) + So(err, ShouldBeNil) + So(len(index.Manifests), ShouldEqual, 0) - resp, err = resty.R().SetQueryParam("artifactType", notreg.ArtifactTypeNotation).Get( - fmt.Sprintf("%s/v2/%s/referrers/%s", baseURL, repoName, newManifestDigest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) + resp, err = resty.R().SetQueryParam("artifactType", notreg.ArtifactTypeNotation).Get( + fmt.Sprintf("%s/v2/%s/referrers/%s", baseURL, repoName, newManifestDigest.String())) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) - err = json.Unmarshal(resp.Body(), &index) - So(err, ShouldBeNil) - So(len(index.Manifests), ShouldEqual, 0) + err = json.Unmarshal(resp.Body(), &index) + So(err, ShouldBeNil) + So(len(index.Manifests), ShouldEqual, 0) - // untagged image should also be gc'ed - resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, untaggedManifestDigest)) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + // untagged image should also be gc'ed + resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, untaggedManifestDigest)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + }) }) Convey("Do not gc manifests which are part of a multiarch image", func(c C) { + repoName := "testrepo" //nolint:goconst + tag := "0.0.1" + + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + + ctlr := makeController(conf, t.TempDir()) + dir := t.TempDir() ctlr.Config.Storage.RootDirectory = dir ctlr.Config.Storage.GC = true - ctlr.Config.Storage.GCDelay = 500 * time.Millisecond + ctlr.Config.Storage.GCDelay = 1 * time.Second + ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Second err := test.WriteImageToFileSystem(test.CreateDefaultImage(), repoName, tag, test.GetDefaultStoreController(dir, ctlr.Log)) @@ -7787,7 +7810,10 @@ func TestPeriodicGC(t *testing.T) { subPaths := make(map[string]config.StorageConfig) - subPaths["/a"] = config.StorageConfig{RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false} //nolint:lll // gofumpt conflicts with lll + subPaths["/a"] = config.StorageConfig{ + RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, + UntaggedImageRetentionDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false, + } //nolint:lll // gofumpt conflicts with lll ctlr.Config.Storage.Dedupe = false ctlr.Config.Storage.SubPaths = subPaths diff --git a/pkg/api/routes.go b/pkg/api/routes.go index fcccc1e96e..0ccafcdfd1 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -832,6 +832,11 @@ func (rh *RouteHandler) DeleteManifest(response http.ResponseWriter, request *ht details["reference"] = reference e := apiErr.NewError(apiErr.UNSUPPORTED).AddDetail(details) zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) + } else if errors.Is(err, zerr.ErrManifestReferenced) { + // manifest is part of an index image, don't allow index manipulations. + details["reference"] = reference + e := apiErr.NewError(apiErr.DENIED).AddDetail(details) + zcommon.WriteJSON(response, http.StatusMethodNotAllowed, apiErr.NewErrorList(e)) } else { rh.c.Log.Error().Err(err).Msg("unexpected error") response.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index 7860543bdc..bc9057cf48 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -468,7 +468,7 @@ func TestNegativeServerResponse(t *testing.T) { dir := t.TempDir() - imageStore := local.NewImageStore(dir, false, 0, false, false, + imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{ diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 2d4876a82c..ce00cbe891 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -27,7 +27,6 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" storageConstants "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/storage/s3" ) // metadataConfig reports metadata after parsing, which we use to track @@ -631,6 +630,10 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z config.Storage.GCDelay = 0 } + if viperInstance.Get("storage::gcdelay") == nil { + config.Storage.UntaggedImageRetentionDelay = 0 + } + if viperInstance.Get("storage::gcinterval") == nil { config.Storage.GCInterval = 0 } @@ -649,7 +652,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z // s3 dedup=false, check for previous dedup usage and set to true if cachedb found if !config.Storage.Dedupe && config.Storage.StorageDriver != nil { cacheDir, _ := config.Storage.StorageDriver["rootdirectory"].(string) - cachePath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + cachePath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err := os.Stat(cachePath); err == nil { log.Info().Msg("Config: dedupe set to false for s3 driver but used to be true.") @@ -667,10 +670,10 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z storageConfig.RemoteCache = true } - // s3 dedup=false, check for previous dedup usage and set to true if cachedb found + // s3 dedup=false, check for previous dedupe usage and set to true if cachedb found if !storageConfig.Dedupe && storageConfig.StorageDriver != nil { subpathCacheDir, _ := storageConfig.StorageDriver["rootdirectory"].(string) - subpathCachePath := path.Join(subpathCacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + subpathCachePath := path.Join(subpathCacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err := os.Stat(subpathCachePath); err == nil { log.Info().Msg("Config: dedupe set to false for s3 driver but used to be true. ") @@ -682,11 +685,21 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z // if gc is enabled if storageConfig.GC { + // and gcReferrers is not set, it is set to default value + if !viperInstance.IsSet("storage::subpaths::" + name + "::gcreferrers") { + storageConfig.GCReferrers = true + } + // and gcDelay is not set, it is set to default value if !viperInstance.IsSet("storage::subpaths::" + name + "::gcdelay") { storageConfig.GCDelay = storageConstants.DefaultGCDelay } + // and retentionDelay is not set, it is set to default value + if !viperInstance.IsSet("storage::subpaths::" + name + "::retentiondelay") { + storageConfig.UntaggedImageRetentionDelay = storageConstants.DefaultUntaggedImgeRetentionDelay + } + // and gcInterval is not set, it is set to default value if !viperInstance.IsSet("storage::subpaths::" + name + "::gcinterval") { storageConfig.GCInterval = storageConstants.DefaultGCInterval diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index d03592765c..f6859e4f99 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -14,7 +14,6 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/cli" storageConstants "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/storage/s3" . "zotregistry.io/zot/pkg/test" ) @@ -521,7 +520,7 @@ func TestVerify(t *testing.T) { // s3 dedup=false, check for previous dedup usage and set to true if cachedb found cacheDir := t.TempDir() - existingDBPath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + existingDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) _, err = os.Create(existingDBPath) So(err, ShouldBeNil) @@ -537,7 +536,7 @@ func TestVerify(t *testing.T) { // subpath s3 dedup=false, check for previous dedup usage and set to true if cachedb found cacheDir = t.TempDir() - existingDBPath = path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + existingDBPath = path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) _, err = os.Create(existingDBPath) So(err, ShouldBeNil) diff --git a/pkg/extensions/extension_image_trust_test.go b/pkg/extensions/extension_image_trust_test.go index 2c81c352f5..e55adb8f51 100644 --- a/pkg/extensions/extension_image_trust_test.go +++ b/pkg/extensions/extension_image_trust_test.go @@ -149,7 +149,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ @@ -267,7 +267,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ @@ -385,7 +385,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ @@ -558,7 +558,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ @@ -813,7 +813,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ diff --git a/pkg/extensions/extension_scrub.go b/pkg/extensions/extension_scrub.go index 2997a23d5b..1d67acde02 100644 --- a/pkg/extensions/extension_scrub.go +++ b/pkg/extensions/extension_scrub.go @@ -4,8 +4,6 @@ package extensions import ( - "errors" - "io" "time" "zotregistry.io/zot/pkg/api/config" @@ -30,19 +28,25 @@ func EnableScrubExtension(config *config.Config, log log.Logger, storeController log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll } - generator := &taskGenerator{ - imgStore: storeController.DefaultStore, - log: log, + // is local imagestore (because of umoci dependency which works only locally) + if config.Storage.StorageDriver == nil { + generator := &taskGenerator{ + imgStore: storeController.DefaultStore, + log: log, + } + sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } - sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) if config.Storage.SubPaths != nil { for route := range config.Storage.SubPaths { - generator := &taskGenerator{ - imgStore: storeController.SubStore[route], - log: log, + // is local imagestore (because of umoci dependency which works only locally) + if config.Storage.SubPaths[route].StorageDriver == nil { + generator := &taskGenerator{ + imgStore: storeController.SubStore[route], + log: log, + } + sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } - sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } } } else { @@ -59,8 +63,7 @@ type taskGenerator struct { func (gen *taskGenerator) Next() (scheduler.Task, error) { repo, err := gen.imgStore.GetNextRepository(gen.lastRepo) - - if err != nil && !errors.Is(err, io.EOF) { + if err != nil { return nil, err } diff --git a/pkg/extensions/lint/lint_test.go b/pkg/extensions/lint/lint_test.go index 8db278a823..240cafaf85 100644 --- a/pkg/extensions/lint/lint_test.go +++ b/pkg/extensions/lint/lint_test.go @@ -490,7 +490,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { var index ispec.Index linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) indexContent, err := imgStore.GetIndexContent("zot-test") @@ -522,7 +522,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { var index ispec.Index linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) indexContent, err := imgStore.GetIndexContent("zot-test") @@ -592,7 +592,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) @@ -654,7 +654,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) @@ -718,7 +718,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) @@ -781,7 +781,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000) @@ -879,7 +879,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) err = os.Chmod(path.Join(dir, "zot-test", "blobs", "sha256", manifest.Config.Digest.Encoded()), 0o000) diff --git a/pkg/extensions/scrub/scrub_test.go b/pkg/extensions/scrub/scrub_test.go index 1fd3f95ac8..298148504e 100644 --- a/pkg/extensions/scrub/scrub_test.go +++ b/pkg/extensions/scrub/scrub_test.go @@ -198,7 +198,7 @@ func TestRunScrubRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, 1*time.Second, true, true, log, metrics, nil, cacheDriver) srcStorageCtlr := test.GetDefaultStoreController(dir, log) @@ -234,7 +234,7 @@ func TestRunScrubRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, 1*time.Second, true, true, log, metrics, nil, cacheDriver) srcStorageCtlr := test.GetDefaultStoreController(dir, log) @@ -276,8 +276,8 @@ func TestRunScrubRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, - true, true, log, metrics, nil, cacheDriver, + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, + 1*time.Second, true, true, log, metrics, nil, cacheDriver, ) srcStorageCtlr := test.GetDefaultStoreController(dir, log) diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 9dc8c8555e..b93f56b473 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -322,8 +322,8 @@ func TestImageFormat(t *testing.T) { dbDir := t.TempDir() metrics := monitoring.NewMetricsServer(false, log) - defaultStore := local.NewImageStore(imgDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + defaultStore := local.NewImageStore(imgDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{DefaultStore: defaultStore} params := boltdb.DBParameters{ diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index c867b7475e..6f791a5577 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -25,6 +25,7 @@ import ( mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" "zotregistry.io/zot/pkg/storage/local" storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" @@ -73,14 +74,14 @@ func TestMultipleStoragePath(t *testing.T) { // Create ImageStore - firstStore := local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, - nil, nil) + firstStore := local.NewImageStore(firstRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) - secondStore := local.NewImageStore(secondRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, - nil, nil) + secondStore := local.NewImageStore(secondRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) - thirdStore := local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, - nil, nil) + thirdStore := local.NewImageStore(thirdRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} @@ -188,7 +189,8 @@ func TestTrivyLibraryErrors(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -405,7 +407,8 @@ func TestImageScannable(t *testing.T) { // Continue with initializing the objects the scanner depends on metrics := monitoring.NewMetricsServer(false, log) - store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -471,7 +474,8 @@ func TestDefaultTrivyDBUrl(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -515,7 +519,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) { func TestIsIndexScanable(t *testing.T) { Convey("IsIndexScanable", t, func() { storeController := storage.StoreController{} - storeController.DefaultStore = &local.ImageStoreLocal{} + storeController.DefaultStore = &imagestore.ImageStore{} metaDB := &boltdb.BoltDB{} log := log.NewLogger("debug", "") diff --git a/pkg/extensions/search/cve/trivy/scanner_test.go b/pkg/extensions/search/cve/trivy/scanner_test.go index 5e55537ab4..1ff033f800 100644 --- a/pkg/extensions/search/cve/trivy/scanner_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_test.go @@ -182,7 +182,7 @@ func TestVulnerableLayer(t *testing.T) { tempDir := t.TempDir() log := log.NewLogger("debug", "") - imageStore := local.NewImageStore(tempDir, false, 0, false, false, + imageStore := local.NewImageStore(tempDir, false, false, 0, 0, false, false, log, monitoring.NewMetricsServer(false, log), nil, nil) storeController := storage.StoreController{ diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 4af69febe8..f1feec88bf 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -1193,7 +1193,7 @@ func TestExpandedRepoInfo(t *testing.T) { ctlr := api.NewController(conf) - imageStore := local.NewImageStore(tempDir, false, 0, false, false, + imageStore := local.NewImageStore(tempDir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{ @@ -1325,8 +1325,8 @@ func TestExpandedRepoInfo(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - testStorage := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + testStorage := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) @@ -1671,7 +1671,7 @@ func TestExpandedRepoInfo(t *testing.T) { conf.Extensions.Search.CVE = nil ctlr := api.NewController(conf) - imageStore := local.NewImageStore(conf.Storage.RootDirectory, false, 0, false, false, + imageStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{ @@ -5420,8 +5420,8 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + storage := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) indexBlob, err := storage.GetIndexContent(repo) So(err, ShouldBeNil) @@ -5497,8 +5497,8 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + storage := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) indexBlob, err := storage.GetIndexContent(repo) So(err, ShouldBeNil) diff --git a/pkg/extensions/search/userprefs_test.go b/pkg/extensions/search/userprefs_test.go index a776007667..4e7315f671 100644 --- a/pkg/extensions/search/userprefs_test.go +++ b/pkg/extensions/search/userprefs_test.go @@ -543,7 +543,7 @@ func TestChangingRepoState(t *testing.T) { } // ------ Create the test repos - defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, 0, false, false, + defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) err = WriteImageToFileSystem(img, accesibleRepo, "tag", storage.StoreController{ diff --git a/pkg/extensions/sync/constants/consts.go b/pkg/extensions/sync/constants/consts.go index 9dff8e9502..69484d241d 100644 --- a/pkg/extensions/sync/constants/consts.go +++ b/pkg/extensions/sync/constants/consts.go @@ -2,7 +2,8 @@ package constants // references type. const ( - Oras = "OrasReference" - Cosign = "CosignSignature" - OCI = "OCIReference" + Oras = "OrasReference" + Cosign = "CosignSignature" + OCI = "OCIReference" + SyncBlobUploadDir = ".sync" ) diff --git a/pkg/extensions/sync/local.go b/pkg/extensions/sync/local.go index 2a7108dc21..ca79f6081f 100644 --- a/pkg/extensions/sync/local.go +++ b/pkg/extensions/sync/local.go @@ -281,8 +281,9 @@ func getImageStoreFromImageReference(imageReference types.ImageReference, repo, metrics := monitoring.NewMetricsServer(false, log.Logger{}) - tempImageStore := local.NewImageStore(tempRootDir, false, - storageConstants.DefaultGCDelay, false, false, log.Logger{}, metrics, nil, nil) + tempImageStore := local.NewImageStore(tempRootDir, false, false, + storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay, + false, false, log.Logger{}, metrics, nil, nil) return tempImageStore } diff --git a/pkg/extensions/sync/oci_layout.go b/pkg/extensions/sync/oci_layout.go index 1d1377193b..5806d96bc1 100644 --- a/pkg/extensions/sync/oci_layout.go +++ b/pkg/extensions/sync/oci_layout.go @@ -12,6 +12,7 @@ import ( "github.com/containers/image/v5/types" "github.com/gofrs/uuid" + "zotregistry.io/zot/pkg/extensions/sync/constants" "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/test/inject" @@ -39,7 +40,7 @@ func (oci OciLayoutStorageImpl) GetContext() *types.SystemContext { func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) (types.ImageReference, error) { localImageStore := oci.storeController.GetImageStore(repo) - tempSyncPath := path.Join(localImageStore.RootDir(), repo, SyncBlobUploadDir) + tempSyncPath := path.Join(localImageStore.RootDir(), repo, constants.SyncBlobUploadDir) // create session folder uuid, err := uuid.NewV4() diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 80415a5b2a..43a77902f2 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -68,8 +68,8 @@ func TestInjectSyncUtils(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil, + imageStore := local.NewImageStore(t.TempDir(), false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil, ) injected = inject.InjectFailure(0) @@ -182,8 +182,8 @@ func TestLocalRegistry(t *testing.T) { UseRelPaths: true, }, log) - syncImgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := "repo" registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log) @@ -300,8 +300,8 @@ func TestLocalRegistry(t *testing.T) { MandatoryAnnotations: []string{"annot1"}, }, log) - syncImgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, linter, cacheDriver) + syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, linter, cacheDriver) repoName := "repo" registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 1a7c019154..644ab0ca4c 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -43,6 +43,7 @@ import ( extconf "zotregistry.io/zot/pkg/extensions/config" syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/extensions/sync" + syncConstants "zotregistry.io/zot/pkg/extensions/sync/constants" "zotregistry.io/zot/pkg/log" mTypes "zotregistry.io/zot/pkg/meta/types" storageConstants "zotregistry.io/zot/pkg/storage/constants" @@ -591,7 +592,7 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - err = os.MkdirAll(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o000) + err = os.MkdirAll(path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir), 0o000) if err != nil { panic(err) } @@ -604,7 +605,7 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - err = os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o755) + err = os.Chmod(path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir), 0o755) if err != nil { panic(err) } @@ -1687,7 +1688,7 @@ func TestPermsDenied(t *testing.T) { defer dcm.StopServer() - syncSubDir := path.Join(destDir, testImage, sync.SyncBlobUploadDir) + syncSubDir := path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir) err := os.MkdirAll(syncSubDir, 0o755) So(err, ShouldBeNil) @@ -1698,7 +1699,7 @@ func TestPermsDenied(t *testing.T) { dcm.StartAndWait(destPort) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't get a local image reference", 20*time.Second) + "couldn't get a local image reference", 50*time.Second) if err != nil { panic(err) } @@ -4911,7 +4912,7 @@ func TestOnDemandPullsOnce(t *testing.T) { done := make(chan bool) var maxLen int - syncBlobUploadDir := path.Join(destDir, testImage, sync.SyncBlobUploadDir) + syncBlobUploadDir := path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir) go func() { for { @@ -4994,7 +4995,7 @@ func TestError(t *testing.T) { }() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) + "couldn't commit image to local image store", 30*time.Second) if err != nil { panic(err) } @@ -6531,7 +6532,7 @@ func pushRepo(url, repoName string) godigest.Digest { func waitSync(rootDir, repoName string) { // wait for .sync subdirs to be removed for { - dirs, err := os.ReadDir(path.Join(rootDir, repoName, sync.SyncBlobUploadDir)) + dirs, err := os.ReadDir(path.Join(rootDir, repoName, syncConstants.SyncBlobUploadDir)) if err == nil && len(dirs) == 0 { // stop watching /.sync/ subdirs return diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 5eb210065c..7513cb562b 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -29,10 +29,6 @@ import ( "zotregistry.io/zot/pkg/test/inject" ) -const ( - SyncBlobUploadDir = ".sync" -) - // Get sync.FileCredentials from file. func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) { credsFile, err := os.ReadFile(filepath) diff --git a/pkg/meta/hooks_test.go b/pkg/meta/hooks_test.go index 733966bd69..f52a25e568 100644 --- a/pkg/meta/hooks_test.go +++ b/pkg/meta/hooks_test.go @@ -31,8 +31,8 @@ func TestOnUpdateManifest(t *testing.T) { storeController := storage.StoreController{} log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storeController.DefaultStore = local.NewImageStore(rootDir, true, 1*time.Second, - true, true, log, metrics, nil, nil, + storeController.DefaultStore = local.NewImageStore(rootDir, true, true, 1*time.Second, + 1*time.Second, true, true, log, metrics, nil, nil, ) params := boltdb.DBParameters{ @@ -72,8 +72,8 @@ func TestOnUpdateManifest(t *testing.T) { storeController := storage.StoreController{} log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storeController.DefaultStore = local.NewImageStore(rootDir, true, 1*time.Second, - true, true, log, metrics, nil, nil, + storeController.DefaultStore = local.NewImageStore(rootDir, true, true, 1*time.Second, + 1*time.Second, true, true, log, metrics, nil, nil, ) metaDB := mocks.MetaDBMock{ diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index 13f016fb26..67a351b047 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -399,7 +399,7 @@ func TestParseStorageDynamoWrapper(t *testing.T) { func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { Convey("Test with simple case", func() { - imageStore := local.NewImageStore(rootDir, false, 0, false, false, + imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} @@ -485,7 +485,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { }) Convey("Accept orphan signatures", func() { - imageStore := local.NewImageStore(rootDir, false, 0, false, false, + imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} @@ -542,7 +542,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { }) Convey("Check statistics after load", func() { - imageStore := local.NewImageStore(rootDir, false, 0, false, false, + imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} diff --git a/pkg/storage/cache/boltdb.go b/pkg/storage/cache/boltdb.go index 89d068911c..f146e0dd2d 100644 --- a/pkg/storage/cache/boltdb.go +++ b/pkg/storage/cache/boltdb.go @@ -77,6 +77,10 @@ func NewBoltDBCache(parameters interface{}, log zlog.Logger) Cache { } } +func (d *BoltDBDriver) UsesRelativePaths() bool { + return d.useRelPaths +} + func (d *BoltDBDriver) Name() string { return "boltdb" } diff --git a/pkg/storage/cache/cacheinterface.go b/pkg/storage/cache/cacheinterface.go index d02fe95cf9..de1df6b2c1 100644 --- a/pkg/storage/cache/cacheinterface.go +++ b/pkg/storage/cache/cacheinterface.go @@ -19,4 +19,7 @@ type Cache interface { // Delete a blob from the cachedb. DeleteBlob(digest godigest.Digest, path string) error + + // UsesRelativePaths returns if cache is storing blobs relative to cache rootDir + UsesRelativePaths() bool } diff --git a/pkg/storage/cache/dynamodb.go b/pkg/storage/cache/dynamodb.go index 890d5177f3..4aafed0900 100644 --- a/pkg/storage/cache/dynamodb.go +++ b/pkg/storage/cache/dynamodb.go @@ -99,6 +99,10 @@ func NewDynamoDBCache(parameters interface{}, log zlog.Logger) Cache { return driver } +func (d *DynamoDBDriver) UsesRelativePaths() bool { + return false +} + func (d *DynamoDBDriver) Name() string { return "dynamodb" } diff --git a/pkg/storage/cache_test.go b/pkg/storage/cache_test.go index 01b31b7712..6158f25eae 100644 --- a/pkg/storage/cache_test.go +++ b/pkg/storage/cache_test.go @@ -35,6 +35,8 @@ func TestCache(t *testing.T) { }, log) So(cacheDriver, ShouldNotBeNil) + So(cacheDriver.UsesRelativePaths(), ShouldBeTrue) + name := cacheDriver.Name() So(name, ShouldEqual, "boltdb") diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index cdb44ece8e..af1e997522 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -5,22 +5,22 @@ import ( "encoding/json" "errors" "fmt" - "io" "math/rand" "path" "strings" "time" + "github.com/docker/distribution/registry/storage/driver" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/schema" imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" oras "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" zerr "zotregistry.io/zot/errors" zcommon "zotregistry.io/zot/pkg/common" + zlog "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/scheduler" storageConstants "zotregistry.io/zot/pkg/storage/constants" storageTypes "zotregistry.io/zot/pkg/storage/types" @@ -62,7 +62,7 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc } func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte, - log zerolog.Logger, + log zlog.Logger, ) (godigest.Digest, error) { // validate the manifest if !IsSupportedMediaType(mediaType) { @@ -105,7 +105,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy continue } - ok, _, err := imgStore.StatBlob(repo, layer.Digest) + ok, _, _, err := imgStore.StatBlob(repo, layer.Digest) if !ok || err != nil { log.Error().Err(err).Str("digest", layer.Digest.String()).Msg("missing layer blob") @@ -136,7 +136,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy } for _, manifest := range indexManifest.Manifests { - if ok, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil { + if ok, _, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil { log.Error().Err(err).Str("digest", manifest.Digest.String()).Msg("missing manifest blob") return "", zerr.ErrBadManifest @@ -147,7 +147,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy return "", nil } -func GetAndValidateRequestDigest(body []byte, digestStr string, log zerolog.Logger) (godigest.Digest, error) { +func GetAndValidateRequestDigest(body []byte, digestStr string, log zlog.Logger) (godigest.Digest, error) { bodyDigest := godigest.FromBytes(body) d, err := godigest.Parse(digestStr) @@ -169,7 +169,7 @@ CheckIfIndexNeedsUpdate verifies if an index needs to be updated given a new man Returns whether or not index needs update, in the latter case it will also return the previous digest. */ func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor, - log zerolog.Logger, + log zlog.Logger, ) (bool, godigest.Digest, error) { var oldDgst godigest.Digest @@ -242,11 +242,15 @@ func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor, } // GetIndex returns the contents of index.json. -func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) (ispec.Index, error) { +func GetIndex(imgStore storageTypes.ImageStore, repo string, log zlog.Logger) (ispec.Index, error) { var index ispec.Index buf, err := imgStore.GetIndexContent(repo) if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return index, zerr.ErrRepoNotFound + } + return index, err } @@ -260,7 +264,7 @@ func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) } // GetImageIndex returns a multiarch type image. -func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger, ) (ispec.Index, error) { var imageIndex ispec.Index @@ -285,7 +289,7 @@ func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godiges return imageIndex, nil } -func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger, ) (ispec.Manifest, error) { var manifestContent ispec.Manifest @@ -352,7 +356,7 @@ index, ensure that they do not have a name or they are not in other manifest indexes else GC can never clean them. */ func UpdateIndexWithPrunedImageManifests(imgStore storageTypes.ImageStore, index *ispec.Index, repo string, - desc ispec.Descriptor, oldDgst godigest.Digest, log zerolog.Logger, + desc ispec.Descriptor, oldDgst godigest.Digest, log zlog.Logger, ) error { if (desc.MediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") { otherImgIndexes := []ispec.Descriptor{} @@ -385,7 +389,7 @@ same constitutent manifests so that they can be garbage-collected correctly PruneImageManifestsFromIndex is a helper routine to achieve this. */ func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, //nolint:gocyclo,lll - outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger, + outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zlog.Logger, ) ([]ispec.Descriptor, error) { dir := path.Join(imgStore.RootDir(), repo) @@ -459,8 +463,8 @@ func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, return prunedManifests, nil } -func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string, - bdigest, mdigest godigest.Digest, log zerolog.Logger, +func isBlobReferencedInImageManifest(imgStore storageTypes.ImageStore, repo string, + bdigest, mdigest godigest.Digest, log zlog.Logger, ) (bool, error) { if bdigest == mdigest { return true, nil @@ -487,16 +491,14 @@ func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string, return false, nil } -func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, - digest godigest.Digest, index ispec.Index, log zerolog.Logger, +func IsBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, + digest godigest.Digest, index ispec.Index, log zlog.Logger, ) (bool, error) { for _, desc := range index.Manifests { var found bool switch desc.MediaType { case ispec.MediaTypeImageIndex: - /* this branch is not needed, because every manifests in index is already checked - when this one is hit, all manifests are referenced in index.json */ indexImage, err := GetImageIndex(imgStore, repo, desc.Digest, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). @@ -505,9 +507,9 @@ func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, return false, err } - found, _ = isBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log) + found, _ = IsBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log) case ispec.MediaTypeImageManifest: - found, _ = isBlobReferencedInManifest(imgStore, repo, digest, desc.Digest, log) + found, _ = isBlobReferencedInImageManifest(imgStore, repo, digest, desc.Digest, log) } if found { @@ -519,7 +521,7 @@ func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, } func IsBlobReferenced(imgStore storageTypes.ImageStore, repo string, - digest godigest.Digest, log zerolog.Logger, + digest godigest.Digest, log zlog.Logger, ) (bool, error) { dir := path.Join(imgStore.RootDir(), repo) if !imgStore.DirExists(dir) { @@ -531,7 +533,133 @@ func IsBlobReferenced(imgStore storageTypes.ImageStore, repo string, return false, err } - return isBlobReferencedInImageIndex(imgStore, repo, digest, index, log) + return IsBlobReferencedInImageIndex(imgStore, repo, digest, index, log) +} + +/* Garbage Collection */ + +func AddImageManifestBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger, +) error { + manifestContent, err := GetImageManifest(imgStore, repo, mdigest, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + refBlobs[mdigest.String()] = true + refBlobs[manifestContent.Config.Digest.String()] = true + + // if there is a Subject, it may not exist yet and that is ok + if manifestContent.Subject != nil { + refBlobs[manifestContent.Subject.Digest.String()] = true + } + + for _, layer := range manifestContent.Layers { + refBlobs[layer.Digest.String()] = true + } + + return nil +} + +func AddORASImageManifestBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger, +) error { + manifestContent, err := GetOrasManifestByDigest(imgStore, repo, mdigest, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + refBlobs[mdigest.String()] = true + + // if there is a Subject, it may not exist yet and that is ok + if manifestContent.Subject != nil { + refBlobs[manifestContent.Subject.Digest.String()] = true + } + + for _, blob := range manifestContent.Blobs { + refBlobs[blob.Digest.String()] = true + } + + return nil +} + +func AddImageIndexBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger, +) error { + index, err := GetImageIndex(imgStore, repo, mdigest, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + refBlobs[mdigest.String()] = true + + // if there is a Subject, it may not exist yet and that is ok + if index.Subject != nil { + refBlobs[index.Subject.Digest.String()] = true + } + + for _, manifest := range index.Manifests { + refBlobs[manifest.Digest.String()] = true + } + + return nil +} + +func AddIndexBlobToReferences(imgStore storageTypes.ImageStore, + repo string, index ispec.Index, refBlobs map[string]bool, log zlog.Logger, +) error { + for _, desc := range index.Manifests { + switch desc.MediaType { + case ispec.MediaTypeImageIndex: + if err := AddImageIndexBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read blobs in multiarch(index) image") + + return err + } + case ispec.MediaTypeImageManifest: + if err := AddImageManifestBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read blobs in image manifest") + + return err + } + case oras.MediaTypeArtifactManifest: + if err := AddORASImageManifestBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read blobs in image manifest") + + return err + } + } + } + + return nil +} + +func AddRepoBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, refBlobs map[string]bool, log zlog.Logger, +) error { + dir := path.Join(imgStore.RootDir(), repo) + if !imgStore.DirExists(dir) { + return zerr.ErrRepoNotFound + } + + index, err := GetIndex(imgStore, repo, log) + if err != nil { + return err + } + + return AddIndexBlobToReferences(imgStore, repo, index, refBlobs, log) } func ApplyLinter(imgStore storageTypes.ImageStore, linter Lint, repo string, descriptor ispec.Descriptor, @@ -580,7 +708,7 @@ func IsSignature(descriptor ispec.Descriptor) bool { } func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactType string, - log zerolog.Logger, + log zlog.Logger, ) ([]oras.Descriptor, error) { if err := gdigest.Validate(); err != nil { return nil, err @@ -638,7 +766,7 @@ func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest god } func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactTypes []string, - log zerolog.Logger, + log zlog.Logger, ) (ispec.Index, error) { nilIndex := ispec.Index{} @@ -741,7 +869,7 @@ func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godiges return index, nil } -func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger, ) (oras.Manifest, error) { var artManifest oras.Manifest @@ -827,7 +955,7 @@ type DedupeTaskGenerator struct { and generating a task for each unprocessed one*/ lastDigests []godigest.Digest done bool - Log zerolog.Logger + Log zlog.Logger } func (gen *DedupeTaskGenerator) Next() (scheduler.Task, error) { @@ -879,11 +1007,11 @@ type dedupeTask struct { // blobs paths with the same digest ^ duplicateBlobs []string dedupe bool - log zerolog.Logger + log zlog.Logger } func newDedupeTask(imgStore storageTypes.ImageStore, digest godigest.Digest, dedupe bool, - duplicateBlobs []string, log zerolog.Logger, + duplicateBlobs []string, log zlog.Logger, ) *dedupeTask { return &dedupeTask{imgStore, digest, duplicateBlobs, dedupe, log} } @@ -929,8 +1057,7 @@ func (gen *GCTaskGenerator) Next() (scheduler.Task, error) { gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second) repo, err := gen.ImgStore.GetNextRepository(gen.lastRepo) - - if err != nil && !errors.Is(err, io.EOF) { + if err != nil { return nil, err } diff --git a/pkg/storage/common/common_test.go b/pkg/storage/common/common_test.go index 315d9640db..782bde5f59 100644 --- a/pkg/storage/common/common_test.go +++ b/pkg/storage/common/common_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "os" + "path" "testing" godigest "github.com/opencontainers/go-digest" @@ -36,8 +37,8 @@ func TestValidateManifest(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) content := []byte("this is a blob") digest := godigest.FromBytes(content) @@ -81,6 +82,37 @@ func TestValidateManifest(t *testing.T) { So(internalErr.GetDetails()["jsonSchemaValidation"], ShouldEqual, "[schemaVersion: Must be less than or equal to 2]") }) + Convey("bad config blob", func() { + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + configBlobPath := imgStore.BlobPath("test", cdigest) + + err := os.WriteFile(configBlobPath, []byte("bad config blob"), 0o000) + So(err, ShouldBeNil) + + body, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + // this was actually an umoci error on config blob + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body) + So(err, ShouldBeNil) + }) + Convey("manifest with non-distributable layers", func() { content := []byte("this blob doesn't exist") digest := godigest.FromBytes(content) @@ -124,29 +156,29 @@ func TestGetReferrersErrors(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, false, - true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, true, log, metrics, nil, cacheDriver) artifactType := "application/vnd.example.icecream.v1" validDigest := godigest.FromBytes([]byte("blob")) Convey("Trigger invalid digest error", func(c C) { _, err := common.GetReferrers(imgStore, "zot-test", "invalidDigest", - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", "invalidDigest", - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) Convey("Trigger repo not found error", func(c C) { _, err := common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) @@ -179,11 +211,11 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) @@ -198,11 +230,11 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) @@ -227,11 +259,11 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", digest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) @@ -245,7 +277,7 @@ func TestGetReferrersErrors(t *testing.T) { }, } - _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger()) + _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log) So(err, ShouldNotBeNil) }) @@ -272,7 +304,7 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) }) @@ -306,7 +338,7 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldBeNil) }) @@ -326,7 +358,7 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{}, log.With().Caller().Logger()) + []string{}, log) So(err, ShouldNotBeNil) }) @@ -348,7 +380,7 @@ func TestGetReferrersErrors(t *testing.T) { } ref, err := common.GetReferrers(imgStore, "zot-test", validDigest, - []string{"art.type"}, log.With().Caller().Logger()) + []string{"art.type"}, log) So(err, ShouldBeNil) So(len(ref.Manifests), ShouldEqual, 0) }) @@ -356,7 +388,7 @@ func TestGetReferrersErrors(t *testing.T) { } func TestGetImageIndexErrors(t *testing.T) { - log := zerolog.New(os.Stdout) + log := log.Logger{Logger: zerolog.New(os.Stdout)} Convey("Trigger invalid digest error", t, func(c C) { imgStore := &mocks.MockedImageStore{} @@ -400,3 +432,193 @@ func TestIsSignature(t *testing.T) { So(isSingature, ShouldBeFalse) }) } + +func TestGarbageCollectManifestErrors(t *testing.T) { + Convey("Make imagestore and upload manifest", t, func(c C) { + dir := t.TempDir() + + repoName := "test" + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) + + Convey("trigger repo not found in GetReferencedBlobs()", func() { + err := common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldNotBeNil) + }) + + content := []byte("this is a blob") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + _, blen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), digest) + So(err, ShouldBeNil) + So(blen, ShouldEqual, len(content)) + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + body, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(body) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, body) + So(err, ShouldBeNil) + + Convey("trigger GetIndex error in GetReferencedBlobs", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o755) + So(err, ShouldBeNil) + }() + + err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldNotBeNil) + }) + + Convey("trigger GetImageManifest error in GetReferencedBlobsInImageManifest", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o755) + So(err, ShouldBeNil) + }() + + err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldNotBeNil) + }) + }) +} + +func TestGarbageCollectIndexErrors(t *testing.T) { + Convey("Make imagestore and upload manifest", t, func(c C) { + dir := t.TempDir() + + repoName := "test" + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) + + content := []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + for i := 0; i < 4; i++ { + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldBeNil) + + Convey("trigger GetImageIndex error in GetReferencedBlobsInImageIndex", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o755) + So(err, ShouldBeNil) + }() + + err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/pkg/storage/constants/constants.go b/pkg/storage/constants/constants.go index 55ea6b038e..7b9b49f849 100644 --- a/pkg/storage/constants/constants.go +++ b/pkg/storage/constants/constants.go @@ -6,20 +6,22 @@ import ( const ( // BlobUploadDir defines the upload directory for blob uploads. - BlobUploadDir = ".uploads" - SchemaVersion = 2 - DefaultFilePerms = 0o600 - DefaultDirPerms = 0o700 - RLOCK = "RLock" - RWLOCK = "RWLock" - BlobsCache = "blobs" - DuplicatesBucket = "duplicates" - OriginalBucket = "original" - DBExtensionName = ".db" - DBCacheLockCheckTimeout = 10 * time.Second - BoltdbName = "cache" - DynamoDBDriverName = "dynamodb" - DefaultGCDelay = 1 * time.Hour - DefaultGCInterval = 1 * time.Hour - S3StorageDriverName = "s3" + BlobUploadDir = ".uploads" + SchemaVersion = 2 + DefaultFilePerms = 0o600 + DefaultDirPerms = 0o700 + RLOCK = "RLock" + RWLOCK = "RWLock" + BlobsCache = "blobs" + DuplicatesBucket = "duplicates" + OriginalBucket = "original" + DBExtensionName = ".db" + DBCacheLockCheckTimeout = 10 * time.Second + BoltdbName = "cache" + DynamoDBDriverName = "dynamodb" + DefaultGCDelay = 1 * time.Hour + DefaultUntaggedImgeRetentionDelay = 24 * time.Hour + DefaultGCInterval = 1 * time.Hour + S3StorageDriverName = "s3" + LocalStorageDriverName = "local" ) diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go new file mode 100644 index 0000000000..3b611d2dc5 --- /dev/null +++ b/pkg/storage/imagestore/imagestore.go @@ -0,0 +1,2252 @@ +package imagestore + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/docker/distribution/registry/storage/driver" + guuid "github.com/gofrs/uuid" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" + + zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/extensions/monitoring" + syncConstants "zotregistry.io/zot/pkg/extensions/sync/constants" + zlog "zotregistry.io/zot/pkg/log" + zreg "zotregistry.io/zot/pkg/regexp" + "zotregistry.io/zot/pkg/scheduler" + "zotregistry.io/zot/pkg/storage/cache" + common "zotregistry.io/zot/pkg/storage/common" + storageConstants "zotregistry.io/zot/pkg/storage/constants" + storageTypes "zotregistry.io/zot/pkg/storage/types" + "zotregistry.io/zot/pkg/test/inject" +) + +const ( + cosignSignatureTagSuffix = "sig" + SBOMTagSuffix = "sbom" +) + +// ImageStore provides the image storage operations. +type ImageStore struct { + rootDir string + storeDriver storageTypes.Driver + lock *sync.RWMutex + log zlog.Logger + metrics monitoring.MetricServer + cache cache.Cache + dedupe bool + linter common.Lint + commit bool + gc bool + gcReferrers bool + gcDelay time.Duration + retentionDelay time.Duration +} + +func (is *ImageStore) RootDir() string { + return is.rootDir +} + +func (is *ImageStore) DirExists(d string) bool { + return is.storeDriver.DirExists(d) +} + +// NewImageStore returns a new image store backed by cloud storages. +// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers +// Use the last argument to properly set a cache database, or it will default to boltDB local storage. +func NewImageStore(rootDir string, cacheDir string, gc bool, gcReferrers bool, gcDelay time.Duration, + untaggedImageRetentionDelay time.Duration, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, + linter common.Lint, storeDriver storageTypes.Driver, cacheDriver cache.Cache, +) storageTypes.ImageStore { + if err := storeDriver.EnsureDir(rootDir); err != nil { + log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") + + return nil + } + + imgStore := &ImageStore{ + rootDir: rootDir, + storeDriver: storeDriver, + lock: &sync.RWMutex{}, + log: log, + metrics: metrics, + dedupe: dedupe, + linter: linter, + commit: commit, + gc: gc, + gcReferrers: gcReferrers, + gcDelay: gcDelay, + retentionDelay: untaggedImageRetentionDelay, + cache: cacheDriver, + } + + return imgStore +} + +// RLock read-lock. +func (is *ImageStore) RLock(lockStart *time.Time) { + *lockStart = time.Now() + + is.lock.RLock() +} + +// RUnlock read-unlock. +func (is *ImageStore) RUnlock(lockStart *time.Time) { + is.lock.RUnlock() + + lockEnd := time.Now() + // includes time spent in acquiring and holding a lock + latency := lockEnd.Sub(*lockStart) + monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram +} + +// Lock write-lock. +func (is *ImageStore) Lock(lockStart *time.Time) { + *lockStart = time.Now() + + is.lock.Lock() +} + +// Unlock write-unlock. +func (is *ImageStore) Unlock(lockStart *time.Time) { + is.lock.Unlock() + + lockEnd := time.Now() + // includes time spent in acquiring and holding a lock + latency := lockEnd.Sub(*lockStart) + monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram +} + +func (is *ImageStore) initRepo(name string) error { + repoDir := path.Join(is.rootDir, name) + + if !utf8.ValidString(name) { + is.log.Error().Msg("input is not valid UTF-8") + + return zerr.ErrInvalidRepositoryName + } + + if !zreg.FullNameRegexp.MatchString(name) { + is.log.Error().Str("repository", name).Msg("invalid repository name") + + return zerr.ErrInvalidRepositoryName + } + + // create "blobs" subdir + err := is.storeDriver.EnsureDir(path.Join(repoDir, "blobs")) + if err != nil { + is.log.Error().Err(err).Msg("error creating blobs subdir") + + return err + } + // create BlobUploadDir subdir + err = is.storeDriver.EnsureDir(path.Join(repoDir, storageConstants.BlobUploadDir)) + if err != nil { + is.log.Error().Err(err).Msg("error creating blob upload subdir") + + return err + } + + // "oci-layout" file - create if it doesn't exist + ilPath := path.Join(repoDir, ispec.ImageLayoutFile) + if _, err := is.storeDriver.Stat(ilPath); err != nil { + il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} + + buf, err := json.Marshal(il) + if err != nil { + is.log.Error().Err(err).Msg("unable to marshal JSON") + + return err + } + + if _, err := is.storeDriver.WriteFile(ilPath, buf); err != nil { + is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") + + return err + } + } + + // "index.json" file - create if it doesn't exist + indexPath := path.Join(repoDir, "index.json") + if _, err := is.storeDriver.Stat(indexPath); err != nil { + index := ispec.Index{} + index.SchemaVersion = 2 + + buf, err := json.Marshal(index) + if err != nil { + is.log.Error().Err(err).Msg("unable to marshal JSON") + + return err + } + + if _, err := is.storeDriver.WriteFile(indexPath, buf); err != nil { + is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") + + return err + } + } + + return nil +} + +// InitRepo creates an image repository under this store. +func (is *ImageStore) InitRepo(name string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + return is.initRepo(name) +} + +// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. +func (is *ImageStore) ValidateRepo(name string) (bool, error) { + if !zreg.FullNameRegexp.MatchString(name) { + return false, zerr.ErrInvalidRepositoryName + } + + // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content + // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] + // and an additional/optional BlobUploadDir in each image store + // for s3 we can not create empty dirs, so we check only against index.json and oci-layout + dir := path.Join(is.rootDir, name) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return false, zerr.ErrRepoNotFound + } + + files, err := is.storeDriver.List(dir) + if err != nil { + is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") + + return false, zerr.ErrRepoNotFound + } + + //nolint:gomnd + if len(files) < 2 { + return false, zerr.ErrRepoBadVersion + } + + found := map[string]bool{ + ispec.ImageLayoutFile: false, + "index.json": false, + } + + for _, file := range files { + fileInfo, err := is.storeDriver.Stat(file) + if err != nil { + return false, err + } + + filename, err := filepath.Rel(dir, file) + if err != nil { + return false, err + } + + if filename == "blobs" && !fileInfo.IsDir() { + return false, nil + } + + found[filename] = true + } + + // check blobs dir exists only for filesystem, in s3 we can't have empty dirs + if is.storeDriver.Name() == storageConstants.LocalStorageDriverName { + if !is.storeDriver.DirExists(path.Join(dir, "blobs")) { + return false, nil + } + } + + for k, v := range found { + if !v && k != storageConstants.BlobUploadDir { + return false, nil + } + } + + buf, err := is.storeDriver.ReadFile(path.Join(dir, ispec.ImageLayoutFile)) + if err != nil { + return false, err + } + + var il ispec.ImageLayout + if err := json.Unmarshal(buf, &il); err != nil { + return false, err + } + + if il.Version != ispec.ImageLayoutVersion { + return false, zerr.ErrRepoBadVersion + } + + return true, nil +} + +// GetRepositories returns a list of all the repositories under this store. +func (is *ImageStore) GetRepositories() ([]string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + stores := make([]string, 0) + + err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + if !fileInfo.IsDir() { + return nil + } + + rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) + if err != nil { + return nil //nolint:nilerr // ignore paths that are not under root dir + } + + if ok, err := is.ValidateRepo(rel); !ok || err != nil { + return nil //nolint:nilerr // ignore invalid repos + } + + stores = append(stores, rel) + + return nil + }) + + // if the root directory is not yet created then return an empty slice of repositories + var perr driver.PathNotFoundError + if errors.As(err, &perr) { + return stores, nil + } + + return stores, err +} + +// GetNextRepository returns next repository under this store. +func (is *ImageStore) GetNextRepository(repo string) (string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + _, err := is.storeDriver.List(dir) + if err != nil { + is.log.Error().Err(err).Msg("failure walking storage root-dir") + + return "", err + } + + found := false + store := "" + err = is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + if !fileInfo.IsDir() { + return nil + } + + rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) + if err != nil { + return nil //nolint:nilerr // ignore paths not relative to root dir + } + + ok, err := is.ValidateRepo(rel) + if !ok || err != nil { + return nil //nolint:nilerr // ignore invalid repos + } + + if repo == "" && ok && err == nil { + store = rel + + return io.EOF + } + + if found { + store = rel + + return io.EOF + } + + if rel == repo { + found = true + } + + return nil + }) + + driverErr := &driver.Error{} + + if errors.Is(err, io.EOF) || + (errors.As(err, driverErr) && errors.Is(driverErr.Enclosed, io.EOF)) { + return store, nil + } + + return store, err +} + +// GetImageTags returns a list of image tags available in the specified repository. +func (is *ImageStore) GetImageTags(repo string) ([]string, error) { + var lockLatency time.Time + + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return nil, zerr.ErrRepoNotFound + } + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return nil, err + } + + return common.GetTagsByIndex(index), nil +} + +// GetImageManifest returns the image manifest of an image in the specific repository. +func (is *ImageStore) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return nil, "", "", zerr.ErrRepoNotFound + } + + var lockLatency time.Time + + var err error + + is.RLock(&lockLatency) + defer func() { + is.RUnlock(&lockLatency) + + if err == nil { + monitoring.IncDownloadCounter(is.metrics, repo) + } + }() + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return nil, "", "", err + } + + manifestDesc, found := common.GetManifestDescByReference(index, reference) + if !found { + return nil, "", "", zerr.ErrManifestNotFound + } + + buf, err := is.GetBlobContent(repo, manifestDesc.Digest) + if err != nil { + if errors.Is(err, zerr.ErrBlobNotFound) { + return nil, "", "", zerr.ErrManifestNotFound + } + + return nil, "", "", err + } + + var manifest ispec.Manifest + if err := json.Unmarshal(buf, &manifest); err != nil { + is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") + + return nil, "", "", err + } + + return buf, manifestDesc.Digest, manifestDesc.MediaType, nil +} + +// PutImageManifest adds an image manifest to the repository. +func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo + body []byte, +) (godigest.Digest, godigest.Digest, error) { + if err := is.InitRepo(repo); err != nil { + is.log.Debug().Err(err).Msg("init repo") + + return "", "", err + } + + var lockLatency time.Time + + var err error + + is.Lock(&lockLatency) + defer func() { + is.Unlock(&lockLatency) + + if err == nil { + monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) + monitoring.IncUploadCounter(is.metrics, repo) + } + }() + + refIsDigest := true + + mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) + if err != nil { + if errors.Is(err, zerr.ErrBadManifest) { + return mDigest, "", err + } + + refIsDigest = false + } + + dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) + if err != nil { + return dig, "", err + } + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return "", "", err + } + + // create a new descriptor + desc := ispec.Descriptor{ + MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, + } + + if !refIsDigest { + desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} + } + + var subjectDigest godigest.Digest + + artifactType := "" + + if mediaType == ispec.MediaTypeImageManifest { + var manifest ispec.Manifest + + err := json.Unmarshal(body, &manifest) + if err != nil { + return "", "", err + } + + if manifest.Subject != nil { + subjectDigest = manifest.Subject.Digest + } + + artifactType = zcommon.GetManifestArtifactType(manifest) + } else if mediaType == ispec.MediaTypeImageIndex { + var index ispec.Index + + err := json.Unmarshal(body, &index) + if err != nil { + return "", "", err + } + + if index.Subject != nil { + subjectDigest = index.Subject.Digest + } + + artifactType = zcommon.GetIndexArtifactType(index) + } + + updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) + if err != nil { + return "", "", err + } + + if !updateIndex { + return desc.Digest, subjectDigest, nil + } + + // write manifest to "blobs" + dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) + manifestPath := path.Join(dir, mDigest.Encoded()) + + if _, err = is.storeDriver.WriteFile(manifestPath, body); err != nil { + is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") + + return "", "", err + } + + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) + if err != nil { + return "", "", err + } + + // now update "index.json" + index.Manifests = append(index.Manifests, desc) + dir = path.Join(is.rootDir, repo) + indexPath := path.Join(dir, "index.json") + + buf, err := json.Marshal(index) + if err != nil { + is.log.Error().Err(err).Str("file", indexPath).Msg("unable to marshal JSON") + + return "", "", err + } + + // update the descriptors artifact type in order to check for signatures when applying the linter + desc.ArtifactType = artifactType + + // apply linter only on images, not signatures + pass, err := common.ApplyLinter(is, is.linter, repo, desc) + if !pass { + is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") + + return "", "", err + } + + if _, err = is.storeDriver.WriteFile(indexPath, buf); err != nil { + is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") + + return "", "", err + } + + return desc.Digest, subjectDigest, nil +} + +// DeleteImageManifest deletes the image manifest from the repository. +func (is *ImageStore) DeleteImageManifest(repo, reference string, detectCollisions bool) error { + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return zerr.ErrRepoNotFound + } + + var lockLatency time.Time + + var err error + + is.Lock(&lockLatency) + defer func() { + is.Unlock(&lockLatency) + + if err == nil { + monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) + } + }() + + err = is.deleteImageManifest(repo, reference, detectCollisions) + if err != nil { + return err + } + + return nil +} + +func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisions bool) error { + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return err + } + + manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollisions) + if err != nil { + return err + } + + /* check if manifest is referenced in image indexes, do not allow index images manipulations + (ie. remove manifest being part of an image index) */ + if manifestDesc.MediaType == ispec.MediaTypeImageManifest { + for _, mDesc := range index.Manifests { + if mDesc.MediaType == ispec.MediaTypeImageIndex { + if ok, _ := common.IsBlobReferencedInImageIndex(is, repo, manifestDesc.Digest, ispec.Index{ + Manifests: []ispec.Descriptor{mDesc}, + }, is.log); ok { + return zerr.ErrManifestReferenced + } + } + } + } + + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) + if err != nil { + return err + } + + // now update "index.json" + dir := path.Join(is.rootDir, repo) + file := path.Join(dir, "index.json") + + buf, err := json.Marshal(index) + if err != nil { + return err + } + + if _, err := is.storeDriver.WriteFile(file, buf); err != nil { + is.log.Debug().Str("deleting reference", reference).Msg("") + + return err + } + + // Delete blob only when blob digest not present in manifest entry. + // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. + toDelete := true + + for _, manifest := range index.Manifests { + if manifestDesc.Digest.String() == manifest.Digest.String() { + toDelete = false + + break + } + } + + if toDelete { + p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) + + err = is.storeDriver.Delete(p) + if err != nil { + return err + } + } + + return nil +} + +// BlobUploadPath returns the upload path for a blob in this store. +func (is *ImageStore) BlobUploadPath(repo, uuid string) string { + dir := path.Join(is.rootDir, repo) + blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) + + return blobUploadPath +} + +// NewBlobUpload returns the unique ID for an upload in progress. +func (is *ImageStore) NewBlobUpload(repo string) (string, error) { + if err := is.InitRepo(repo); err != nil { + is.log.Error().Err(err).Msg("error initializing repo") + + return "", err + } + + uuid, err := guuid.NewV4() + if err != nil { + return "", err + } + + uid := uuid.String() + + blobUploadPath := is.BlobUploadPath(repo, uid) + + // create multipart upload (append false) + writer, err := is.storeDriver.Writer(blobUploadPath, false) + if err != nil { + is.log.Debug().Err(err).Str("blob", blobUploadPath).Msg("failed to start multipart writer") + + return "", zerr.ErrRepoNotFound + } + + defer writer.Close() + + return uid, nil +} + +// GetBlobUpload returns the current size of a blob upload. +func (is *ImageStore) GetBlobUpload(repo, uuid string) (int64, error) { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + if !utf8.ValidString(blobUploadPath) { + is.log.Error().Msg("input is not valid UTF-8") + + return -1, zerr.ErrInvalidRepositoryName + } + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + return -1, err + } + + defer writer.Close() + + return writer.Size(), nil +} + +// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns +// the number of actual bytes to the blob. +func (is *ImageStore) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { + if err := is.InitRepo(repo); err != nil { + return -1, err + } + + blobUploadPath := is.BlobUploadPath(repo, uuid) + + file, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + is.log.Error().Err(err).Msg("failed to continue multipart upload") + + return -1, err + } + + var n int64 //nolint: varnamelen + + defer func() { + err = file.Close() + }() + + n, err = io.Copy(file, body) + + return n, err +} + +// PutBlobChunk writes another chunk of data to the specified blob. It returns +// the number of actual bytes to the blob. +func (is *ImageStore) PutBlobChunk(repo, uuid string, from, to int64, + body io.Reader, +) (int64, error) { + if err := is.InitRepo(repo); err != nil { + return -1, err + } + + blobUploadPath := is.BlobUploadPath(repo, uuid) + + file, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + is.log.Error().Err(err).Msg("failed to continue multipart upload") + + return -1, err + } + + defer file.Close() + + if from != file.Size() { + is.log.Error().Int64("expected", from).Int64("actual", file.Size()). + Msg("invalid range start for blob upload") + + return -1, zerr.ErrBadUploadRange + } + + n, err := io.Copy(file, body) + + return n, err +} + +// BlobUploadInfo returns the current blob size in bytes. +func (is *ImageStore) BlobUploadInfo(repo, uuid string) (int64, error) { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + return -1, err + } + + defer writer.Close() + + return writer.Size(), nil +} + +// FinishBlobUpload finalizes the blob upload and moves blob the repository. +func (is *ImageStore) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { + if err := dstDigest.Validate(); err != nil { + return err + } + + src := is.BlobUploadPath(repo, uuid) + + // complete multiUploadPart + fileWriter, err := is.storeDriver.Writer(src, true) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") + + return zerr.ErrUploadNotFound + } + + if err := fileWriter.Commit(); err != nil { + is.log.Error().Err(err).Msg("failed to commit file") + + return err + } + + if err := fileWriter.Close(); err != nil { + is.log.Error().Err(err).Msg("failed to close file") + + return err + } + + fileReader, err := is.storeDriver.Reader(src, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open file") + + return zerr.ErrUploadNotFound + } + + defer fileReader.Close() + + srcDigest, err := godigest.FromReader(fileReader) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") + + return zerr.ErrBadBlobDigest + } + + if srcDigest != dstDigest { + is.log.Error().Str("srcDigest", srcDigest.String()). + Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") + + return zerr.ErrBadBlobDigest + } + + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + + err = is.storeDriver.EnsureDir(dir) + if err != nil { + is.log.Error().Err(err).Msg("error creating blobs/sha256 dir") + + return err + } + + dst := is.BlobPath(repo, dstDigest) + + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + err = is.DedupeBlob(src, dstDigest, dst) + if err := inject.Error(err); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + + return err + } + } else { + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to finish blob") + + return err + } + } + + return nil +} + +// FullBlobUpload handles a full blob upload, and no partial session is created. +func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest) (string, int64, error) { + if err := dstDigest.Validate(); err != nil { + return "", -1, err + } + + if err := is.InitRepo(repo); err != nil { + return "", -1, err + } + + u, err := guuid.NewV4() + if err != nil { + return "", -1, err + } + + uuid := u.String() + src := is.BlobUploadPath(repo, uuid) + digester := sha256.New() + buf := new(bytes.Buffer) + + _, err = buf.ReadFrom(body) + if err != nil { + is.log.Error().Err(err).Msg("failed to read blob") + + return "", -1, err + } + + nbytes, err := is.storeDriver.WriteFile(src, buf.Bytes()) + if err != nil { + is.log.Error().Err(err).Msg("failed to write blob") + + return "", -1, err + } + + _, err = digester.Write(buf.Bytes()) + if err != nil { + is.log.Error().Err(err).Msg("digester failed to write") + + return "", -1, err + } + + srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) + if srcDigest != dstDigest { + is.log.Error().Str("srcDigest", srcDigest.String()). + Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") + + return "", -1, zerr.ErrBadBlobDigest + } + + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + _ = is.storeDriver.EnsureDir(dir) + + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + dst := is.BlobPath(repo, dstDigest) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + if err := is.DedupeBlob(src, dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + + return "", -1, err + } + } else { + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to finish blob") + + return "", -1, err + } + } + + return uuid, int64(nbytes), nil +} + +func (is *ImageStore) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { +retry: + is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") + + dstRecord, err := is.cache.GetBlob(dstDigest) + if err := inject.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") + + return err + } + + if dstRecord == "" { + // cache record doesn't exist, so first disk and cache entry for this digest + if err := is.cache.PutBlob(dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") + + return err + } + + // move the blob from uploads to final dest + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") + + return err + } + + is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") + } else { + // cache record exists, but due to GC and upgrades from older versions, + // disk content and cache records may go out of sync + + if is.cache.UsesRelativePaths() { + dstRecord = path.Join(is.rootDir, dstRecord) + } + + _, err := is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") + // the actual blob on disk may have been removed by GC, so sync the cache + err := is.cache.DeleteBlob(dstDigest, dstRecord) + if err = inject.Error(err); err != nil { + //nolint:lll + is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") + + return err + } + + goto retry + } + + // prevent overwrite original blob + if !is.storeDriver.SameFile(dst, dstRecord) { + if err := is.storeDriver.Link(dstRecord, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to link blobs") + + return err + } + + if err := is.cache.PutBlob(dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") + + return err + } + } + + // remove temp blobupload + if err := is.storeDriver.Delete(src); err != nil { + is.log.Error().Err(err).Str("src", src).Msg("dedupe: unable to remove blob") + + return err + } + + is.log.Debug().Str("src", src).Msg("dedupe: remove") + } + + return nil +} + +// DeleteBlobUpload deletes an existing blob upload that is currently in progress. +func (is *ImageStore) DeleteBlobUpload(repo, uuid string) error { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return zerr.ErrUploadNotFound + } + + return err + } + + defer writer.Close() + + if err := writer.Cancel(); err != nil { + is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") + + return err + } + + return nil +} + +// BlobPath returns the repository path of a blob. +func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string { + return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) +} + +/* + CheckBlob verifies a blob and returns true if the blob is correct + +If the blob is not found but it's found in cache then it will be copied over. +*/ +func (is *ImageStore) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return false, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + } else { + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + } + + binfo, err := is.storeDriver.Stat(blobPath) + if err == nil && binfo.Size() > 0 { + is.log.Debug().Str("blob path", blobPath).Msg("blob path found") + + return true, binfo.Size(), nil + } + // otherwise is a 'deduped' blob (empty file) + + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return false, -1, zerr.ErrBlobNotFound + } + + blobSize, err := is.copyBlob(repo, blobPath, dstRecord) + if err != nil { + return false, -1, zerr.ErrBlobNotFound + } + + // put deduped blob in cache + if err := is.cache.PutBlob(digest, blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") + + return false, -1, err + } + + return true, blobSize, nil +} + +// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. +func (is *ImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) { + if err := digest.Validate(); err != nil { + return false, -1, time.Time{}, err + } + + blobPath := is.BlobPath(repo, digest) + + binfo, err := is.storeDriver.Stat(blobPath) + if err == nil && binfo.Size() > 0 { + is.log.Debug().Str("blob path", blobPath).Msg("blob path found") + + return true, binfo.Size(), binfo.ModTime(), nil + } + + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + // then it's a 'deduped' blob + + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + binfo, err = is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + return true, binfo.Size(), binfo.ModTime(), nil +} + +func (is *ImageStore) checkCacheBlob(digest godigest.Digest) (string, error) { + if err := digest.Validate(); err != nil { + return "", err + } + + if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { + return "", zerr.ErrBlobNotFound + } + + dstRecord, err := is.cache.GetBlob(digest) + if err != nil { + return "", err + } + + if is.cache.UsesRelativePaths() { + dstRecord = path.Join(is.rootDir, dstRecord) + } + + if _, err := is.storeDriver.Stat(dstRecord); err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") + + // the actual blob on disk may have been removed by GC, so sync the cache + if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). + Msg("unable to remove blob path from cache") + + return "", err + } + + return "", zerr.ErrBlobNotFound + } + + is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") + + return dstRecord, nil +} + +func (is *ImageStore) copyBlob(repo string, blobPath, dstRecord string) (int64, error) { + if err := is.initRepo(repo); err != nil { + is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") + + return -1, err + } + + _ = is.storeDriver.EnsureDir(filepath.Dir(blobPath)) + + if err := is.storeDriver.Link(dstRecord, blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to hard link") + + return -1, zerr.ErrBlobNotFound + } + + // return original blob with content instead of the deduped one (blobPath) + binfo, err := is.storeDriver.Stat(dstRecord) + if err == nil { + return binfo.Size(), nil + } + + return -1, zerr.ErrBlobNotFound +} + +// GetBlobPartial returns a partial stream to read the blob. +// blob selector instead of directly downloading the blob. +func (is *ImageStore) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, +) (io.ReadCloser, int64, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return nil, -1, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + + // is a deduped blob + if binfo.Size() == 0 { + // Check blobs in cache + blobPath, err = is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + + binfo, err = is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + } + + end := to + + if to < 0 || to >= binfo.Size() { + end = binfo.Size() - 1 + } + + blobHandle, err := is.storeDriver.Reader(blobPath, from) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, -1, -1, err + } + + blobReadCloser, err := newBlobStream(blobHandle, from, end) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") + + return nil, -1, -1, err + } + + // The caller function is responsible for calling Close() + return blobReadCloser, end - from + 1, binfo.Size(), nil +} + +// GetBlob returns a stream to read the blob. +// blob selector instead of directly downloading the blob. +func (is *ImageStore) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return nil, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, zerr.ErrBlobNotFound + } + + blobReadCloser, err := is.storeDriver.Reader(blobPath, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, -1, err + } + + // is a 'deduped' blob? + if binfo.Size() == 0 { + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, -1, zerr.ErrBlobNotFound + } + + binfo, err := is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") + + return nil, -1, zerr.ErrBlobNotFound + } + + blobReadCloser, err := is.storeDriver.Reader(dstRecord, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") + + return nil, -1, err + } + + return blobReadCloser, binfo.Size(), nil + } + + // The caller function is responsible for calling Close() + return blobReadCloser, binfo.Size(), nil +} + +// GetBlobContent returns blob contents, the caller function SHOULD lock from outside. +func (is *ImageStore) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { + if err := digest.Validate(); err != nil { + return []byte{}, err + } + + blobPath := is.BlobPath(repo, digest) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return []byte{}, zerr.ErrBlobNotFound + } + + blobBuf, err := is.storeDriver.ReadFile(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, err + } + + // is a 'deduped' blob? + if binfo.Size() == 0 { + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, zerr.ErrBlobNotFound + } + + blobBuf, err := is.storeDriver.ReadFile(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") + + return nil, err + } + + return blobBuf, nil + } + + return blobBuf, nil +} + +func (is *ImageStore) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, +) (ispec.Index, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) +} + +func (is *ImageStore) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, +) ([]artifactspec.Descriptor, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) +} + +// GetIndexContent returns index.json contents, the caller function SHOULD lock from outside. +func (is *ImageStore) GetIndexContent(repo string) ([]byte, error) { + dir := path.Join(is.rootDir, repo) + + buf, err := is.storeDriver.ReadFile(path.Join(dir, "index.json")) + if err != nil { + if errors.Is(err, driver.PathNotFoundError{}) { + is.log.Error().Err(err).Str("dir", dir).Msg("index.json doesn't exist") + + return []byte{}, zerr.ErrRepoNotFound + } + + is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") + + return []byte{}, err + } + + return buf, nil +} + +// DeleteBlob removes the blob from the repository. +func (is *ImageStore) DeleteBlob(repo string, digest godigest.Digest) error { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return err + } + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + return is.deleteBlob(repo, digest) +} + +func (is *ImageStore) deleteBlob(repo string, digest godigest.Digest) error { + blobPath := is.BlobPath(repo, digest) + + _, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return zerr.ErrBlobNotFound + } + + // first check if this blob is not currently in use + if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { + return zerr.ErrBlobReferenced + } + + if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + dstRecord, err := is.cache.GetBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") + + return err + } + + // remove cache entry and move blob contents to the next candidate if there is any + if ok := is.cache.HasBlob(digest, blobPath); ok { + if err := is.cache.DeleteBlob(digest, blobPath); err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). + Msg("unable to remove blob path from cache") + + return err + } + } + + // if the deleted blob is one with content + if dstRecord == blobPath { + // get next candidate + dstRecord, err := is.cache.GetBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") + + return err + } + + // if we have a new candidate move the blob content to it + if dstRecord != "" { + /* check to see if we need to move the content from original blob to duplicate one + (in case of filesystem, this should not be needed */ + binfo, err := is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return err + } + + if binfo.Size() == 0 { + if err := is.storeDriver.Move(blobPath, dstRecord); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") + + return err + } + } + + return nil + } + } + } + + if err := is.storeDriver.Delete(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") + + return err + } + + return nil +} + +func (is *ImageStore) garbageCollect(repo string) error { + if is.gcReferrers { + is.log.Info().Msg("gc: manifests with missing referrers") + + // gc all manifests that have a missing subject, stop when no gc happened in a full loop over index.json. + stop := false + for !stop { + // because we gc manifests in the loop, need to get latest index.json content + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return err + } + + gced, err := is.garbageCollectIndexReferrers(repo, index, index) + if err != nil { + return err + } + + /* if we delete any manifest then loop again and gc manifests with + a subject pointing to the last ones which were gc'ed. */ + stop = !gced + } + } + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return err + } + + is.log.Info().Msg("gc: manifests without tags") + + // apply image retention policy + if err := is.garbageCollectUntaggedManifests(index, repo); err != nil { + return err + } + + is.log.Info().Msg("gc: blobs") + + if err := is.garbageCollectBlobs(is, repo, is.gcDelay, is.log); err != nil { + return err + } + + return nil +} + +/* +garbageCollectIndexReferrers will gc all referrers with a missing subject recursively + +rootIndex is indexJson, need to pass it down to garbageCollectReferrer() +rootIndex is the place we look for referrers. +*/ +func (is *ImageStore) garbageCollectIndexReferrers(repo string, rootIndex ispec.Index, index ispec.Index, +) (bool, error) { + var count int + + var err error + + for _, desc := range index.Manifests { + switch desc.MediaType { + case ispec.MediaTypeImageIndex: + indexImage, err := common.GetImageIndex(is, repo, desc.Digest, is.log) + if err != nil { + is.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read multiarch(index) image") + + return false, err + } + + gced, err := is.garbageCollectReferrer(repo, rootIndex, desc, indexImage.Subject) + if err != nil { + return false, err + } + + /* if we gc index then no need to continue searching for referrers inside it. + they will be gced when the next garbage collect is executed(if they are older than retentionDelay), + because manifests part of indexes will still be referenced in index.json */ + if gced { + return true, nil + } + + if gced, err = is.garbageCollectIndexReferrers(repo, rootIndex, indexImage); err != nil { + return false, err + } + + if gced { + count++ + } + + case ispec.MediaTypeImageManifest, artifactspec.MediaTypeArtifactManifest: + image, err := common.GetImageManifest(is, repo, desc.Digest, is.log) + if err != nil { + is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read manifest image") + + return false, err + } + + gced, err := is.garbageCollectReferrer(repo, rootIndex, desc, image.Subject) + if err != nil { + return false, err + } + + if gced { + count++ + } + } + } + + return count > 0, err +} + +func (is *ImageStore) garbageCollectReferrer(repo string, index ispec.Index, manifestDesc ispec.Descriptor, + subject *ispec.Descriptor, +) (bool, error) { + var gced bool + + var err error + + if subject != nil { + // try to find subject in index.json + if ok := isManifestReferencedInIndex(index, subject.Digest); !ok { + gced, err = garbageCollectManifest(is, repo, manifestDesc.Digest, is.gcDelay) + if err != nil { + return false, err + } + } + } + + tag, ok := manifestDesc.Annotations[ispec.AnnotationRefName] + if ok { + if strings.HasPrefix(tag, "sha256-") && (strings.HasSuffix(tag, cosignSignatureTagSuffix) || + strings.HasSuffix(tag, SBOMTagSuffix)) { + if ok := isManifestReferencedInIndex(index, getSubjectFromCosignTag(tag)); !ok { + gced, err = garbageCollectManifest(is, repo, manifestDesc.Digest, is.gcDelay) + if err != nil { + return false, err + } + } + } + } + + return gced, err +} + +func (is *ImageStore) garbageCollectUntaggedManifests(index ispec.Index, repo string) error { + referencedByImageIndex := make([]string, 0) + + if err := identifyManifestsReferencedInIndex(is, index, repo, &referencedByImageIndex); err != nil { + return err + } + + // first gather manifests part of image indexes and referrers, we want to skip checking them + for _, desc := range index.Manifests { + // skip manifests referenced in image indexes + if zcommon.Contains(referencedByImageIndex, desc.Digest.String()) { + continue + } + + // remove untagged images + if desc.MediaType == ispec.MediaTypeImageManifest || desc.MediaType == ispec.MediaTypeImageIndex { + _, ok := desc.Annotations[ispec.AnnotationRefName] + if !ok { + _, err := garbageCollectManifest(is, repo, desc.Digest, is.retentionDelay) + if err != nil { + return err + } + } + } + } + + return nil +} + +// Adds both referenced manifests and referrers from an index. +func identifyManifestsReferencedInIndex(imgStore *ImageStore, index ispec.Index, repo string, referenced *[]string, +) error { + for _, desc := range index.Manifests { + switch desc.MediaType { + case ispec.MediaTypeImageIndex: + indexImage, err := common.GetImageIndex(imgStore, repo, desc.Digest, imgStore.log) + if err != nil { + imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read multiarch(index) image") + + return err + } + + if indexImage.Subject != nil { + *referenced = append(*referenced, desc.Digest.String()) + } + + for _, indexDesc := range indexImage.Manifests { + *referenced = append(*referenced, indexDesc.Digest.String()) + } + + if err := identifyManifestsReferencedInIndex(imgStore, indexImage, repo, referenced); err != nil { + return err + } + case ispec.MediaTypeImageManifest, artifactspec.MediaTypeArtifactManifest: + image, err := common.GetImageManifest(imgStore, repo, desc.Digest, imgStore.log) + if err != nil { + imgStore.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + if image.Subject != nil { + *referenced = append(*referenced, desc.Digest.String()) + } + } + } + + return nil +} + +func garbageCollectManifest(imgStore *ImageStore, repo string, digest godigest.Digest, delay time.Duration, +) (bool, error) { + canGC, err := isBlobOlderThan(imgStore, repo, digest, delay, imgStore.log) + if err != nil { + imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", digest.String()). + Str("delay", imgStore.gcDelay.String()).Msg("gc: failed to check if blob is older than delay") + + return false, err + } + + if canGC { + imgStore.log.Info().Str("repository", repo).Str("digest", digest.String()). + Msg("gc: removing unreferenced manifest") + + if err := imgStore.deleteImageManifest(repo, digest.String(), true); err != nil { + if errors.Is(err, zerr.ErrManifestConflict) { + imgStore.log.Info().Str("repository", repo).Str("digest", digest.String()). + Msg("gc: skipping removing manifest due to conflict") + + return false, nil + } + + return false, err + } + + return true, nil + } + + return false, nil +} + +func (is *ImageStore) garbageCollectBlobs(imgStore *ImageStore, repo string, + delay time.Duration, log zlog.Logger, +) error { + refBlobs := map[string]bool{} + + err := common.AddRepoBlobsToReferences(imgStore, repo, refBlobs, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Msg("unable to get referenced blobs in repo") + + return err + } + + allBlobs, err := imgStore.GetAllBlobs(repo) + if err != nil { + // /blobs/sha256/ may be empty in the case of s3, no need to return err, we want to skip + if errors.As(err, &driver.PathNotFoundError{}) { + return nil + } + + log.Error().Err(err).Str("repository", repo).Msg("unable to get all blobs") + + return err + } + + reaped := 0 + + for _, blob := range allBlobs { + digest := godigest.NewDigestFromEncoded(godigest.SHA256, blob) + if err = digest.Validate(); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to parse digest") + + return err + } + + if _, ok := refBlobs[digest.String()]; !ok { + ok, err := isBlobOlderThan(imgStore, repo, digest, delay, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to determine GC delay") + + return err + } + + if !ok { + continue + } + + if err := imgStore.deleteBlob(repo, digest); err != nil { + if errors.Is(err, zerr.ErrBlobReferenced) { + if err := imgStore.deleteImageManifest(repo, digest.String(), true); err != nil { + if errors.Is(err, zerr.ErrManifestConflict) { + continue + } + + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to delete blob") + + return err + } + } else { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to delete blob") + + return err + } + } + + log.Info().Str("repository", repo).Str("digest", blob).Msg("garbage collected blob") + + reaped++ + } + } + + // if we cleaned all blobs let's also remove the repo so that it won't be returned by catalog + if reaped == len(allBlobs) { + log.Info().Str("repository", repo).Msg("garbage collected all blobs, cleaning repo...") + + if err := is.storeDriver.Delete(path.Join(is.rootDir, repo)); err != nil { + log.Error().Err(err).Str("repository", repo).Msg("unable to delete repo") + + return err + } + } + + log.Info().Str("repository", repo).Int("count", reaped).Msg("garbage collected blobs") + + return nil +} + +func (is *ImageStore) gcRepo(repo string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + err := is.garbageCollect(repo) + is.Unlock(&lockLatency) + + if err != nil { + return err + } + + return nil +} + +func (is *ImageStore) GetAllBlobs(repo string) ([]string, error) { + dir := path.Join(is.rootDir, repo, "blobs", "sha256") + + files, err := is.storeDriver.List(dir) + if err != nil { + return []string{}, err + } + + ret := []string{} + + for _, file := range files { + ret = append(ret, filepath.Base(file)) + } + + return ret, nil +} + +func (is *ImageStore) RunGCRepo(repo string) error { + is.log.Info().Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(is.RootDir(), repo))) + + if err := is.gcRepo(repo); err != nil { + errMessage := fmt.Sprintf("error while running GC for %s", path.Join(is.RootDir(), repo)) + is.log.Error().Err(err).Msg(errMessage) + is.log.Info().Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(is.RootDir(), repo))) + + return err + } + + is.log.Info().Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(is.RootDir(), repo))) + + return nil +} + +func (is *ImageStore) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { + generator := &common.GCTaskGenerator{ + ImgStore: is, + } + + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) +} + +func (is *ImageStore) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + var duplicateBlobs []string + + var digest godigest.Digest + + err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + // skip blobs under .sync + if strings.HasSuffix(fileInfo.Path(), syncConstants.SyncBlobUploadDir) { + return driver.ErrSkipDir + } + + if fileInfo.IsDir() { + return nil + } + + blobDigest := godigest.NewDigestFromEncoded("sha256", path.Base(fileInfo.Path())) + if err := blobDigest.Validate(); err != nil { //nolint: nilerr + return nil //nolint: nilerr // ignore files which are not blobs + } + + if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { + digest = blobDigest + } + + if blobDigest == digest { + duplicateBlobs = append(duplicateBlobs, fileInfo.Path()) + } + + return nil + }) + + // if the root directory is not yet created + var perr driver.PathNotFoundError + + if errors.As(err, &perr) { + return digest, duplicateBlobs, nil + } + + return digest, duplicateBlobs, err +} + +func (is *ImageStore) getOriginalBlobFromDisk(duplicateBlobs []string) (string, error) { + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return "", zerr.ErrBlobNotFound + } + + if binfo.Size() > 0 { + return blobPath, nil + } + } + + return "", zerr.ErrBlobNotFound +} + +func (is *ImageStore) getOriginalBlob(digest godigest.Digest, duplicateBlobs []string) (string, error) { + var originalBlob string + + var err error + + originalBlob, err = is.checkCacheBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrBlobNotFound) && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find blob in cache") + + return originalBlob, err + } + + // if we still don't have, search it + if originalBlob == "" { + is.log.Warn().Msg("rebuild dedupe: failed to find blob in cache, searching it in s3...") + // a rebuild dedupe was attempted in the past + // get original blob, should be found otherwise exit with error + + originalBlob, err = is.getOriginalBlobFromDisk(duplicateBlobs) + if err != nil { + return originalBlob, err + } + } + + is.log.Info().Str("originalBlob", originalBlob).Msg("rebuild dedupe: found original blob") + + return originalBlob, nil +} + +func (is *ImageStore) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { + if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { + is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") + + return zerr.ErrDedupeRebuild + } + + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") + + var originalBlob string + + // rebuild from dedupe false to true + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return err + } + + if binfo.Size() == 0 { + is.log.Warn().Msg("rebuild dedupe: found file without content, trying to find the original blob") + // a rebuild dedupe was attempted in the past + // get original blob, should be found otherwise exit with error + if originalBlob == "" { + originalBlob, err = is.getOriginalBlob(digest, duplicateBlobs) + if err != nil { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") + + return zerr.ErrDedupeRebuild + } + + // cache original blob + if ok := is.cache.HasBlob(digest, originalBlob); !ok { + if err := is.cache.PutBlob(digest, originalBlob); err != nil { + return err + } + } + } + + // cache dedupe blob + if ok := is.cache.HasBlob(digest, blobPath); !ok { + if err := is.cache.PutBlob(digest, blobPath); err != nil { + return err + } + } + } else { + // if we have an original blob cached then we can safely dedupe the rest of them + if originalBlob != "" { + if err := is.storeDriver.Link(originalBlob, blobPath); err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: unable to dedupe blob") + + return err + } + } + + // cache it + if ok := is.cache.HasBlob(digest, blobPath); !ok { + if err := is.cache.PutBlob(digest, blobPath); err != nil { + return err + } + } + + // mark blob as preserved + originalBlob = blobPath + } + } + + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") + + return nil +} + +func (is *ImageStore) restoreDedupedBlobs(digest godigest.Digest, duplicateBlobs []string) error { + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: restoring deduped blobs for digest") + + // first we need to find the original blob, either in cache or by checking each blob size + originalBlob, err := is.getOriginalBlob(digest, duplicateBlobs) + if err != nil { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") + + return zerr.ErrDedupeRebuild + } + + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return err + } + + // if we find a deduped blob, then copy original blob content to deduped one + if binfo.Size() == 0 { + // move content from original blob to deduped one + buf, err := is.storeDriver.ReadFile(originalBlob) + if err != nil { + is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to get original blob content") + + return err + } + + _, err = is.storeDriver.WriteFile(blobPath, buf) + if err != nil { + return err + } + } + } + + is.log.Info().Str("digest", digest.String()). + Msg("rebuild dedupe: restoring deduped blobs for digest finished successfully") + + return nil +} + +func (is *ImageStore) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + if dedupe { + return is.dedupeBlobs(digest, duplicateBlobs) + } + + return is.restoreDedupedBlobs(digest, duplicateBlobs) +} + +func (is *ImageStore) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { + generator := &common.DedupeTaskGenerator{ + ImgStore: is, + Dedupe: is.dedupe, + Log: is.log, + } + + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) +} + +type blobStream struct { + reader io.Reader + closer io.Closer +} + +func newBlobStream(readCloser io.ReadCloser, from, to int64) (io.ReadCloser, error) { + if from < 0 || to < from { + return nil, zerr.ErrBadRange + } + + return &blobStream{reader: io.LimitReader(readCloser, to-from+1), closer: readCloser}, nil +} + +func (bs *blobStream) Read(buf []byte) (int, error) { + return bs.reader.Read(buf) +} + +func (bs *blobStream) Close() error { + return bs.closer.Close() +} + +func isBlobOlderThan(imgStore storageTypes.ImageStore, repo string, + digest godigest.Digest, delay time.Duration, log zlog.Logger, +) (bool, error) { + _, _, modtime, err := imgStore.StatBlob(repo, digest) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", digest.String()). + Msg("gc: failed to stat blob") + + return false, err + } + + if modtime.Add(delay).After(time.Now()) { + return false, nil + } + + log.Info().Str("repository", repo).Str("digest", digest.String()).Msg("perform GC on blob") + + return true, nil +} + +func getSubjectFromCosignTag(tag string) godigest.Digest { + alg := strings.Split(tag, "-")[0] + encoded := strings.Split(strings.Split(tag, "-")[1], ".sig")[0] + + return godigest.NewDigestFromEncoded(godigest.Algorithm(alg), encoded) +} + +func isManifestReferencedInIndex(index ispec.Index, digest godigest.Digest) bool { + for _, manifest := range index.Manifests { + if manifest.Digest == digest { + return true + } + } + + return false +} diff --git a/pkg/storage/local/driver.go b/pkg/storage/local/driver.go new file mode 100644 index 0000000000..2e12ac9a48 --- /dev/null +++ b/pkg/storage/local/driver.go @@ -0,0 +1,481 @@ +package local + +import ( + "bufio" + "bytes" + "errors" + "io" + "io/fs" + "os" + "path" + "sort" + "syscall" + "time" + "unicode/utf8" + + storagedriver "github.com/docker/distribution/registry/storage/driver" + + zerr "zotregistry.io/zot/errors" + storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/test/inject" +) + +type Driver struct { + commit bool +} + +func New(commit bool) *Driver { + return &Driver{commit: commit} +} + +func (driver *Driver) Name() string { + return storageConstants.LocalStorageDriverName +} + +func (driver *Driver) EnsureDir(path string) error { + err := os.MkdirAll(path, storageConstants.DefaultDirPerms) + + return driver.formatErr(err) +} + +func (driver *Driver) DirExists(path string) bool { + if !utf8.ValidString(path) { + return false + } + + fileInfo, err := os.Stat(path) + if err != nil { + if e, ok := err.(*fs.PathError); ok && errors.Is(e.Err, syscall.ENAMETOOLONG) || //nolint: errorlint + errors.Is(e.Err, syscall.EINVAL) { + return false + } + } + + if err != nil && os.IsNotExist(err) { + return false + } + + if !fileInfo.IsDir() { + return false + } + + return true +} + +func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) { + file, err := os.OpenFile(path, os.O_RDONLY, storageConstants.DefaultFilePerms) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: path} + } + + return nil, driver.formatErr(err) + } + + seekPos, err := file.Seek(offset, io.SeekStart) + if err != nil { + file.Close() + + return nil, driver.formatErr(err) + } else if seekPos < offset { + file.Close() + + return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset} + } + + return file, nil +} + +func (driver *Driver) ReadFile(path string) ([]byte, error) { + reader, err := driver.Reader(path, 0) + if err != nil { + return nil, err + } + + defer reader.Close() + + buf, err := io.ReadAll(reader) + if err != nil { + return nil, driver.formatErr(err) + } + + return buf, nil +} + +func (driver *Driver) Delete(path string) error { + _, err := os.Stat(path) + if err != nil && !os.IsNotExist(err) { + return driver.formatErr(err) + } else if err != nil { + return storagedriver.PathNotFoundError{Path: path} + } + + return os.RemoveAll(path) +} + +func (driver *Driver) Stat(path string) (storagedriver.FileInfo, error) { + fi, err := os.Stat(path) //nolint: varnamelen + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: path} + } + + return nil, driver.formatErr(err) + } + + return fileInfo{ + path: path, + FileInfo: fi, + }, nil +} + +func (driver *Driver) Writer(filepath string, append bool) (storagedriver.FileWriter, error) { //nolint:predeclared + if append { + _, err := os.Stat(filepath) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: filepath} + } + + return nil, driver.formatErr(err) + } + } + + parentDir := path.Dir(filepath) + if err := os.MkdirAll(parentDir, storageConstants.DefaultDirPerms); err != nil { + return nil, driver.formatErr(err) + } + + file, err := os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) + if err != nil { + return nil, driver.formatErr(err) + } + + var offset int64 + + if !append { + err := file.Truncate(0) + if err != nil { + file.Close() + + return nil, driver.formatErr(err) + } + } else { + n, err := file.Seek(0, io.SeekEnd) //nolint: varnamelen + if err != nil { + file.Close() + + return nil, driver.formatErr(err) + } + offset = n + } + + return newFileWriter(file, offset, driver.commit), nil +} + +func (driver *Driver) WriteFile(filepath string, content []byte) (int, error) { + writer, err := driver.Writer(filepath, false) + if err != nil { + return -1, err + } + + nbytes, err := io.Copy(writer, bytes.NewReader(content)) + if err != nil { + _ = writer.Cancel() + + return -1, driver.formatErr(err) + } + + return int(nbytes), writer.Close() +} + +func (driver *Driver) Walk(path string, walkFn storagedriver.WalkFn) error { + children, err := driver.List(path) + if err != nil { + return err + } + + sort.Stable(sort.StringSlice(children)) + + for _, child := range children { + // Calling driver.Stat for every entry is quite + // expensive when running against backends with a slow Stat + // implementation, such as s3. This is very likely a serious + // performance bottleneck. + fileInfo, err := driver.Stat(child) + if err != nil { + switch errors.As(err, &storagedriver.PathNotFoundError{}) { + case true: + // repository was removed in between listing and enumeration. Ignore it. + continue + default: + return err + } + } + err = walkFn(fileInfo) + //nolint: gocritic + if err == nil && fileInfo.IsDir() { + if err := driver.Walk(child, walkFn); err != nil { + return err + } + } else if errors.Is(err, storagedriver.ErrSkipDir) { + // Stop iteration if it's a file, otherwise noop if it's a directory + if !fileInfo.IsDir() { + return nil + } + } else if err != nil { + return driver.formatErr(err) + } + } + + return nil +} + +func (driver *Driver) List(fullpath string) ([]string, error) { + dir, err := os.Open(fullpath) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: fullpath} + } + + return nil, driver.formatErr(err) + } + + defer dir.Close() + + fileNames, err := dir.Readdirnames(0) + if err != nil { + return nil, driver.formatErr(err) + } + + keys := make([]string, 0, len(fileNames)) + for _, fileName := range fileNames { + keys = append(keys, path.Join(fullpath, fileName)) + } + + return keys, nil +} + +func (driver *Driver) Move(sourcePath string, destPath string) error { + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + return storagedriver.PathNotFoundError{Path: sourcePath} + } + + if err := os.MkdirAll(path.Dir(destPath), storageConstants.DefaultDirPerms); err != nil { + return driver.formatErr(err) + } + + return driver.formatErr(os.Rename(sourcePath, destPath)) +} + +func (driver *Driver) SameFile(path1, path2 string) bool { + file1, err := os.Stat(path1) + if err != nil { + return false + } + + file2, err := os.Stat(path2) + if err != nil { + return false + } + + return os.SameFile(file1, file2) +} + +func (driver *Driver) Link(src, dest string) error { + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return err + } + + return driver.formatErr(os.Link(src, dest)) +} + +func (driver *Driver) formatErr(err error) error { + switch actual := err.(type) { //nolint: errorlint + case nil: + return nil + case storagedriver.PathNotFoundError: + actual.DriverName = driver.Name() + + return actual + case storagedriver.InvalidPathError: + actual.DriverName = driver.Name() + + return actual + case storagedriver.InvalidOffsetError: + actual.DriverName = driver.Name() + + return actual + default: + storageError := storagedriver.Error{ + DriverName: driver.Name(), + Enclosed: err, + } + + return storageError + } +} + +type fileInfo struct { + os.FileInfo + path string +} + +// asserts fileInfo implements storagedriver.FileInfo. +var _ storagedriver.FileInfo = fileInfo{} + +// Path provides the full path of the target of this file info. +func (fi fileInfo) Path() string { + return fi.path +} + +// Size returns current length in bytes of the file. The return value can +// be used to write to the end of the file at path. The value is +// meaningless if IsDir returns true. +func (fi fileInfo) Size() int64 { + if fi.IsDir() { + return 0 + } + + return fi.FileInfo.Size() +} + +// ModTime returns the modification time for the file. For backends that +// don't have a modification time, the creation time should be returned. +func (fi fileInfo) ModTime() time.Time { + return fi.FileInfo.ModTime() +} + +// IsDir returns true if the path is a directory. +func (fi fileInfo) IsDir() bool { + return fi.FileInfo.IsDir() +} + +type fileWriter struct { + file *os.File + size int64 + bw *bufio.Writer + closed bool + committed bool + cancelled bool + commit bool +} + +func newFileWriter(file *os.File, size int64, commit bool) *fileWriter { + return &fileWriter{ + file: file, + size: size, + commit: commit, + bw: bufio.NewWriter(file), + } +} + +func (fw *fileWriter) Write(buf []byte) (int, error) { + //nolint: gocritic + if fw.closed { + return 0, zerr.ErrFileAlreadyClosed + } else if fw.committed { + return 0, zerr.ErrFileAlreadyCommitted + } else if fw.cancelled { + return 0, zerr.ErrFileAlreadyCancelled + } + + n, err := fw.bw.Write(buf) + fw.size += int64(n) + + return n, err +} + +func (fw *fileWriter) Size() int64 { + return fw.size +} + +func (fw *fileWriter) Close() error { + if fw.closed { + return zerr.ErrFileAlreadyClosed + } + + if err := fw.bw.Flush(); err != nil { + return err + } + + if fw.commit { + if err := inject.Error(fw.file.Sync()); err != nil { + return err + } + } + + if err := inject.Error(fw.file.Close()); err != nil { + return err + } + + fw.closed = true + + return nil +} + +func (fw *fileWriter) Cancel() error { + if fw.closed { + return zerr.ErrFileAlreadyClosed + } + + fw.cancelled = true + fw.file.Close() + + return os.Remove(fw.file.Name()) +} + +func (fw *fileWriter) Commit() error { + //nolint: gocritic + if fw.closed { + return zerr.ErrFileAlreadyClosed + } else if fw.committed { + return zerr.ErrFileAlreadyCommitted + } else if fw.cancelled { + return zerr.ErrFileAlreadyCancelled + } + + if err := fw.bw.Flush(); err != nil { + return err + } + + if fw.commit { + if err := fw.file.Sync(); err != nil { + return err + } + } + + fw.committed = true + + return nil +} + +func ValidateHardLink(rootDir string) error { + if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { + return err + } + + err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), + []byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms) + if err != nil { + return err + } + + err = os.Link(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt")) + if err != nil { + // Remove hardlinkcheck.txt if hardlink fails + zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) + if zerr != nil { + return zerr + } + + return err + } + + err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) + if err != nil { + return err + } + + return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt")) +} diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index 6f8abde6b6..0492b727d1 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -1,1974 +1,35 @@ package local import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - "sync" "time" - "unicode/utf8" - apexlog "github.com/apex/log" - guuid "github.com/gofrs/uuid" - "github.com/minio/sha256-simd" - notreg "github.com/notaryproject/notation-go/registry" - godigest "github.com/opencontainers/go-digest" - imeta "github.com/opencontainers/image-spec/specs-go" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/opencontainers/umoci" - "github.com/opencontainers/umoci/oci/casext" - oras "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" - - zerr "zotregistry.io/zot/errors" - zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" - zreg "zotregistry.io/zot/pkg/regexp" - "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage/cache" common "zotregistry.io/zot/pkg/storage/common" - storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" storageTypes "zotregistry.io/zot/pkg/storage/types" - "zotregistry.io/zot/pkg/test/inject" -) - -const ( - cosignSignatureTagSuffix = "sig" - SBOMTagSuffix = "sbom" ) -// ImageStoreLocal provides the image storage operations. -type ImageStoreLocal struct { - rootDir string - lock *sync.RWMutex - cache cache.Cache - gc bool - dedupe bool - commit bool - gcDelay time.Duration - log zerolog.Logger - metrics monitoring.MetricServer - linter common.Lint -} - -func (is *ImageStoreLocal) RootDir() string { - return is.rootDir -} - -func (is *ImageStoreLocal) DirExists(d string) bool { - return zcommon.DirExists(d) -} - // NewImageStore returns a new image store backed by a file storage. // Use the last argument to properly set a cache database, or it will default to boltDB local storage. -func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, +func NewImageStore(rootDir string, gc bool, gcReferrers bool, gcDelay time.Duration, + untaggedImageRetentionDelay time.Duration, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache, ) storageTypes.ImageStore { - if _, err := os.Stat(rootDir); os.IsNotExist(err) { - if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { - log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") - - return nil - } - } - - imgStore := &ImageStoreLocal{ - rootDir: rootDir, - lock: &sync.RWMutex{}, - gc: gc, - gcDelay: gcDelay, - dedupe: dedupe, - commit: commit, - log: log.With().Caller().Logger(), - metrics: metrics, - linter: linter, - } - - imgStore.cache = cacheDriver - - if gc { - // we use umoci GC to perform garbage-collection, but it uses its own logger - // - so capture those logs, could be useful - apexlog.SetLevel(apexlog.DebugLevel) - apexlog.SetHandler(apexlog.HandlerFunc(func(entry *apexlog.Entry) error { - e := log.Debug() - for k, v := range entry.Fields { - e = e.Interface(k, v) - } - e.Msg(entry.Message) - - return nil - })) - } - - return imgStore -} - -// RLock read-lock. -func (is *ImageStoreLocal) RLock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.RLock() -} - -// RUnlock read-unlock. -func (is *ImageStoreLocal) RUnlock(lockStart *time.Time) { - is.lock.RUnlock() - - lockEnd := time.Now() - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram -} - -// Lock write-lock. -func (is *ImageStoreLocal) Lock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.Lock() -} - -// Unlock write-unlock. -func (is *ImageStoreLocal) Unlock(lockStart *time.Time) { - is.lock.Unlock() - - lockEnd := time.Now() - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram -} - -func (is *ImageStoreLocal) initRepo(name string) error { - repoDir := path.Join(is.rootDir, name) - - if !utf8.ValidString(name) { - is.log.Error().Msg("input is not valid UTF-8") - - return zerr.ErrInvalidRepositoryName - } - - if !zreg.FullNameRegexp.MatchString(name) { - is.log.Error().Str("repository", name).Msg("invalid repository name") - - return zerr.ErrInvalidRepositoryName - } - - // create "blobs" subdir - err := ensureDir(path.Join(repoDir, "blobs"), is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blobs subdir") - - return err - } - // create BlobUploadDir subdir - err = ensureDir(path.Join(repoDir, storageConstants.BlobUploadDir), is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blob upload subdir") - - return err - } - - // "oci-layout" file - create if it doesn't exist - ilPath := path.Join(repoDir, ispec.ImageLayoutFile) - if _, err := os.Stat(ilPath); err != nil { - il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} - - buf, err := json.Marshal(il) - if err != nil { - is.log.Panic().Err(err).Msg("unable to marshal JSON") - } - - if err := is.writeFile(ilPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - // "index.json" file - create if it doesn't exist - indexPath := path.Join(repoDir, "index.json") - if _, err := os.Stat(indexPath); err != nil { - index := ispec.Index{Versioned: imeta.Versioned{SchemaVersion: storageConstants.SchemaVersion}} - - buf, err := json.Marshal(index) - if err != nil { - is.log.Panic().Err(err).Msg("unable to marshal JSON") - } - - if err := is.writeFile(indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", indexPath).Msg("unable to write file") - - return err - } - } - - return nil -} - -// InitRepo creates an image repository under this store. -func (is *ImageStoreLocal) InitRepo(name string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - return is.initRepo(name) -} - -// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. -func (is *ImageStoreLocal) ValidateRepo(name string) (bool, error) { - // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content - // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] - // and an additional/optional BlobUploadDir in each image store - if !zreg.FullNameRegexp.MatchString(name) { - return false, zerr.ErrInvalidRepositoryName - } - - dir := path.Join(is.rootDir, name) - if !is.DirExists(dir) { - return false, zerr.ErrRepoNotFound - } - - files, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") - - return false, zerr.ErrRepoNotFound - } - - if len(files) < 3 { //nolint:gomnd - return false, zerr.ErrRepoBadVersion - } - - found := map[string]bool{ - "blobs": false, - ispec.ImageLayoutFile: false, - "index.json": false, - } - - for _, file := range files { - if file.Name() == "blobs" && !file.IsDir() { - return false, nil - } - - found[file.Name()] = true - } - - for k, v := range found { - if !v && k != storageConstants.BlobUploadDir { - return false, nil - } - } - - buf, err := os.ReadFile(path.Join(dir, ispec.ImageLayoutFile)) - if err != nil { - return false, err - } - - var il ispec.ImageLayout - if err := json.Unmarshal(buf, &il); err != nil { - return false, err - } - - if il.Version != ispec.ImageLayoutVersion { - return false, zerr.ErrRepoBadVersion - } - - return true, nil -} - -// GetRepositories returns a list of all the repositories under this store. -func (is *ImageStoreLocal) GetRepositories() ([]string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - _, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Msg("failure walking storage root-dir") - - return nil, err - } - - stores := make([]string, 0) - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, path) - if err != nil { - return nil //nolint:nilerr // ignore paths not relative to root dir - } - - if ok, err := is.ValidateRepo(rel); !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - // is.log.Debug().Str("dir", path).Str("name", info.Name()).Msg("found image store") - stores = append(stores, rel) - - return nil - }) - - return stores, err -} - -// GetNextRepository returns next repository under this store. -func (is *ImageStoreLocal) GetNextRepository(repo string) (string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - _, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Msg("failure walking storage root-dir") - - return "", err - } - - found := false - store := "" - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, path) - if err != nil { - return nil //nolint:nilerr // ignore paths not relative to root dir - } - - ok, err := is.ValidateRepo(rel) - if !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - if repo == "" && ok && err == nil { - store = rel - - return io.EOF - } - - if found { - store = rel - - return io.EOF - } - - if rel == repo { - found = true - } - - return nil - }) - - return store, err -} - -// GetImageTags returns a list of image tags available in the specified repository. -func (is *ImageStoreLocal) GetImageTags(repo string) ([]string, error) { - var lockLatency time.Time - - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return nil, zerr.ErrRepoNotFound - } - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, err - } - - return common.GetTagsByIndex(index), nil -} - -// GetImageManifest returns the image manifest of an image in the specific repository. -func (is *ImageStoreLocal) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return nil, "", "", zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.RLock(&lockLatency) - defer func() { - is.RUnlock(&lockLatency) - - if err == nil { - monitoring.IncDownloadCounter(is.metrics, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, "", "", err - } - - manifestDesc, found := common.GetManifestDescByReference(index, reference) - if !found { - return nil, "", "", zerr.ErrManifestNotFound - } - - buf, err := is.GetBlobContent(repo, manifestDesc.Digest) - if err != nil { - if errors.Is(err, zerr.ErrBlobNotFound) { - return nil, "", "", zerr.ErrManifestNotFound - } - - return nil, "", "", err - } - - var manifest ispec.Manifest - if err := json.Unmarshal(buf, &manifest); err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") - - return nil, "", "", err - } - - return buf, manifestDesc.Digest, manifestDesc.MediaType, nil -} - -// PutImageManifest adds an image manifest to the repository. -func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo - body []byte, -) (godigest.Digest, godigest.Digest, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Debug().Err(err).Msg("init repo") - - return "", "", err - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - monitoring.IncUploadCounter(is.metrics, repo) - } - }() - - refIsDigest := true - - mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) - if err != nil { - if errors.Is(err, zerr.ErrBadManifest) { - return mDigest, "", err - } - - refIsDigest = false - } - - digest, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) - if err != nil { - return digest, "", err - } - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return "", "", err - } - - // create a new descriptor - desc := ispec.Descriptor{ - MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, - } - - if !refIsDigest { - desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} - } - - var subjectDigest godigest.Digest - - artifactType := "" - - if mediaType == ispec.MediaTypeImageManifest { - var manifest ispec.Manifest - - err := json.Unmarshal(body, &manifest) - if err != nil { - return "", "", err - } - - if manifest.Subject != nil { - subjectDigest = manifest.Subject.Digest - } - - artifactType = zcommon.GetManifestArtifactType(manifest) - } else if mediaType == ispec.MediaTypeImageIndex { - var index ispec.Index - - err := json.Unmarshal(body, &index) - if err != nil { - return "", "", err - } - - if index.Subject != nil { - subjectDigest = index.Subject.Digest - } - - artifactType = zcommon.GetIndexArtifactType(index) - } - - updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) - if err != nil { - return "", "", err - } - - if !updateIndex { - return desc.Digest, subjectDigest, nil - } - - // write manifest to "blobs" - dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) - _ = ensureDir(dir, is.log) - file := path.Join(dir, mDigest.Encoded()) - - // in case the linter will not pass, it will be garbage collected - if err := is.writeFile(file, body); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to write") - - return "", "", err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) - if err != nil { - return "", "", err - } - - // now update "index.json" - index.Manifests = append(index.Manifests, desc) - dir = path.Join(is.rootDir, repo) - file = path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to marshal JSON") - - return "", "", err - } - - // update the descriptors artifact type in order to check for signatures when applying the linter - desc.ArtifactType = artifactType - - // apply linter only on images, not signatures or indexes - pass, err := common.ApplyLinter(is, is.linter, repo, desc) - if !pass { - is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") - - return "", "", err - } - - err = is.writeFile(file, buf) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to write") - - return "", "", err - } - - return desc.Digest, subjectDigest, nil -} - -// DeleteImageManifest deletes the image manifest from the repository. -func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string, detectCollision bool) error { - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return err - } - - manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollision) - if err != nil { - return err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) - if err != nil { - return err - } - - // now update "index.json" - dir = path.Join(is.rootDir, repo) - file := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - return err - } - - if err := is.writeFile(file, buf); err != nil { - return err - } - - // Delete blob only when blob digest not present in manifest entry. - // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. - toDelete := true - - for _, manifest := range index.Manifests { - if manifestDesc.Digest.String() == manifest.Digest.String() { - toDelete = false - - break - } - } - - if toDelete { - p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) - - _ = os.Remove(p) - } - - return nil -} - -// BlobUploadPath returns the upload path for a blob in this store. -func (is *ImageStoreLocal) BlobUploadPath(repo, uuid string) string { - dir := path.Join(is.rootDir, repo) - blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) - - return blobUploadPath -} - -// NewBlobUpload returns the unique ID for an upload in progress. -func (is *ImageStoreLocal) NewBlobUpload(repo string) (string, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Error().Err(err).Msg("error initializing repo") - - return "", err - } - - uuid, err := guuid.NewV4() - if err != nil { - return "", err - } - - uid := uuid.String() - - blobUploadPath := is.BlobUploadPath(repo, uid) - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - return "", zerr.ErrRepoNotFound - } - - defer file.Close() - - return uid, nil -} - -// GetBlobUpload returns the current size of a blob upload. -func (is *ImageStoreLocal) GetBlobUpload(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - if !utf8.ValidString(blobUploadPath) { - is.log.Error().Msg("input is not valid UTF-8") - - return -1, zerr.ErrInvalidRepositoryName - } - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - if os.IsNotExist(err) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return binfo.Size(), nil -} - -// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ImageStoreLocal) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - _, err := os.Stat(blobUploadPath) - if err != nil { - return -1, zerr.ErrUploadNotFound - } - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - is.log.Error().Err(err).Msg("failed to open file") - - return -1, err - } - - defer func() { - if is.commit { - _ = file.Sync() - } - - _ = file.Close() - }() - - if _, err := file.Seek(0, io.SeekEnd); err != nil { - is.log.Error().Err(err).Msg("failed to seek file") - - return -1, err - } - - n, err := io.Copy(file, body) - - return n, err -} - -// PutBlobChunk writes another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ImageStoreLocal) PutBlobChunk(repo, uuid string, from, to int64, - body io.Reader, -) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - return -1, zerr.ErrUploadNotFound - } - - if from != binfo.Size() { - is.log.Error().Int64("expected", from).Int64("actual", binfo.Size()). - Msg("invalid range start for blob upload") - - return -1, zerr.ErrBadUploadRange - } - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - is.log.Error().Err(err).Msg("failed to open file") - - return -1, err - } - - defer func() { - if is.commit { - _ = file.Sync() - } - - _ = file.Close() - }() - - if _, err := file.Seek(from, io.SeekStart); err != nil { - is.log.Error().Err(err).Msg("failed to seek file") - - return -1, err - } - - n, err := io.Copy(file, body) - - return n, err -} - -// BlobUploadInfo returns the current blob size in bytes. -func (is *ImageStoreLocal) BlobUploadInfo(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobUploadPath).Msg("failed to stat blob") - - return -1, err - } - - size := binfo.Size() - - return size, nil -} - -// FinishBlobUpload finalizes the blob upload and moves blob the repository. -func (is *ImageStoreLocal) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { - if err := dstDigest.Validate(); err != nil { - return err - } - - src := is.BlobUploadPath(repo, uuid) - - _, err := os.Stat(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to stat blob") - - return zerr.ErrUploadNotFound - } - - blobFile, err := os.Open(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrUploadNotFound - } - - defer blobFile.Close() - - digester := sha256.New() - - _, err = io.Copy(digester, blobFile) - if err != nil { - is.log.Error().Err(err).Str("repository", repo).Str("blob", src).Str("digest", dstDigest.String()). - Msg("unable to compute hash") - - return err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return zerr.ErrBadBlobDigest - } - - dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - err = ensureDir(dir, is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blobs/sha256 dir") - - return err - } - - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - err = is.DedupeBlob(src, dstDigest, dst) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return err - } - } else { - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return err - } - } - - return nil -} - -// FullBlobUpload handles a full blob upload, and no partial session is created. -func (is *ImageStoreLocal) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest, -) (string, int64, error) { - if err := dstDigest.Validate(); err != nil { - return "", -1, err - } - - if err := is.InitRepo(repo); err != nil { - return "", -1, err - } - - u, err := guuid.NewV4() - if err != nil { - return "", -1, err - } - - uuid := u.String() - - src := is.BlobUploadPath(repo, uuid) - - blobFile, err := os.Create(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return "", -1, zerr.ErrUploadNotFound - } - - defer func() { - if is.commit { - _ = blobFile.Sync() - } - - _ = blobFile.Close() - }() - - digester := sha256.New() - mw := io.MultiWriter(blobFile, digester) - - nbytes, err := io.Copy(mw, body) - if err != nil { - return "", -1, err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return "", -1, zerr.ErrBadBlobDigest - } - - dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _ = ensureDir(dir, is.log) - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return "", -1, err - } - } else { - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return "", -1, err - } - } - - return uuid, nbytes, nil -} - -func (is *ImageStoreLocal) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { -retry: - is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") - - dstRecord, err := is.cache.GetBlob(dstDigest) - - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") - - return err - } - - if dstRecord == "" { - // cache record doesn't exist, so first disk and cache entry for this diges - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - // move the blob from uploads to final dest - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") - - return err - } - - is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") - } else { - // cache record exists, but due to GC and upgrades from older versions, - // disk content and cache records may go out of sync - dstRecord = path.Join(is.rootDir, dstRecord) - - dstRecordFi, err := os.Stat(dstRecord) - if err != nil { - is.log.Warn().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat cache record, removing it") - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(dstDigest, dstRecord); err != nil { - //nolint:lll // gofumpt conflicts with lll - is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") - - return err - } - - goto retry - } - - dstFi, err := os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - - return err - } - - if !os.SameFile(dstFi, dstRecordFi) { - // blob lookup cache out of sync with actual disk contents - if err := os.Remove(dst); err != nil && !os.IsNotExist(err) { - is.log.Error().Err(err).Str("dst", dst).Msg("dedupe: unable to remove blob") - - return err - } - - is.log.Debug().Str("blobPath", dst).Str("dstRecord", dstRecord).Msg("dedupe: creating hard link") - - if err := os.Link(dstRecord, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("dedupe: unable to hard link") - - return err - } - } - - // also put dedupe blob in cache - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - if err := os.Remove(src); err != nil { - is.log.Error().Err(err).Str("src", src).Msg("dedupe: uname to remove blob") - - return err - } - - is.log.Debug().Str("src", src).Msg("dedupe: remove") - } - - return nil -} - -// DeleteBlobUpload deletes an existing blob upload that is currently in progress. -func (is *ImageStoreLocal) DeleteBlobUpload(repo, uuid string) error { - blobUploadPath := is.BlobUploadPath(repo, uuid) - if err := os.Remove(blobUploadPath); err != nil { - is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") - - return err - } - - return nil -} - -// BlobPath returns the repository path of a blob. -func (is *ImageStoreLocal) BlobPath(repo string, digest godigest.Digest) string { - return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) -} - -/* - CheckBlob verifies a blob and returns true if the blob is correct - -If the blob is not found but it's found in cache then it will be copied over. -*/ -func (is *ImageStoreLocal) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return false, -1, err - } - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - } else { - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - } - - if ok, size, err := is.StatBlob(repo, digest); err == nil || ok { - return true, size, nil - } - - blobPath := is.BlobPath(repo, digest) - - is.log.Debug().Str("blob", blobPath).Msg("failed to find blob, searching it in cache") - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - // If found copy to location - blobSize, err := is.copyBlob(repo, blobPath, dstRecord) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - if err := is.cache.PutBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") - - return false, -1, err - } - - return true, blobSize, nil -} - -// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. -func (is *ImageStoreLocal) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Str("blob path", blobPath).Msg("failed to find blob") - - return false, -1, zerr.ErrBlobNotFound - } - - return true, binfo.Size(), nil -} - -func (is *ImageStoreLocal) checkCacheBlob(digest godigest.Digest) (string, error) { - if err := digest.Validate(); err != nil { - return "", err - } - - if !is.dedupe || fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - return "", zerr.ErrBlobNotFound - } - - dstRecord, err := is.cache.GetBlob(digest) - if err != nil { - if errors.Is(err, zerr.ErrCacheMiss) { - is.log.Debug().Err(err).Str("digest", string(digest)).Msg("unable to find blob in cache") - } else { - is.log.Error().Err(err).Str("digest", string(digest)).Msg("unable to search blob in cache") - } - - return "", err - } - - dstRecord = path.Join(is.rootDir, dstRecord) - - if _, err := os.Stat(dstRecord); err != nil { - is.log.Warn().Err(err).Str("blob", dstRecord).Msg("unable to stat cache record, removing it") - - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). - Msg("unable to remove blob path from cache") - - return "", err - } - - return "", zerr.ErrBlobNotFound - } - - is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") - - return dstRecord, nil -} - -func (is *ImageStoreLocal) copyBlob(repo, blobPath, dstRecord string) (int64, error) { - if err := is.initRepo(repo); err != nil { - is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") - - return -1, err - } - - _ = ensureDir(filepath.Dir(blobPath), is.log) - - if err := os.Link(dstRecord, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to hard link") - - return -1, zerr.ErrBlobNotFound - } - - binfo, err := os.Stat(blobPath) - if err == nil { - return binfo.Size(), nil - } - - return -1, zerr.ErrBlobNotFound -} - -// blobStream is using to serve blob range requests. -type blobStream struct { - reader io.Reader - closer io.Closer -} - -func newBlobStream(blobPath string, from, to int64) (io.ReadCloser, error) { - blobFile, err := os.Open(blobPath) - if err != nil { - return nil, err - } - - if from > 0 { - _, err = blobFile.Seek(from, io.SeekStart) - if err != nil { - return nil, err - } - } - - if from < 0 || to < from { - return nil, zerr.ErrBadRange - } - - blobstrm := blobStream{reader: blobFile, closer: blobFile} - - blobstrm.reader = io.LimitReader(blobFile, to-from+1) - - return &blobstrm, nil -} - -func (bs *blobStream) Read(buf []byte) (int, error) { - return bs.reader.Read(buf) -} - -func (bs *blobStream) Close() error { - return bs.closer.Close() -} - -// GetBlobPartial returns a partial stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ImageStoreLocal) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, -) (io.ReadCloser, int64, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - if to < 0 || to >= binfo.Size() { - to = binfo.Size() - 1 - } - - blobReadCloser, err := newBlobStream(blobPath, from, to) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, -1, err - } - - // The caller function is responsible for calling Close() - return blobReadCloser, to - from + 1, binfo.Size(), nil -} - -// GetBlob returns a stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ImageStoreLocal) GetBlob(repo string, digest godigest.Digest, mediaType string, -) (io.ReadCloser, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := os.Open(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, err - } - - // The caller function is responsible for calling Close() - return blobReadCloser, binfo.Size(), nil -} - -// GetBlobContent returns blob contents, SHOULD lock from outside. -func (is *ImageStoreLocal) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { - if err := digest.Validate(); err != nil { - return []byte{}, err - } - - blobPath := is.BlobPath(repo, digest) - - blob, err := os.ReadFile(blobPath) - if err != nil { - if os.IsNotExist(err) { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("blob doesn't exist") - - return []byte{}, zerr.ErrBlobNotFound - } - - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to read blob") - - return []byte{}, err - } - - return blob, nil -} - -// GetIndexContent returns index.json contents, SHOULD lock from outside. -func (is *ImageStoreLocal) GetIndexContent(repo string) ([]byte, error) { - dir := path.Join(is.rootDir, repo) - - buf, err := os.ReadFile(path.Join(dir, "index.json")) - if err != nil { - if os.IsNotExist(err) { - is.log.Debug().Err(err).Str("dir", dir).Msg("index.json doesn't exist") - - return []byte{}, zerr.ErrRepoNotFound - } - - is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") - - return []byte{}, err - } - - return buf, nil -} - -// DeleteBlob removes the blob from the repository. -func (is *ImageStoreLocal) DeleteBlob(repo string, digest godigest.Digest) error { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return err - } - - blobPath := is.BlobPath(repo, digest) - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return zerr.ErrBlobNotFound - } - - // first check if this blob is not currently in use - if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { - return zerr.ErrBlobReferenced - } - - if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.cache.DeleteBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("unable to remove blob path from cache") - - return err - } - } - - if err := os.Remove(blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil -} - -func (is *ImageStoreLocal) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, -) (ispec.Index, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) -} - -func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, -) ([]oras.Descriptor, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) -} - -func (is *ImageStoreLocal) writeFile(filename string, data []byte) error { - if !is.commit { - return os.WriteFile(filename, data, storageConstants.DefaultFilePerms) - } - - fhandle, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, storageConstants.DefaultFilePerms) - if err != nil { - return err - } - - _, err = fhandle.Write(data) - - if err1 := inject.Error(fhandle.Sync()); err1 != nil && err == nil { - err = err1 - is.log.Error().Err(err).Str("filename", filename).Msg("unable to sync file") - } - - if err1 := inject.Error(fhandle.Close()); err1 != nil && err == nil { - err = err1 - } - - return err -} - -func ValidateHardLink(rootDir string) error { - if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { - return err - } - - err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), - []byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms) - if err != nil { - return err - } - - err = os.Link(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt")) - if err != nil { - // Remove hardlinkcheck.txt if hardlink fails - zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) - if zerr != nil { - return zerr - } - - return err - } - - err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) - if err != nil { - return err - } - - return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt")) -} - -// utility routines. -func ensureDir(dir string, log zerolog.Logger) error { - if err := os.MkdirAll(dir, storageConstants.DefaultDirPerms); err != nil { - log.Error().Err(err).Str("dir", dir).Msg("unable to create dir") - - return err - } - - return nil -} - -type extendedManifest struct { - ispec.Manifest - - Digest godigest.Digest -} - -func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { - oci, err := umoci.OpenLayout(dir) - if err := inject.Error(err); err != nil { - return err - } - defer oci.Close() - - // gc untagged manifests and signatures - index, err := oci.GetIndex(context.Background()) - if err != nil { - return err - } - - referencedByImageIndex := []string{} - cosignDescriptors := []ispec.Descriptor{} - notationManifests := []extendedManifest{} - - /* gather manifests references by multiarch images (to skip gc) - gather cosign and notation signatures descriptors */ - for _, desc := range index.Manifests { - switch desc.MediaType { - case ispec.MediaTypeImageIndex: - indexImage, err := common.GetImageIndex(is, repo, desc.Digest, is.log) - if err != nil { - is.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read multiarch(index) image") - - return err - } - - for _, indexDesc := range indexImage.Manifests { - referencedByImageIndex = append(referencedByImageIndex, indexDesc.Digest.String()) - } - case ispec.MediaTypeImageManifest: - tag, ok := desc.Annotations[ispec.AnnotationRefName] - if ok { - // gather cosign references - if strings.HasPrefix(tag, "sha256-") && (strings.HasSuffix(tag, cosignSignatureTagSuffix) || - strings.HasSuffix(tag, SBOMTagSuffix)) { - cosignDescriptors = append(cosignDescriptors, desc) - - continue - } - } - - manifestContent, err := common.GetImageManifest(is, repo, desc.Digest, is.log) - if err != nil { - is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read manifest image") - - return err - } - - if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation { - notationManifests = append(notationManifests, extendedManifest{ - Digest: desc.Digest, - Manifest: manifestContent, - }) - - continue - } - } - } - - is.log.Info().Msg("gc: untagged manifests") - - if err := gcUntaggedManifests(is, oci, &index, repo, referencedByImageIndex); err != nil { - return err - } - - is.log.Info().Msg("gc: cosign references") - - if err := gcCosignReferences(is, oci, &index, repo, cosignDescriptors); err != nil { - return err - } - - is.log.Info().Msg("gc: notation signatures") - - if err := gcNotationSignatures(is, oci, &index, repo, notationManifests); err != nil { - return err - } - - is.log.Info().Msg("gc: blobs") - - err = oci.GC(context.Background(), ifOlderThan(is, repo, is.gcDelay)) - if err := inject.Error(err); err != nil { - return err - } - - return nil -} - -func gcUntaggedManifests(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - referencedByImageIndex []string, -) error { - for _, desc := range index.Manifests { - // skip manifests referenced in image indexex - if zcommon.Contains(referencedByImageIndex, desc.Digest.String()) { - continue - } - - // remove untagged images - if desc.MediaType == ispec.MediaTypeImageManifest { - _, ok := desc.Annotations[ispec.AnnotationRefName] - if !ok { - // check if is indeed an image and not an artifact by checking it's config blob - buf, err := imgStore.GetBlobContent(repo, desc.Digest) - if err != nil { - imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read image manifest") - - return err - } - - manifest := ispec.Manifest{} - - err = json.Unmarshal(buf, &manifest) - if err != nil { - return err - } - - // skip manifests which are not of type image - if manifest.Config.MediaType != ispec.MediaTypeImageConfig { - imgStore.log.Info().Str("config mediaType", manifest.Config.MediaType). - Msg("skipping gc untagged manifest, because config blob is not application/vnd.oci.image.config.v1+json") - - continue - } - - // remove manifest if it's older than gc.delay - canGC, err := isBlobOlderThan(imgStore, repo, desc.Digest, imgStore.gcDelay) - if err != nil { - imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Str("delay", imgStore.gcDelay.String()).Msg("gc: failed to check if blob is older than delay") - - return err - } - - if canGC { - imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: removing manifest without tag") - - _, err = common.RemoveManifestDescByReference(index, desc.Digest.String(), true) - if errors.Is(err, zerr.ErrManifestConflict) { - imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: skipping removing manifest due to conflict") - - continue - } - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - } - } - - return nil -} - -func gcCosignReferences(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - cosignDescriptors []ispec.Descriptor, -) error { - for _, cosignDesc := range cosignDescriptors { - foundSubject := false - // check if we can find the manifest which the reference points to - for _, desc := range index.Manifests { - // signature - subject := fmt.Sprintf("sha256-%s.%s", desc.Digest.Encoded(), cosignSignatureTagSuffix) - if subject == cosignDesc.Annotations[ispec.AnnotationRefName] { - foundSubject = true - } - - // sbom - subject = fmt.Sprintf("sha256-%s.%s", desc.Digest.Encoded(), SBOMTagSuffix) - if subject == cosignDesc.Annotations[ispec.AnnotationRefName] { - foundSubject = true - } - } - - if !foundSubject { - // remove manifest - imgStore.log.Info().Str("repository", repo).Str("digest", cosignDesc.Digest.String()). - Msg("gc: removing cosign reference without subject") - - // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = common.RemoveManifestDescByReference(index, cosignDesc.Digest.String(), false) - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - - return nil -} - -func gcNotationSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - notationManifests []extendedManifest, -) error { - for _, notationManifest := range notationManifests { - foundSubject := false - - for _, desc := range index.Manifests { - if desc.Digest == notationManifest.Subject.Digest { - foundSubject = true - } - } - - if !foundSubject { - // remove manifest - imgStore.log.Info().Str("repository", repo).Str("digest", notationManifest.Digest.String()). - Msg("gc: removing notation signature without subject") - - // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = common.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false) - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - - return nil -} - -func ifOlderThan(imgStore *ImageStoreLocal, repo string, delay time.Duration) casext.GCPolicy { - return func(ctx context.Context, digest godigest.Digest) (bool, error) { - return isBlobOlderThan(imgStore, repo, digest, delay) - } -} - -func isBlobOlderThan(imgStore *ImageStoreLocal, repo string, digest godigest.Digest, delay time.Duration, -) (bool, error) { - blobPath := imgStore.BlobPath(repo, digest) - - fileInfo, err := os.Stat(blobPath) - if err != nil { - imgStore.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("gc: failed to stat blob") - - return false, err - } - - if fileInfo.ModTime().Add(delay).After(time.Now()) { - return false, nil - } - - imgStore.log.Info().Str("digest", digest.String()).Str("blobPath", blobPath).Msg("perform GC on blob") - - return true, nil -} - -func (is *ImageStoreLocal) gcRepo(repo string) error { - dir := path.Join(is.RootDir(), repo) - - var lockLatency time.Time - - is.Lock(&lockLatency) - err := is.garbageCollect(dir, repo) - is.Unlock(&lockLatency) - - if err != nil { - return err - } - - return nil -} - -func (is *ImageStoreLocal) RunGCRepo(repo string) error { - is.log.Info().Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(is.RootDir(), repo))) - - if err := is.gcRepo(repo); err != nil { - errMessage := fmt.Sprintf("error while running GC for %s", path.Join(is.RootDir(), repo)) - is.log.Error().Err(err).Msg(errMessage) - is.log.Info().Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(is.RootDir(), repo))) - - return err - } - - is.log.Info().Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(is.RootDir(), repo))) - - return nil -} - -func (is *ImageStoreLocal) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { - generator := &common.GCTaskGenerator{ - ImgStore: is, - } - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) -} - -func (is *ImageStoreLocal) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest, -) (godigest.Digest, []string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - var duplicateBlobs []string - - var digest godigest.Digest - - err := filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { - if err != nil { - is.log.Warn().Err(err).Msg("unable to walk dir, skipping it") - // skip files/dirs which can't be walked - return filepath.SkipDir - } - - if info.IsDir() { - return nil - } - - blobDigest := godigest.NewDigestFromEncoded("sha256", info.Name()) - if err := blobDigest.Validate(); err != nil { - return nil //nolint:nilerr // ignore files which are not blobs - } - - if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { - digest = blobDigest - } - - if blobDigest == digest { - duplicateBlobs = append(duplicateBlobs, path) - } - - return nil - }) - - return digest, duplicateBlobs, err -} - -func (is *ImageStoreLocal) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") - - return zerr.ErrDedupeRebuild - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") - - var originalBlob string - - var originalBlobFi fs.FileInfo - - var err error - // rebuild from dedupe false to true - for _, blobPath := range duplicateBlobs { - /* for local storage, because we use hard links, we can assume that any blob can be original - so we skip the first one and hard link the rest of them with the first*/ - if originalBlob == "" { - originalBlob = blobPath - - originalBlobFi, err = os.Stat(originalBlob) - if err != nil { - is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - - continue - } - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // dedupe blob - if !os.SameFile(originalBlobFi, binfo) { - // we should link to a temp file instead of removing blob and then linking - // to make this more atomic - uuid, err := guuid.NewV4() - if err != nil { - return err - } - - // put temp blob in /.uploads dir - tempLinkBlobDir := path.Join(strings.Replace(blobPath, path.Join("blobs/sha256", binfo.Name()), "", 1), - storageConstants.BlobUploadDir) - - if err := os.MkdirAll(tempLinkBlobDir, storageConstants.DefaultDirPerms); err != nil { - is.log.Error().Err(err).Str("dir", tempLinkBlobDir).Msg("rebuild dedupe: unable to mkdir") - - return err - } - - tempLinkBlobPath := path.Join(tempLinkBlobDir, uuid.String()) - - if err := os.Link(originalBlob, tempLinkBlobPath); err != nil { - is.log.Error().Err(err).Str("src", originalBlob). - Str("dst", tempLinkBlobPath).Msg("rebuild dedupe: unable to hard link") - - return err - } - - if err := os.Rename(tempLinkBlobPath, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("rebuild dedupe: unable to rename temp link") - - return err - } - - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - } - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") - - return nil -} - -func (is *ImageStoreLocal) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if dedupe { - return is.dedupeBlobs(digest, duplicateBlobs) - } - - // otherwise noop - return nil -} - -func (is *ImageStoreLocal) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { - // for local storage no need to undedupe blobs - if is.dedupe { - generator := &common.DedupeTaskGenerator{ - ImgStore: is, - Dedupe: is.dedupe, - Log: is.log, - } - - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) - } + return imagestore.NewImageStore( + rootDir, + rootDir, + gc, + gcReferrers, + gcDelay, + untaggedImageRetentionDelay, + dedupe, + commit, + log, + metrics, + linter, + New(commit), + cacheDriver, + ) } diff --git a/pkg/storage/local/local_elevated_test.go b/pkg/storage/local/local_elevated_test.go index 07568921de..d7921aa421 100644 --- a/pkg/storage/local/local_elevated_test.go +++ b/pkg/storage/local/local_elevated_test.go @@ -36,8 +36,8 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, - metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) upload, err := imgStore.NewBlobUpload("dedupe1") So(err, ShouldBeNil) diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index 58be419ba9..c504f54a63 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -37,7 +37,6 @@ import ( "zotregistry.io/zot/pkg/storage/local" storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" - "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" ) @@ -68,8 +67,9 @@ func TestStorageFSAPIs(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) Convey("Repo layout", t, func(c C) { Convey("Bad image manifest", func() { @@ -205,7 +205,9 @@ func TestGetOrasReferrers(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) Convey("Get referrers", t, func(c C) { err := test.WriteImageToFileSystem(test.CreateDefaultVulnerableImage(), "zot-test", "0.0.1", storage.StoreController{ @@ -263,8 +265,9 @@ func FuzzNewBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) _, err := imgStore.NewBlobUpload(data) if err != nil { @@ -289,8 +292,8 @@ func FuzzPutBlobChunk(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := data uuid, err := imgStore.NewBlobUpload(repoName) @@ -323,8 +326,8 @@ func FuzzPutBlobChunkStreamed(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := data @@ -356,7 +359,8 @@ func FuzzGetBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) _, err := imgStore.GetBlobUpload(data1, data2) @@ -382,8 +386,8 @@ func FuzzTestPutGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) cblob, cdigest := test.GetRandomImageConfig() @@ -434,8 +438,8 @@ func FuzzTestPutDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) cblob, cdigest := test.GetRandomImageConfig() @@ -493,8 +497,8 @@ func FuzzTestDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest, _, err := newRandomBlobForFuzz(data) if err != nil { @@ -529,8 +533,8 @@ func FuzzInitRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) err := imgStore.InitRepo(data) if err != nil { if isKnownErr(err) { @@ -554,8 +558,8 @@ func FuzzInitValidateRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) err := imgStore.InitRepo(data) if err != nil { if isKnownErr(err) { @@ -586,8 +590,8 @@ func FuzzGetImageTags(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) _, err := imgStore.GetImageTags(data) if err != nil { if errors.Is(err, zerr.ErrRepoNotFound) || isKnownErr(err) { @@ -611,8 +615,8 @@ func FuzzBlobUploadPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) _ = imgStore.BlobUploadPath(repo, uuid) }) @@ -631,8 +635,8 @@ func FuzzBlobUploadInfo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) repo := data _, err := imgStore.BlobUploadInfo(repo, uuid) @@ -657,8 +661,8 @@ func FuzzTestGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := data @@ -686,8 +690,8 @@ func FuzzFinishBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := data @@ -736,8 +740,8 @@ func FuzzFullBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) ldigest, lblob, err := newRandomBlobForFuzz(data) if err != nil { @@ -767,7 +771,9 @@ func TestStorageCacheErrors(t *testing.T) { cblob, cdigest := test.GetRandomImageConfig() getBlobPath := "" - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, + + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, &mocks.CacheMock{ PutBlobFn: func(digest godigest.Digest, path string) error { if strings.Contains(path, dedupedRepo) { @@ -790,7 +796,7 @@ func TestStorageCacheErrors(t *testing.T) { _, _, err = imgStore.FullBlobUpload(originRepo, bytes.NewReader(cblob), cdigest) So(err, ShouldBeNil) - getBlobPath = strings.ReplaceAll(imgStore.BlobPath(originRepo, cdigest), imgStore.RootDir(), "") + getBlobPath = imgStore.BlobPath(originRepo, cdigest) _, _, err = imgStore.FullBlobUpload(dedupedRepo, bytes.NewReader(cblob), cdigest) So(err, ShouldNotBeNil) }) @@ -809,8 +815,8 @@ func FuzzDedupeBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) blobDigest := godigest.FromString(data) @@ -851,8 +857,8 @@ func FuzzDeleteBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) uuid, err := imgStore.NewBlobUpload(repoName) if err != nil { @@ -883,8 +889,8 @@ func FuzzBlobPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _ = imgStore.BlobPath(repoName, digest) @@ -905,8 +911,8 @@ func FuzzCheckBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -937,8 +943,8 @@ func FuzzGetBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -976,8 +982,8 @@ func FuzzDeleteBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1012,8 +1018,8 @@ func FuzzGetIndexContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1048,8 +1054,8 @@ func FuzzGetBlobContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1083,8 +1089,9 @@ func FuzzGetOrasReferrers(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) storageCtlr := storage.StoreController{DefaultStore: imgStore} err := test.WriteImageToFileSystem(test.CreateDefaultVulnerableImage(), "zot-test", "0.0.1", storageCtlr) @@ -1145,8 +1152,8 @@ func FuzzRunGCRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) if err := imgStore.RunGCRepo(data); err != nil { t.Error(err) @@ -1185,11 +1192,11 @@ func TestDedupeLinks(t *testing.T) { var imgStore storageTypes.ImageStore if testCase.dedupe { - imgStore = local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - testCase.dedupe, true, log, metrics, nil, cacheDriver) + imgStore = local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, testCase.dedupe, true, log, metrics, nil, cacheDriver) } else { - imgStore = local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - testCase.dedupe, true, log, metrics, nil, nil) + imgStore = local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, testCase.dedupe, true, log, metrics, nil, nil) } // manifest1 @@ -1338,8 +1345,8 @@ func TestDedupeLinks(t *testing.T) { Convey("test RunDedupeForDigest directly, trigger stat error on original blob", func() { // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) duplicateBlobs := []string{ path.Join(dir, "dedupe1", "blobs", "sha256", blobDigest1), @@ -1358,8 +1365,8 @@ func TestDedupeLinks(t *testing.T) { for i := 0; i < 10; i++ { taskScheduler, cancel := runAndGetScheduler() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) sleepValue := i * 50 @@ -1371,8 +1378,8 @@ func TestDedupeLinks(t *testing.T) { taskScheduler, cancel := runAndGetScheduler() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1391,8 +1398,8 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, nil) + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, nil) // rebuild with dedupe true imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) @@ -1414,7 +1421,8 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, &mocks.CacheMock{ HasBlobFn: func(digest godigest.Digest, path string) bool { return false @@ -1443,7 +1451,8 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, &mocks.CacheMock{ HasBlobFn: func(digest godigest.Digest, path string) bool { return false @@ -1469,7 +1478,6 @@ func TestDedupeLinks(t *testing.T) { fi2, err := os.Stat(path.Join(dir, "dedupe2", "blobs", "sha256", blobDigest2)) So(err, ShouldBeNil) - // deduped happened, but didn't cached So(os.SameFile(fi1, fi2), ShouldEqual, true) }) } @@ -1539,7 +1547,9 @@ func TestDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - il := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + + il := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(il.DedupeBlob("", "", ""), ShouldNotBeNil) }) @@ -1558,7 +1568,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - So(local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, + + So(local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver), ShouldNotBeNil) if os.Geteuid() != 0 { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ @@ -1566,8 +1578,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - So(local.NewImageStore("/deadBEEF", true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver), ShouldBeNil) + So(local.NewImageStore("/deadBEEF", true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, + true, log, metrics, nil, cacheDriver), ShouldBeNil) } }) @@ -1581,8 +1594,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) err := os.Chmod(dir, 0o000) // remove all perms if err != nil { @@ -1631,8 +1645,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1719,9 +1734,8 @@ func TestNegativeCases(t *testing.T) { panic(err) } - if os.Geteuid() != 0 { - So(func() { _, _ = imgStore.ValidateRepo("test") }, ShouldPanic) - } + _, err = imgStore.GetRepositories() + So(err, ShouldNotBeNil) err = os.Chmod(dir, 0o755) // add perms if err != nil { @@ -1732,16 +1746,9 @@ func TestNegativeCases(t *testing.T) { if err != nil { panic(err) } - - _, err = imgStore.GetRepositories() - So(err, ShouldNotBeNil) }) Convey("Invalid get image tags", t, func(c C) { - var ilfs local.ImageStoreLocal - _, err := ilfs.GetImageTags("test") - So(err, ShouldNotBeNil) - dir := t.TempDir() log := log.Logger{Logger: zerolog.New(os.Stdout)} @@ -1751,13 +1758,14 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil) - _, err = imgStore.GetImageTags("test") + _, err := imgStore.GetImageTags("test") So(err, ShouldNotBeNil) So(os.RemoveAll(path.Join(dir, "test")), ShouldBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1767,10 +1775,6 @@ func TestNegativeCases(t *testing.T) { }) Convey("Invalid get image manifest", t, func(c C) { - var ilfs local.ImageStoreLocal - _, _, _, err := ilfs.GetImageManifest("test", "") - So(err, ShouldNotBeNil) - dir := t.TempDir() log := log.Logger{Logger: zerolog.New(os.Stdout)} @@ -1780,13 +1784,14 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) - err = os.Chmod(path.Join(dir, "test", "index.json"), 0o000) + err := os.Chmod(path.Join(dir, "test", "index.json"), 0o000) if err != nil { panic(err) } @@ -1827,8 +1832,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1988,42 +1994,6 @@ func TestHardLink(t *testing.T) { } func TestInjectWriteFile(t *testing.T) { - Convey("writeFile with commit", t, func() { - dir := t.TempDir() - - log := log.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) - - Convey("Failure path1", func() { - injected := inject.InjectFailure(0) - - err := imgStore.InitRepo("repo1") - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - }) - - Convey("Failure path2", func() { - injected := inject.InjectFailure(1) - - err := imgStore.InitRepo("repo2") - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - }) - }) - Convey("writeFile without commit", t, func() { dir := t.TempDir() @@ -2034,8 +2004,9 @@ func TestInjectWriteFile(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, false, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, false, log, metrics, nil, cacheDriver) Convey("Failure path not reached", func() { err := imgStore.InitRepo("repo1") @@ -2044,647 +2015,162 @@ func TestInjectWriteFile(t *testing.T) { }) } -func TestGCInjectFailure(t *testing.T) { - Convey("code coverage: error inside garbageCollect method of img store", t, func() { +func TestGarbageCollectForImageStore(t *testing.T) { + Convey("Garbage collect for a specific repo from an ImageStore", t, func(c C) { dir := t.TempDir() - logFile, _ := os.CreateTemp("", "zot-log*.txt") - defer os.Remove(logFile.Name()) // clean up + Convey("Garbage collect error for repo with config removed", func() { + logFile, _ := os.CreateTemp("", "zot-log*.txt") - log := log.NewLogger("debug", logFile.Name()) - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) - repoName := "test-gc" + defer os.Remove(logFile.Name()) // clean up - upload, err := imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) + log := log.NewLogger("debug", logFile.Name()) + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) - content := []byte("test-data1") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - bdigest := godigest.FromBytes(content) + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, storageConstants.DefaultUntaggedImgeRetentionDelay, + true, true, log, metrics, nil, cacheDriver) + repoName := "gc-all-repos-short" - blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) + image := test.CreateDefaultVulnerableImage() + err := test.WriteImageToFileSystem(image, repoName, "0.0.1", storage.StoreController{ + DefaultStore: imgStore, + }) + So(err, ShouldBeNil) - err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) - So(err, ShouldBeNil) + manifestDigest := image.ManifestDescriptor.Digest + err = os.Remove(path.Join(dir, repoName, "blobs/sha256", manifestDigest.Encoded())) + if err != nil { + panic(err) + } - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag + err = imgStore.RunGCRepo(repoName) + So(err, ShouldNotBeNil) - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) + time.Sleep(500 * time.Millisecond) - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } + data, err := os.ReadFile(logFile.Name()) + So(err, ShouldBeNil) + So(string(data), ShouldContainSubstring, + fmt.Sprintf("error while running GC for %s", path.Join(imgStore.RootDir(), repoName))) + }) - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) + Convey("Garbage collect error - not enough permissions to access index.json", func() { + logFile, _ := os.CreateTemp("", "zot-log*.txt") - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) + defer os.Remove(logFile.Name()) // clean up - // umoci.OpenLayout error - injected := inject.InjectFailure(0) + log := log.NewLogger("debug", logFile.Name()) + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) - err = imgStore.RunGCRepo(repoName) + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, storageConstants.DefaultUntaggedImgeRetentionDelay, + true, true, log, metrics, nil, cacheDriver) + repoName := "gc-all-repos-short" - if injected { - So(err, ShouldNotBeNil) - } else { + image := test.CreateDefaultVulnerableImage() + err := test.WriteImageToFileSystem(image, repoName, "0.0.1", storage.StoreController{ + DefaultStore: imgStore, + }) So(err, ShouldBeNil) - } - - // oci.GC - injected = inject.InjectFailure(1) - err = imgStore.RunGCRepo(repoName) + So(os.Chmod(path.Join(dir, repoName, "index.json"), 0o000), ShouldBeNil) - if injected { + err = imgStore.RunGCRepo(repoName) So(err, ShouldNotBeNil) - } else { + + time.Sleep(500 * time.Millisecond) + + data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) - } - }) -} + So(string(data), ShouldContainSubstring, + fmt.Sprintf("error while running GC for %s", path.Join(imgStore.RootDir(), repoName))) + So(os.Chmod(path.Join(dir, repoName, "index.json"), 0o755), ShouldBeNil) + }) -func TestGarbageCollect(t *testing.T) { - Convey("Repo layout", t, func(c C) { - dir := t.TempDir() + Convey("Garbage collect - the manifest which the reference points to can be found", func() { + logFile, _ := os.CreateTemp("", "zot-log*.txt") - log := log.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) + defer os.Remove(logFile.Name()) // clean up - Convey("Garbage collect with default/long delay", func() { + log := log.NewLogger("debug", logFile.Name()) + metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) - repoName := "gc-long" - - upload, err := imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) + repoName := "gc-sig" - content := []byte("test-data1") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - bdigest := godigest.FromBytes(content) + storeController := storage.StoreController{DefaultStore: imgStore} + img := test.CreateRandomImage() - blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + err := test.WriteImageToFileSystem(img, repoName, "tag1", storeController) So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) + // add fake signature for tag1 + cosignTag, err := test.GetCosignSignatureTagForManifest(img.Manifest) So(err, ShouldBeNil) - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) + cosignSig := test.CreateRandomImage() So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) + err = test.WriteImageToFileSystem(cosignSig, repoName, cosignTag, storeController) So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + // add sbom + manifestBlob, err := json.Marshal(img.Manifest) So(err, ShouldBeNil) - err = imgStore.RunGCRepo(repoName) + manifestDigest := godigest.FromBytes(manifestBlob) + sbomTag := fmt.Sprintf("sha256-%s.%s", manifestDigest.Encoded(), "sbom") So(err, ShouldBeNil) - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + sbomImg := test.CreateRandomImage() So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - err = imgStore.DeleteImageManifest(repoName, digest.String(), false) + err = test.WriteImageToFileSystem(sbomImg, repoName, sbomTag, storeController) So(err, ShouldBeNil) - err = imgStore.RunGCRepo(repoName) + // add fake signature for tag1 + notationSig := test.CreateImageWith(). + RandomLayers(1, 10). + ArtifactConfig("application/vnd.cncf.notary.signature"). + Subject(img.DescriptorRef()).Build() + + err = test.WriteImageToFileSystem(notationSig, repoName, "notation", storeController) So(err, ShouldBeNil) - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + err = imgStore.RunGCRepo(repoName) So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) }) + }) +} - Convey("Garbage collect with short delay", func() { - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) - repoName := "gc-short" +func TestGarbageCollectErrors(t *testing.T) { + Convey("Make image store", t, func(c C) { + dir := t.TempDir() - // upload orphan blob - upload, err := imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) + log := log.NewLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) - content := []byte("test-data1") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - odigest := godigest.FromBytes(content) - - blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) - So(err, ShouldBeNil) - - // sleep so orphan blob can be GC'ed - time.Sleep(5 * time.Second) - - // upload blob - upload, err = imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content = []byte("test-data2") - buf = bytes.NewBuffer(content) - buflen = buf.Len() - bdigest := godigest.FromBytes(content) - - blob, err = imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - - hasBlob, _, err = imgStore.StatBlob(repoName, odigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - hasBlob, _, err = imgStore.StatBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // sleep so orphan blob can be GC'ed - time.Sleep(5 * time.Second) - - err = imgStore.DeleteImageManifest(repoName, digest.String(), false) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - }) - - Convey("Garbage collect with dedupe", func() { - // garbage-collect is repo-local and dedupe is global and they can interact in strange ways - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics, nil, cacheDriver) - - // first upload an image to the first repo and wait for GC timeout - - repo1Name := "gc1" - - // upload blob - upload, err := imgStore.NewBlobUpload(repo1Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content := []byte("test-data") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - bdigest := godigest.FromBytes(content) - tdigest := bdigest - - blob, err := imgStore.PutBlobChunk(repo1Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo1Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repo1Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repo1Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - _, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // sleep so past GC timeout - time.Sleep(10 * time.Second) - - hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // upload another image into a second repo with the same blob contents so dedupe is triggered - - repo2Name := "gc2" - - upload, err = imgStore.NewBlobUpload(repo2Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - buf = bytes.NewBuffer(content) - buflen = buf.Len() - - blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap = make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest = test.GetRandomImageConfig() - _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest = ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - - _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // immediately upload any other image to second repo and run GC, but expect layers to persist - - upload, err = imgStore.NewBlobUpload(repo2Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content = []byte("test-data-more") - buf = bytes.NewBuffer(content) - buflen = buf.Len() - bdigest = godigest.FromBytes(content) - - blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap = make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest = test.GetRandomImageConfig() - _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest = ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - - _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repo2Name) - So(err, ShouldBeNil) - - // original blob should exist - - hasBlob, _, err = imgStore.CheckBlob(repo2Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - _, _, _, err = imgStore.GetImageManifest(repo2Name, digest.String()) - So(err, ShouldBeNil) - }) - }) -} - -func TestGarbageCollectForImageStore(t *testing.T) { - Convey("Garbage collect for a specific repo from an ImageStore", t, func(c C) { - dir := t.TempDir() - - Convey("Garbage collect error for repo with config removed", func() { - logFile, _ := os.CreateTemp("", "zot-log*.txt") - - defer os.Remove(logFile.Name()) // clean up - - log := log.NewLogger("debug", logFile.Name()) - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) - repoName := "gc-all-repos-short" - - image := test.CreateDefaultVulnerableImage() - err := test.WriteImageToFileSystem(image, repoName, "0.0.1", storage.StoreController{ - DefaultStore: imgStore, - }) - So(err, ShouldBeNil) - - manifestDigest := image.ManifestDescriptor.Digest - err = os.Remove(path.Join(dir, repoName, "blobs/sha256", manifestDigest.Encoded())) - if err != nil { - panic(err) - } - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldNotBeNil) - - time.Sleep(500 * time.Millisecond) - - data, err := os.ReadFile(logFile.Name()) - So(err, ShouldBeNil) - So(string(data), ShouldContainSubstring, - fmt.Sprintf("error while running GC for %s", path.Join(imgStore.RootDir(), repoName))) - }) - - Convey("Garbage collect error - not enough permissions to access index.json", func() { - logFile, _ := os.CreateTemp("", "zot-log*.txt") - - defer os.Remove(logFile.Name()) // clean up - - log := log.NewLogger("debug", logFile.Name()) - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) - repoName := "gc-all-repos-short" - - image := test.CreateDefaultVulnerableImage() - err := test.WriteImageToFileSystem(image, repoName, "0.0.1", storage.StoreController{ - DefaultStore: imgStore, - }) - So(err, ShouldBeNil) - - So(os.Chmod(path.Join(dir, repoName, "index.json"), 0o000), ShouldBeNil) - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldNotBeNil) - - time.Sleep(500 * time.Millisecond) - - data, err := os.ReadFile(logFile.Name()) - So(err, ShouldBeNil) - So(string(data), ShouldContainSubstring, - fmt.Sprintf("error while running GC for %s", path.Join(imgStore.RootDir(), repoName))) - So(os.Chmod(path.Join(dir, repoName, "index.json"), 0o755), ShouldBeNil) - }) - - Convey("Garbage collect - the manifest which the reference points to can be found", func() { - logFile, _ := os.CreateTemp("", "zot-log*.txt") - - defer os.Remove(logFile.Name()) // clean up - - log := log.NewLogger("debug", logFile.Name()) - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) - repoName := "gc-sig" - - storeController := storage.StoreController{DefaultStore: imgStore} - img := test.CreateRandomImage() - - err := test.WriteImageToFileSystem(img, repoName, "tag1", storeController) - So(err, ShouldBeNil) - - // add fake signature for tag1 - cosignTag, err := test.GetCosignSignatureTagForManifest(img.Manifest) - So(err, ShouldBeNil) - - cosignSig := test.CreateRandomImage() - So(err, ShouldBeNil) - - err = test.WriteImageToFileSystem(cosignSig, repoName, cosignTag, storeController) - So(err, ShouldBeNil) - - // add sbom - manifestBlob, err := json.Marshal(img.Manifest) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBlob) - sbomTag := fmt.Sprintf("sha256-%s.%s", manifestDigest.Encoded(), "sbom") - So(err, ShouldBeNil) - - sbomImg := test.CreateRandomImage() - So(err, ShouldBeNil) - - err = test.WriteImageToFileSystem(sbomImg, repoName, sbomTag, storeController) - So(err, ShouldBeNil) - - // add fake signature for tag1 - notationSig := test.CreateImageWith(). - RandomLayers(1, 10). - ArtifactConfig("application/vnd.cncf.notary.signature"). - Subject(img.DescriptorRef()).Build() - - err = test.WriteImageToFileSystem(notationSig, repoName, "notation", storeController) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldBeNil) - }) - }) -} - -func TestGarbageCollectErrors(t *testing.T) { - Convey("Make image store", t, func(c C) { - dir := t.TempDir() - - log := log.NewLogger("debug", "") - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 500*time.Millisecond, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, 500*time.Millisecond, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := "gc-index" // create a blob/layer @@ -2919,8 +2405,9 @@ func TestInitRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -2941,8 +2428,9 @@ func TestValidateRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -2961,8 +2449,9 @@ func TestValidateRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) _, err := imgStore.ValidateRepo(".") So(err, ShouldNotBeNil) @@ -3006,8 +2495,9 @@ func TestGetRepositories(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver, + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver, ) // Create valid directory with permissions @@ -3103,8 +2593,8 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver, + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver, ) // Root dir does not contain repos @@ -3151,7 +2641,8 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(rootDir, true, storageConstants.DefaultGCDelay, + imgStore := local.NewImageStore(rootDir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver, ) @@ -3194,8 +2685,9 @@ func TestGetNextRepository(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver, + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver, ) firstRepoName := "repo1" secondRepoName := "repo2" @@ -3218,15 +2710,13 @@ func TestGetNextRepository(t *testing.T) { Convey("Return first repository", t, func() { firstRepo, err := imgStore.GetNextRepository("") So(firstRepo, ShouldEqual, firstRepoName) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, io.EOF) + So(err, ShouldBeNil) }) Convey("Return second repository", t, func() { secondRepo, err := imgStore.GetNextRepository(firstRepoName) So(secondRepo, ShouldEqual, secondRepoName) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, io.EOF) + So(err, ShouldBeNil) }) Convey("Return error", t, func() { @@ -3250,8 +2740,9 @@ func TestPutBlobChunkStreamed(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) uuid, err := imgStore.NewBlobUpload("test") So(err, ShouldBeNil) @@ -3279,8 +2770,9 @@ func TestPullRange(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := "pull-range" upload, err := imgStore.NewBlobUpload(repoName) @@ -3317,6 +2809,100 @@ func TestPullRange(t *testing.T) { }) } +func TestStorageDriverErr(t *testing.T) { + dir := t.TempDir() + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) + + Convey("Init repo", t, func() { + err := imgStore.InitRepo(repoName) + So(err, ShouldBeNil) + + Convey("New blob upload error", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, storageConstants.BlobUploadDir), 0o000) + So(err, ShouldBeNil) + + _, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(imgStore.RootDir(), repoName, storageConstants.BlobUploadDir), + storageConstants.DefaultDirPerms) + So(err, ShouldBeNil) + + uuid, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + size, err := imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + So(size, ShouldEqual, 0) + + content := []byte("test-blob") + buf := bytes.NewBuffer(content) + bufLen := buf.Len() + digest := godigest.FromBytes(content) + + size, err = imgStore.PutBlobChunkStreamed(repoName, uuid, buf) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + size, err = imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + err = imgStore.DeleteBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + + err = imgStore.DeleteBlobUpload(repoName, uuid) + So(err, ShouldNotBeNil) + + _, err = imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldNotBeNil) + + // push again + buf = bytes.NewBuffer(content) + + uuid, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + size, err = imgStore.PutBlobChunkStreamed(repoName, uuid, buf) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + // finish blob upload + err = os.Chmod(path.Join(imgStore.BlobUploadPath(repoName, uuid)), 0o000) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(imgStore.BlobUploadPath(repoName, uuid)), storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldNotBeNil) + + // delete blob + err = imgStore.DeleteBlob(repoName, digest) + So(err, ShouldBeNil) + + err = imgStore.DeleteBlob(repoName, digest) + So(err, ShouldNotBeNil) + }) + }) +} + func NewRandomImgManifest(data []byte, cdigest, ldigest godigest.Digest, cblob, lblob []byte) (*ispec.Manifest, error) { annotationsMap := make(map[string]string) diff --git a/pkg/storage/s3/driver.go b/pkg/storage/s3/driver.go new file mode 100644 index 0000000000..19b0c860f9 --- /dev/null +++ b/pkg/storage/s3/driver.go @@ -0,0 +1,115 @@ +package s3 + +import ( + "context" + "io" + + // Add s3 support. + "github.com/docker/distribution/registry/storage/driver" + _ "github.com/docker/distribution/registry/storage/driver/s3-aws" + + storageConstants "zotregistry.io/zot/pkg/storage/constants" +) + +type Driver struct { + store driver.StorageDriver +} + +func New(storeDriver driver.StorageDriver) *Driver { + return &Driver{store: storeDriver} +} + +func (driver *Driver) Name() string { + return storageConstants.S3StorageDriverName +} + +func (driver *Driver) EnsureDir(path string) error { + return nil +} + +func (driver *Driver) DirExists(path string) bool { + if fi, err := driver.store.Stat(context.Background(), path); err == nil && fi.IsDir() { + return true + } + + return false +} + +func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) { + return driver.store.Reader(context.Background(), path, offset) +} + +func (driver *Driver) ReadFile(path string) ([]byte, error) { + return driver.store.GetContent(context.Background(), path) +} + +func (driver *Driver) Delete(path string) error { + return driver.store.Delete(context.Background(), path) +} + +func (driver *Driver) Stat(path string) (driver.FileInfo, error) { + return driver.store.Stat(context.Background(), path) +} + +func (driver *Driver) Writer(filepath string, append bool) (driver.FileWriter, error) { //nolint:predeclared + return driver.store.Writer(context.Background(), filepath, append) +} + +func (driver *Driver) WriteFile(filepath string, content []byte) (int, error) { + var n int + + if stwr, err := driver.store.Writer(context.Background(), filepath, false); err == nil { + defer stwr.Close() + + if n, err = stwr.Write(content); err != nil { + return -1, err + } + + if err := stwr.Commit(); err != nil { + return -1, err + } + } else { + return -1, err + } + + return n, nil +} + +func (driver *Driver) Walk(path string, f driver.WalkFn) error { + return driver.store.Walk(context.Background(), path, f) +} + +func (driver *Driver) List(fullpath string) ([]string, error) { + return driver.store.List(context.Background(), fullpath) +} + +func (driver *Driver) Move(sourcePath string, destPath string) error { + return driver.store.Move(context.Background(), sourcePath, destPath) +} + +func (driver *Driver) SameFile(path1, path2 string) bool { + fi1, _ := driver.store.Stat(context.Background(), path1) + + fi2, _ := driver.store.Stat(context.Background(), path2) + + if fi1 != nil && fi2 != nil { + if fi1.IsDir() == fi2.IsDir() && + fi1.ModTime() == fi2.ModTime() && + fi1.Path() == fi2.Path() && + fi1.Size() == fi2.Size() { + return true + } + } + + return false +} + +/* + Link put an empty file that will act like a link between the original file and deduped one + +because s3 doesn't support symlinks, wherever the storage will encounter an empty file, it will get the original one +from cache. +*/ +func (driver *Driver) Link(src, dest string) error { + return driver.store.PutContent(context.Background(), dest, []byte{}) +} diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index b02e5d2dcb..16f7050b6d 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -1,1729 +1,41 @@ package s3 import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "path" - "path/filepath" - "sync" "time" // Add s3 support. "github.com/docker/distribution/registry/storage/driver" // Load s3 driver. _ "github.com/docker/distribution/registry/storage/driver/s3-aws" - guuid "github.com/gofrs/uuid" - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" - zerr "zotregistry.io/zot/errors" - zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" - zreg "zotregistry.io/zot/pkg/regexp" - "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage/cache" common "zotregistry.io/zot/pkg/storage/common" - storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" storageTypes "zotregistry.io/zot/pkg/storage/types" - "zotregistry.io/zot/pkg/test/inject" ) -const ( - CacheDBName = "s3_cache" -) - -// ObjectStorage provides the image storage operations. -type ObjectStorage struct { - rootDir string - store driver.StorageDriver - lock *sync.RWMutex - log zerolog.Logger - metrics monitoring.MetricServer - cache cache.Cache - dedupe bool - linter common.Lint -} - -func (is *ObjectStorage) RootDir() string { - return is.rootDir -} - -func (is *ObjectStorage) DirExists(d string) bool { - if fi, err := is.store.Stat(context.Background(), d); err == nil && fi.IsDir() { - return true - } - - return false -} - // NewObjectStorage returns a new image store backed by cloud storages. // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers // Use the last argument to properly set a cache database, or it will default to boltDB local storage. -func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, - log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, - store driver.StorageDriver, cacheDriver cache.Cache, +func NewImageStore(rootDir string, cacheDir string, gc bool, gcReferrers bool, gcDelay time.Duration, + untaggedImageRetentionDelay time.Duration, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, + linter common.Lint, store driver.StorageDriver, cacheDriver cache.Cache, ) storageTypes.ImageStore { - imgStore := &ObjectStorage{ - rootDir: rootDir, - store: store, - lock: &sync.RWMutex{}, - log: log.With().Caller().Logger(), - metrics: metrics, - dedupe: dedupe, - linter: linter, - } - - imgStore.cache = cacheDriver - - return imgStore -} - -// RLock read-lock. -func (is *ObjectStorage) RLock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.RLock() -} - -// RUnlock read-unlock. -func (is *ObjectStorage) RUnlock(lockStart *time.Time) { - is.lock.RUnlock() - - lockEnd := time.Now() - // includes time spent in acquiring and holding a lock - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram -} - -// Lock write-lock. -func (is *ObjectStorage) Lock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.Lock() -} - -// Unlock write-unlock. -func (is *ObjectStorage) Unlock(lockStart *time.Time) { - is.lock.Unlock() - - lockEnd := time.Now() - // includes time spent in acquiring and holding a lock - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram -} - -func (is *ObjectStorage) initRepo(name string) error { - repoDir := path.Join(is.rootDir, name) - - if !zreg.FullNameRegexp.MatchString(name) { - is.log.Error().Str("repository", name).Msg("invalid repository name") - - return zerr.ErrInvalidRepositoryName - } - - // "oci-layout" file - create if it doesn't exist - ilPath := path.Join(repoDir, ispec.ImageLayoutFile) - if _, err := is.store.Stat(context.Background(), ilPath); err != nil { - il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} - - buf, err := json.Marshal(il) - if err != nil { - is.log.Error().Err(err).Msg("unable to marshal JSON") - - return err - } - - if _, err := writeFile(is.store, ilPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - // "index.json" file - create if it doesn't exist - indexPath := path.Join(repoDir, "index.json") - if _, err := is.store.Stat(context.Background(), indexPath); err != nil { - index := ispec.Index{} - index.SchemaVersion = 2 - - buf, err := json.Marshal(index) - if err != nil { - is.log.Error().Err(err).Msg("unable to marshal JSON") - - return err - } - - if _, err := writeFile(is.store, indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - return nil -} - -// InitRepo creates an image repository under this store. -func (is *ObjectStorage) InitRepo(name string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - return is.initRepo(name) -} - -// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. -func (is *ObjectStorage) ValidateRepo(name string) (bool, error) { - if !zreg.FullNameRegexp.MatchString(name) { - return false, zerr.ErrInvalidRepositoryName - } - - // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content - // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] - // and an additional/optional BlobUploadDir in each image store - // for objects storage we can not create empty dirs, so we check only against index.json and oci-layout - dir := path.Join(is.rootDir, name) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return false, zerr.ErrRepoNotFound - } - - files, err := is.store.List(context.Background(), dir) - if err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") - - return false, zerr.ErrRepoNotFound - } - - //nolint:gomnd - if len(files) < 2 { - return false, zerr.ErrRepoBadVersion - } - - found := map[string]bool{ - ispec.ImageLayoutFile: false, - "index.json": false, - } - - for _, file := range files { - _, err := is.store.Stat(context.Background(), file) - if err != nil { - return false, err - } - - filename, err := filepath.Rel(dir, file) - if err != nil { - return false, err - } - - found[filename] = true - } - - for k, v := range found { - if !v && k != storageConstants.BlobUploadDir { - return false, nil - } - } - - buf, err := is.store.GetContent(context.Background(), path.Join(dir, ispec.ImageLayoutFile)) - if err != nil { - return false, err - } - - var il ispec.ImageLayout - if err := json.Unmarshal(buf, &il); err != nil { - return false, err - } - - if il.Version != ispec.ImageLayoutVersion { - return false, zerr.ErrRepoBadVersion - } - - return true, nil -} - -// GetRepositories returns a list of all the repositories under this store. -func (is *ObjectStorage) GetRepositories() ([]string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - stores := make([]string, 0) - err := is.store.Walk(context.Background(), dir, func(fileInfo driver.FileInfo) error { - if !fileInfo.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) - if err != nil { - return nil //nolint:nilerr // ignore paths that are not under root dir - } - - if ok, err := is.ValidateRepo(rel); !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - stores = append(stores, rel) - - return nil - }) - - // if the root directory is not yet created then return an empty slice of repositories - var perr driver.PathNotFoundError - if errors.As(err, &perr) { - return stores, nil - } - - return stores, err -} - -// GetNextRepository returns next repository under this store. -func (is *ObjectStorage) GetNextRepository(repo string) (string, error) { - return "", nil -} - -// GetImageTags returns a list of image tags available in the specified repository. -func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) { - var lockLatency time.Time - - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return nil, zerr.ErrRepoNotFound - } - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, err - } - - return common.GetTagsByIndex(index), nil -} - -// GetImageManifest returns the image manifest of an image in the specific repository. -func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return nil, "", "", zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.RLock(&lockLatency) - defer func() { - is.RUnlock(&lockLatency) - - if err == nil { - monitoring.IncDownloadCounter(is.metrics, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, "", "", zerr.ErrRepoNotFound - } - - manifestDesc, found := common.GetManifestDescByReference(index, reference) - if !found { - return nil, "", "", zerr.ErrManifestNotFound - } - - buf, err := is.GetBlobContent(repo, manifestDesc.Digest) - if err != nil { - if errors.Is(err, zerr.ErrBlobNotFound) { - return nil, "", "", zerr.ErrManifestNotFound - } - - return nil, "", "", err - } - - var manifest ispec.Manifest - if err := json.Unmarshal(buf, &manifest); err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") - - return nil, "", "", err - } - - return buf, manifestDesc.Digest, manifestDesc.MediaType, nil -} - -// PutImageManifest adds an image manifest to the repository. -func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo - body []byte, -) (godigest.Digest, godigest.Digest, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Debug().Err(err).Msg("init repo") - - return "", "", err - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - monitoring.IncUploadCounter(is.metrics, repo) - } - }() - - refIsDigest := true - - mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) - if err != nil { - if errors.Is(err, zerr.ErrBadManifest) { - return mDigest, "", err - } - - refIsDigest = false - } - - dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) - if err != nil { - return dig, "", err - } - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return "", "", err - } - - // create a new descriptor - desc := ispec.Descriptor{ - MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, - } - - if !refIsDigest { - desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} - } - - var subjectDigest godigest.Digest - - artifactType := "" - - if mediaType == ispec.MediaTypeImageManifest { - var manifest ispec.Manifest - - err := json.Unmarshal(body, &manifest) - if err != nil { - return "", "", err - } - - if manifest.Subject != nil { - subjectDigest = manifest.Subject.Digest - } - - artifactType = zcommon.GetManifestArtifactType(manifest) - } else if mediaType == ispec.MediaTypeImageIndex { - var index ispec.Index - - err := json.Unmarshal(body, &index) - if err != nil { - return "", "", err - } - - if index.Subject != nil { - subjectDigest = index.Subject.Digest - } - - artifactType = zcommon.GetIndexArtifactType(index) - } - - updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) - if err != nil { - return "", "", err - } - - if !updateIndex { - return desc.Digest, subjectDigest, nil - } - - // write manifest to "blobs" - dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) - manifestPath := path.Join(dir, mDigest.Encoded()) - - if err = is.store.PutContent(context.Background(), manifestPath, body); err != nil { - is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") - - return "", "", err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) - if err != nil { - return "", "", err - } - - // now update "index.json" - index.Manifests = append(index.Manifests, desc) - dir = path.Join(is.rootDir, repo) - indexPath := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - is.log.Error().Err(err).Str("file", indexPath).Msg("unable to marshal JSON") - - return "", "", err - } - - // update the descriptors artifact type in order to check for signatures when applying the linter - desc.ArtifactType = artifactType - - // apply linter only on images, not signatures - pass, err := common.ApplyLinter(is, is.linter, repo, desc) - if !pass { - is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") - - return "", "", err - } - - if err = is.store.PutContent(context.Background(), indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") - - return "", "", err - } - - return desc.Digest, subjectDigest, nil -} - -// DeleteImageManifest deletes the image manifest from the repository. -func (is *ObjectStorage) DeleteImageManifest(repo, reference string, detectCollisions bool) error { - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return err - } - - manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollisions) - if err != nil { - return err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) - if err != nil { - return err - } - - // now update "index.json" - dir = path.Join(is.rootDir, repo) - file := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - return err - } - - if _, err := writeFile(is.store, file, buf); err != nil { - is.log.Debug().Str("deleting reference", reference).Msg("") - - return err - } - - // Delete blob only when blob digest not present in manifest entry. - // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. - toDelete := true - - for _, manifest := range index.Manifests { - if manifestDesc.Digest.String() == manifest.Digest.String() { - toDelete = false - - break - } - } - - if toDelete { - p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) - - err = is.store.Delete(context.Background(), p) - if err != nil { - return err - } - } - - return nil -} - -// BlobUploadPath returns the upload path for a blob in this store. -func (is *ObjectStorage) BlobUploadPath(repo, uuid string) string { - dir := path.Join(is.rootDir, repo) - blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) - - return blobUploadPath -} - -// NewBlobUpload returns the unique ID for an upload in progress. -func (is *ObjectStorage) NewBlobUpload(repo string) (string, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Error().Err(err).Msg("error initializing repo") - - return "", err - } - - uuid, err := guuid.NewV4() - if err != nil { - return "", err - } - - uid := uuid.String() - - blobUploadPath := is.BlobUploadPath(repo, uid) - - // create multipart upload (append false) - _, err = is.store.Writer(context.Background(), blobUploadPath, false) - if err != nil { - return "", err - } - - return uid, nil -} - -// GetBlobUpload returns the current size of a blob upload. -func (is *ObjectStorage) GetBlobUpload(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return writer.Size(), nil -} - -// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ObjectStorage) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - file, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - is.log.Error().Err(err).Msg("failed to continue multipart upload") - - return -1, err - } - - defer file.Close() - - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return -1, err - } - - nbytes, err := file.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to append to file") - - return -1, err - } - - return int64(nbytes), err -} - -// PutBlobChunk writes another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ObjectStorage) PutBlobChunk(repo, uuid string, from, to int64, - body io.Reader, -) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - file, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - is.log.Error().Err(err).Msg("failed to continue multipart upload") - - return -1, err - } - - defer file.Close() - - if from != file.Size() { - is.log.Error().Int64("expected", from).Int64("actual", file.Size()). - Msg("invalid range start for blob upload") - - return -1, zerr.ErrBadUploadRange - } - - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return -1, err - } - - nbytes, err := file.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to append to file") - - return -1, err - } - - return int64(nbytes), err -} - -// BlobUploadInfo returns the current blob size in bytes. -func (is *ObjectStorage) BlobUploadInfo(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return writer.Size(), nil -} - -// FinishBlobUpload finalizes the blob upload and moves blob the repository. -func (is *ObjectStorage) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { - if err := dstDigest.Validate(); err != nil { - return err - } - - src := is.BlobUploadPath(repo, uuid) - - // complete multiUploadPart - fileWriter, err := is.store.Writer(context.Background(), src, true) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrBadBlobDigest - } - - if err := fileWriter.Commit(); err != nil { - is.log.Error().Err(err).Msg("failed to commit file") - - return err - } - - if err := fileWriter.Close(); err != nil { - is.log.Error().Err(err).Msg("failed to close file") - - return err - } - - fileReader, err := is.store.Reader(context.Background(), src, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open file") - - return zerr.ErrUploadNotFound - } - - defer fileReader.Close() - - srcDigest, err := godigest.FromReader(fileReader) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrBadBlobDigest - } - - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return zerr.ErrBadBlobDigest - } - - dst := is.BlobPath(repo, dstDigest) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return err - } - } else { - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return err - } - } - - return nil -} - -// FullBlobUpload handles a full blob upload, and no partial session is created. -func (is *ObjectStorage) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest) (string, int64, error) { - if err := dstDigest.Validate(); err != nil { - return "", -1, err - } - - if err := is.InitRepo(repo); err != nil { - return "", -1, err - } - - u, err := guuid.NewV4() - if err != nil { - return "", -1, err - } - - uuid := u.String() - src := is.BlobUploadPath(repo, uuid) - digester := sha256.New() - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return "", -1, err - } - - nbytes, err := writeFile(is.store, src, buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to write blob") - - return "", -1, err - } - - _, err = digester.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("digester failed to write") - - return "", -1, err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return "", -1, zerr.ErrBadBlobDigest - } - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return "", -1, err - } - } else { - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return "", -1, err - } - } - - return uuid, int64(nbytes), nil -} - -func (is *ObjectStorage) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { -retry: - is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") - - dstRecord, err := is.cache.GetBlob(dstDigest) - if err := inject.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") - - return err - } - - if dstRecord == "" { - // cache record doesn't exist, so first disk and cache entry for this digest - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - // move the blob from uploads to final dest - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") - - return err - } - - is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") - } else { - // cache record exists, but due to GC and upgrades from older versions, - // disk content and cache records may go out of sync - _, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - // the actual blob on disk may have been removed by GC, so sync the cache - err := is.cache.DeleteBlob(dstDigest, dstRecord) - if err = inject.Error(err); err != nil { - //nolint:lll - is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") - - return err - } - - goto retry - } - - fileInfo, err := is.store.Stat(context.Background(), dst) - if err != nil && !errors.As(err, &driver.PathNotFoundError{}) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - - return err - } - - // prevent overwrite original blob - if fileInfo == nil && dstRecord != dst { - // put empty file so that we are compliant with oci layout, this will act as a deduped blob - err = is.store.PutContent(context.Background(), dst, []byte{}) - if err != nil { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to write empty file") - - return err - } - - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - } - - // remove temp blobupload - if err := is.store.Delete(context.Background(), src); err != nil { - is.log.Error().Err(err).Str("src", src).Msg("dedupe: unable to remove blob") - - return err - } - - is.log.Debug().Str("src", src).Msg("dedupe: remove") - } - - return nil -} - -func (is *ObjectStorage) RunGCRepo(repo string) error { - return nil -} - -func (is *ObjectStorage) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { -} - -// DeleteBlobUpload deletes an existing blob upload that is currently in progress. -func (is *ObjectStorage) DeleteBlobUpload(repo, uuid string) error { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return zerr.ErrUploadNotFound - } - - return err - } - - defer writer.Close() - - if err := writer.Cancel(); err != nil { - is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") - - return err - } - - return nil -} - -// BlobPath returns the repository path of a blob. -func (is *ObjectStorage) BlobPath(repo string, digest godigest.Digest) string { - return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) -} - -/* - CheckBlob verifies a blob and returns true if the blob is correct - -If the blob is not found but it's found in cache then it will be copied over. -*/ -func (is *ObjectStorage) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - } else { - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - } - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err == nil && binfo.Size() > 0 { - is.log.Debug().Str("blob path", blobPath).Msg("blob path found") - - return true, binfo.Size(), nil - } - // otherwise is a 'deduped' blob (empty file) - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return false, -1, zerr.ErrBlobNotFound - } - - blobSize, err := is.copyBlob(repo, blobPath, dstRecord) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - // put deduped blob in cache - if err := is.cache.PutBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") - - return false, -1, err - } - - return true, blobSize, nil -} - -// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. -func (is *ObjectStorage) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err == nil && binfo.Size() > 0 { - is.log.Debug().Str("blob path", blobPath).Msg("blob path found") - - return true, binfo.Size(), nil - } - - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return false, -1, zerr.ErrBlobNotFound - } - - // then it's a 'deduped' blob - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return false, -1, zerr.ErrBlobNotFound - } - - binfo, err = is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return false, -1, zerr.ErrBlobNotFound - } - - return true, binfo.Size(), nil -} - -func (is *ObjectStorage) checkCacheBlob(digest godigest.Digest) (string, error) { - if err := digest.Validate(); err != nil { - return "", err - } - - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - return "", zerr.ErrBlobNotFound - } - - dstRecord, err := is.cache.GetBlob(digest) - if err != nil { - return "", err - } - - if _, err := is.store.Stat(context.Background(), dstRecord); err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). - Msg("unable to remove blob path from cache") - - return "", err - } - - return "", zerr.ErrBlobNotFound - } - - is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") - - return dstRecord, nil -} - -func (is *ObjectStorage) copyBlob(repo string, blobPath, dstRecord string) (int64, error) { - if err := is.initRepo(repo); err != nil { - is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") - - return -1, err - } - - if err := is.store.PutContent(context.Background(), blobPath, []byte{}); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to link") - - return -1, zerr.ErrBlobNotFound - } - - // return original blob with content instead of the deduped one (blobPath) - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err == nil { - return binfo.Size(), nil - } - - return -1, zerr.ErrBlobNotFound -} - -// blobStream is using to serve blob range requests. -type blobStream struct { - reader io.Reader - closer io.Closer -} - -func NewBlobStream(readCloser io.ReadCloser, from, to int64) (io.ReadCloser, error) { - return &blobStream{reader: io.LimitReader(readCloser, to-from+1), closer: readCloser}, nil -} - -func (bs *blobStream) Read(buf []byte) (int, error) { - return bs.reader.Read(buf) -} - -func (bs *blobStream) Close() error { - return bs.closer.Close() -} - -// GetBlobPartial returns a partial stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ObjectStorage) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, -) (io.ReadCloser, int64, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - end := to - - if to < 0 || to >= binfo.Size() { - end = binfo.Size() - 1 - } - - blobHandle, err := is.store.Reader(context.Background(), blobPath, from) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, -1, err - } - - blobReadCloser, err := NewBlobStream(blobHandle, from, end) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") - - return nil, -1, -1, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - defer blobReadCloser.Close() - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - end := to - - if to < 0 || to >= binfo.Size() { - end = binfo.Size() - 1 - } - - blobHandle, err := is.store.Reader(context.Background(), dstRecord, from) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, -1, -1, err - } - - blobReadCloser, err := NewBlobStream(blobHandle, from, end) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") - - return nil, -1, -1, err - } - - return blobReadCloser, end - from + 1, binfo.Size(), nil - } - - // The caller function is responsible for calling Close() - return blobReadCloser, end - from + 1, binfo.Size(), nil -} - -// GetBlob returns a stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ObjectStorage) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := is.store.Reader(context.Background(), blobPath, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, -1, zerr.ErrBlobNotFound - } - - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := is.store.Reader(context.Background(), dstRecord, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, -1, err - } - - return blobReadCloser, binfo.Size(), nil - } - - // The caller function is responsible for calling Close() - return blobReadCloser, binfo.Size(), nil -} - -// GetBlobContent returns blob contents, the caller function SHOULD lock from outside. -func (is *ObjectStorage) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { - if err := digest.Validate(); err != nil { - return []byte{}, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return []byte{}, zerr.ErrBlobNotFound - } - - blobBuf, err := is.store.GetContent(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, zerr.ErrBlobNotFound - } - - blobBuf, err := is.store.GetContent(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, err - } - - return blobBuf, nil - } - - return blobBuf, nil -} - -func (is *ObjectStorage) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, -) (ispec.Index, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) -} - -func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, -) ([]artifactspec.Descriptor, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) -} - -// GetIndexContent returns index.json contents, the caller function SHOULD lock from outside. -func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) { - dir := path.Join(is.rootDir, repo) - - buf, err := is.store.GetContent(context.Background(), path.Join(dir, "index.json")) - if err != nil { - if errors.Is(err, driver.PathNotFoundError{}) { - is.log.Error().Err(err).Str("dir", dir).Msg("index.json doesn't exist") - - return []byte{}, zerr.ErrRepoNotFound - } - - is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") - - return []byte{}, err - } - - return buf, nil -} - -// DeleteBlob removes the blob from the repository. -func (is *ObjectStorage) DeleteBlob(repo string, digest godigest.Digest) error { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return err - } - - blobPath := is.BlobPath(repo, digest) - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return zerr.ErrBlobNotFound - } - - // first check if this blob is not currently in use - if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { - return zerr.ErrBlobReferenced - } - - if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - dstRecord, err := is.cache.GetBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") - - return err - } - - // remove cache entry and move blob contents to the next candidate if there is any - if ok := is.cache.HasBlob(digest, blobPath); ok { - if err := is.cache.DeleteBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("unable to remove blob path from cache") - - return err - } - } - - // if the deleted blob is one with content - if dstRecord == blobPath { - // get next candidate - dstRecord, err := is.cache.GetBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") - - return err - } - - // if we have a new candidate move the blob content to it - if dstRecord != "" { - if err := is.store.Move(context.Background(), blobPath, dstRecord); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil - } - } - } - - if err := is.store.Delete(context.Background(), blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil -} - -// Do not use for multipart upload, buf must not be empty. -// If you want to create an empty file use is.store.PutContent(). -func writeFile(store driver.StorageDriver, filepath string, buf []byte) (int, error) { - var n int - - if stwr, err := store.Writer(context.Background(), filepath, false); err == nil { - defer stwr.Close() - - if n, err = stwr.Write(buf); err != nil { - return -1, err - } - - if err := stwr.Commit(); err != nil { - return -1, err - } - } else { - return -1, err - } - - return n, nil -} - -func (is *ObjectStorage) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - var duplicateBlobs []string - - var digest godigest.Digest - - err := is.store.Walk(context.Background(), dir, func(fileInfo driver.FileInfo) error { - if fileInfo.IsDir() { - return nil - } - - blobDigest := godigest.NewDigestFromEncoded("sha256", path.Base(fileInfo.Path())) - if err := blobDigest.Validate(); err != nil { - return nil //nolint:nilerr // ignore files which are not blobs - } - - if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { - digest = blobDigest - } - - if blobDigest == digest { - duplicateBlobs = append(duplicateBlobs, fileInfo.Path()) - } - - return nil - }) - - // if the root directory is not yet created - var perr driver.PathNotFoundError - - if errors.As(err, &perr) { - return digest, duplicateBlobs, nil - } - - return digest, duplicateBlobs, err -} - -func (is *ObjectStorage) getOriginalBlobFromDisk(duplicateBlobs []string) (string, error) { - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return "", zerr.ErrBlobNotFound - } - - if binfo.Size() > 0 { - return blobPath, nil - } - } - - return "", zerr.ErrBlobNotFound -} - -func (is *ObjectStorage) getOriginalBlob(digest godigest.Digest, duplicateBlobs []string) (string, error) { - var originalBlob string - - var err error - - originalBlob, err = is.checkCacheBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrBlobNotFound) && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find blob in cache") - - return originalBlob, err - } - - // if we still don't have, search it - if originalBlob == "" { - is.log.Warn().Msg("rebuild dedupe: failed to find blob in cache, searching it in s3...") - // a rebuild dedupe was attempted in the past - // get original blob, should be found otherwise exit with error - - originalBlob, err = is.getOriginalBlobFromDisk(duplicateBlobs) - if err != nil { - return originalBlob, err - } - } - - is.log.Info().Str("originalBlob", originalBlob).Msg("rebuild dedupe: found original blob") - - return originalBlob, nil -} - -func (is *ObjectStorage) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") - - return zerr.ErrDedupeRebuild - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") - - var originalBlob string - - // rebuild from dedupe false to true - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - if binfo.Size() == 0 { - is.log.Warn().Msg("rebuild dedupe: found file without content, trying to find the original blob") - // a rebuild dedupe was attempted in the past - // get original blob, should be found otherwise exit with error - if originalBlob == "" { - originalBlob, err = is.getOriginalBlob(digest, duplicateBlobs) - if err != nil { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") - - return zerr.ErrDedupeRebuild - } - - // cache original blob - if ok := is.cache.HasBlob(digest, originalBlob); !ok { - if err := is.cache.PutBlob(digest, originalBlob); err != nil { - return err - } - } - } - - // cache dedupe blob - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - } else { - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - - // if we have an original blob cached then we can safely dedupe the rest of them - if originalBlob != "" { - if err := is.store.PutContent(context.Background(), blobPath, []byte{}); err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: unable to dedupe blob") - - return err - } - } - - // mark blob as preserved - originalBlob = blobPath - } - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") - - return nil -} - -func (is *ObjectStorage) restoreDedupedBlobs(digest godigest.Digest, duplicateBlobs []string) error { - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: restoring deduped blobs for digest") - - // first we need to find the original blob, either in cache or by checking each blob size - originalBlob, err := is.getOriginalBlob(digest, duplicateBlobs) - if err != nil { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") - - return zerr.ErrDedupeRebuild - } - - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // if we find a deduped blob, then copy original blob content to deduped one - if binfo.Size() == 0 { - // move content from original blob to deduped one - buf, err := is.store.GetContent(context.Background(), originalBlob) - if err != nil { - is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to get original blob content") - - return err - } - - _, err = writeFile(is.store, blobPath, buf) - if err != nil { - return err - } - } - } - - is.log.Info().Str("digest", digest.String()). - Msg("rebuild dedupe: restoring deduped blobs for digest finished successfully") - - return nil -} - -func (is *ObjectStorage) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if dedupe { - return is.dedupeBlobs(digest, duplicateBlobs) - } - - return is.restoreDedupedBlobs(digest, duplicateBlobs) -} - -func (is *ObjectStorage) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { - generator := &common.DedupeTaskGenerator{ - ImgStore: is, - Dedupe: is.dedupe, - Log: is.log, - } - - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) + return imagestore.NewImageStore( + rootDir, + cacheDir, + gc, + gcReferrers, + gcDelay, + untaggedImageRetentionDelay, + dedupe, + commit, + log, + metrics, + linter, + New(store), + cacheDriver, + ) } diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 9430b2cc62..6a1036a3da 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -77,15 +77,17 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive var cacheDriver cache.Cache // from pkg/cli/root.go/applyDefaultValues, s3 magic - if _, err := os.Stat(path.Join(cacheDir, "s3_cache.db")); dedupe || (!dedupe && err == nil) { + if _, err := os.Stat(path.Join(cacheDir, + storageConstants.BoltdbName+storageConstants.DBExtensionName)); dedupe || (!dedupe && err == nil) { cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) } - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, - dedupe, false, log, metrics, nil, store, cacheDriver, + + il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver, ) return il @@ -97,8 +99,8 @@ func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.St log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := s3.NewImageStore(rootDir, "", false, storageConstants.DefaultGCDelay, - dedupe, false, log, metrics, nil, store, cacheDriver, + il := s3.NewImageStore(rootDir, "", true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver, ) return il @@ -150,17 +152,17 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( var err error // from pkg/cli/root.go/applyDefaultValues, s3 magic - s3CacheDBPath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + s3CacheDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err = os.Stat(s3CacheDBPath); dedupe || (!dedupe && err == nil) { cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) } - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, - dedupe, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver) return store, il, err } @@ -194,8 +196,8 @@ func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tabl panic(err) } - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, - dedupe, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver) return store, il, err } @@ -893,7 +895,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { _, _, err = imgStore.CheckBlob(testImage, digest) So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob(testImage, digest) + _, _, _, err = imgStore.StatBlob(testImage, digest) So(err, ShouldNotBeNil) }) @@ -1050,7 +1052,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { return &FileWriterMock{WriteFn: func(b []byte) (int, error) { return 0, errS3 - }}, nil + }}, errS3 }, }) _, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", io.NopCloser(strings.NewReader(""))) @@ -1091,7 +1093,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { WriteFn: func(b []byte) (int, error) { return 0, errS3 }, - }, nil + }, errS3 }, }) _, err := imgStore.PutBlobChunk(testImage, "uuid", 12, 100, io.NopCloser(strings.NewReader(""))) @@ -1280,7 +1282,7 @@ func TestS3Dedupe(t *testing.T) { So(checkBlobSize1, ShouldBeGreaterThan, 0) So(err, ShouldBeNil) - ok, checkBlobSize1, err = imgStore.StatBlob("dedupe1", digest) + ok, checkBlobSize1, _, err = imgStore.StatBlob("dedupe1", digest) So(ok, ShouldBeTrue) So(checkBlobSize1, ShouldBeGreaterThan, 0) So(err, ShouldBeNil) @@ -1466,12 +1468,12 @@ func TestS3Dedupe(t *testing.T) { Convey("Check backward compatibility - switch dedupe to false", func() { /* copy cache to the new storage with dedupe false (doing this because we already have a cache object holding the lock on cache db file) */ - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, false) @@ -3306,6 +3308,7 @@ func TestS3ManifestImageIndex(t *testing.T) { err = imgStore.DeleteImageManifest("index", "test:index1", false) So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") So(err, ShouldNotBeNil) @@ -3599,7 +3602,7 @@ func TestS3DedupeErr(t *testing.T) { imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{}) - err = os.Remove(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + err = os.Remove(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest") // trigger unable to insert blob record @@ -3640,8 +3643,9 @@ func TestS3DedupeErr(t *testing.T) { err := imgStore.DedupeBlob("", digest, "dst") So(err, ShouldBeNil) + // error will be triggered in driver.SameFile() err = imgStore.DedupeBlob("", digest, "dst2") - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) }) Convey("Test DedupeBlob - error on store.PutContent()", t, func(c C) { @@ -3776,12 +3780,12 @@ func TestS3DedupeErr(t *testing.T) { So(err, ShouldBeNil) // copy cache db to the new imagestore - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ @@ -3797,12 +3801,14 @@ func TestS3DedupeErr(t *testing.T) { _, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldNotBeNil) + // now it should move content from /repo1/dst1 to /repo2/dst2 _, err = imgStore.GetBlobContent("repo2", digest) - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) - _, _, err = imgStore.StatBlob("repo2", digest) - So(err, ShouldNotBeNil) + _, _, _, err = imgStore.StatBlob("repo2", digest) + So(err, ShouldBeNil) + // it errors out because of bad range, as mock store returns a driver.FileInfo with 0 size _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) So(err, ShouldNotBeNil) }) @@ -3822,12 +3828,12 @@ func TestS3DedupeErr(t *testing.T) { So(err, ShouldBeNil) // copy cache db to the new imagestore - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ @@ -3887,7 +3893,7 @@ func TestS3DedupeErr(t *testing.T) { _, err = imgStore.GetBlobContent("repo2", digest) So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob("repo2", digest) + _, _, _, err = imgStore.StatBlob("repo2", digest) So(err, ShouldNotBeNil) _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index 460695ec79..1253784b30 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -39,8 +39,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) Convey("Scrub only one repo", t, func(c C) { // initialize repo @@ -113,7 +113,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { // verify error message So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json") - index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 1) @@ -193,7 +193,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { err = os.Chmod(layerFile, 0x0200) So(err, ShouldBeNil) - index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 1) @@ -327,7 +327,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { So(actual, ShouldContainSubstring, "test 1.0 affected") So(actual, ShouldContainSubstring, "no such file or directory") - index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 2) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 7aab199d7b..4ecc7c094c 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -51,8 +51,9 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer if config.Storage.StorageDriver == nil { // false positive lint - linter does not implement Lint method //nolint:typecheck,contextcheck - defaultStore = local.NewImageStore(config.Storage.RootDirectory, - config.Storage.GC, config.Storage.GCDelay, + rootDir := config.Storage.RootDirectory + defaultStore = local.NewImageStore(rootDir, + config.Storage.GC, config.Storage.GCReferrers, config.Storage.GCDelay, config.Storage.UntaggedImageRetentionDelay, config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, CreateCacheDatabaseDriver(config.Storage.StorageConfig, log), ) @@ -80,7 +81,8 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer // false positive lint - linter does not implement Lint method //nolint: typecheck,contextcheck defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory, - config.Storage.GC, config.Storage.GCDelay, config.Storage.Dedupe, + config.Storage.GC, config.Storage.GCReferrers, config.Storage.GCDelay, + config.Storage.UntaggedImageRetentionDelay, config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, CreateCacheDatabaseDriver(config.Storage.StorageConfig, log)) } @@ -152,9 +154,13 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // add it to uniqueSubFiles // Create a new image store and assign it to imgStoreMap if isUnique { - imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(storageConfig.RootDirectory, - storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, - storageConfig.Commit, log, metrics, linter, CreateCacheDatabaseDriver(storageConfig, log)) + rootDir := storageConfig.RootDirectory + imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(rootDir, + storageConfig.GC, storageConfig.GCReferrers, storageConfig.GCDelay, + storageConfig.UntaggedImageRetentionDelay, storageConfig.Dedupe, + storageConfig.Commit, log, metrics, linter, + CreateCacheDatabaseDriver(storageConfig, log), + ) subImageStore[route] = imgStoreMap[storageConfig.RootDirectory] } @@ -183,8 +189,9 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // false positive lint - linter does not implement Lint method //nolint: typecheck subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, - storageConfig.GC, storageConfig.GCDelay, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, + storageConfig.GC, storageConfig.GCReferrers, storageConfig.GCDelay, + storageConfig.UntaggedImageRetentionDelay, storageConfig.Dedupe, + storageConfig.Commit, log, metrics, linter, store, CreateCacheDatabaseDriver(storageConfig, log), ) } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 53ba4b0d1e..ea25f053d3 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -22,6 +22,7 @@ import ( guuid "github.com/gofrs/uuid" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" @@ -33,6 +34,7 @@ import ( "zotregistry.io/zot/pkg/storage/cache" storageCommon "zotregistry.io/zot/pkg/storage/common" storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/storage/s3" storageTypes "zotregistry.io/zot/pkg/storage/types" @@ -52,7 +54,9 @@ func skipIt(t *testing.T) { } } -func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, storageTypes.ImageStore, error) { +func createObjectsStore(rootDir string, cacheDir string, gcDelay time.Duration, imageRetention time.Duration) ( + driver.StorageDriver, storageTypes.ImageStore, error, +) { bucket := "zot-storage-test" endpoint := os.Getenv("S3MOCK_ENDPOINT") storageDriverParams := map[string]interface{}{ @@ -85,11 +89,11 @@ func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, + il := s3.NewImageStore(rootDir, cacheDir, true, true, gcDelay, imageRetention, true, false, log, metrics, nil, store, cacheDriver, ) @@ -103,11 +107,11 @@ var testCases = []struct { }{ { testCaseName: "S3APIs", - storageType: "s3", + storageType: storageConstants.S3StorageDriverName, }, { testCaseName: "FileSystemAPIs", - storageType: "fs", + storageType: storageConstants.LocalStorageDriverName, }, } @@ -116,7 +120,7 @@ func TestStorageAPIs(t *testing.T) { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { var imgStore storageTypes.ImageStore - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -128,7 +132,8 @@ func TestStorageAPIs(t *testing.T) { tdir := t.TempDir() var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(store, testDir) } else { dir := t.TempDir() @@ -140,8 +145,11 @@ func TestStorageAPIs(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) } Convey("Repo layout", t, func(c C) { @@ -166,15 +174,15 @@ func TestStorageAPIs(t *testing.T) { }) Convey("Validate repo", func() { - v, err := imgStore.ValidateRepo(repoName) + repos, err := imgStore.ValidateRepo(repoName) So(err, ShouldBeNil) - So(v, ShouldEqual, true) + So(repos, ShouldEqual, true) }) Convey("Get repos", func() { - v, err := imgStore.GetRepositories() + repos, err := imgStore.GetRepositories() So(err, ShouldBeNil) - So(v, ShouldNotBeEmpty) + So(repos, ShouldNotBeEmpty) }) Convey("Get image tags", func() { @@ -261,7 +269,7 @@ func TestStorageAPIs(t *testing.T) { _, _, err = imgStore.CheckBlob("test", digest) So(err, ShouldBeNil) - ok, _, err := imgStore.StatBlob("test", digest) + ok, _, _, err := imgStore.StatBlob("test", digest) So(ok, ShouldBeTrue) So(err, ShouldBeNil) @@ -385,6 +393,11 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldBeNil) So(len(tags), ShouldEqual, 2) + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos[0], ShouldEqual, "test") + // We deleted only one tag, make sure blob should not be removed. hasBlob, _, err = imgStore.CheckBlob("test", digest) So(err, ShouldBeNil) @@ -407,7 +420,7 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldNotBeNil) So(hasBlob, ShouldEqual, false) - hasBlob, _, err = imgStore.StatBlob("test", digest) + hasBlob, _, _, err = imgStore.StatBlob("test", digest) So(err, ShouldNotBeNil) So(hasBlob, ShouldEqual, false) @@ -464,6 +477,10 @@ func TestStorageAPIs(t *testing.T) { err = imgStore.FinishBlobUpload("test", "inexistent", buf, digest) So(err, ShouldNotBeNil) + // invalid digest + err = imgStore.FinishBlobUpload("test", "inexistent", buf, "sha256:invalid") + So(err, ShouldNotBeNil) + err = imgStore.FinishBlobUpload("test", bupload, buf, digest) So(err, ShouldBeNil) @@ -471,7 +488,7 @@ func TestStorageAPIs(t *testing.T) { So(ok, ShouldBeTrue) So(err, ShouldBeNil) - ok, _, err = imgStore.StatBlob("test", digest) + ok, _, _, err = imgStore.StatBlob("test", digest) So(ok, ShouldBeTrue) So(err, ShouldBeNil) @@ -502,7 +519,7 @@ func TestStorageAPIs(t *testing.T) { _, _, err = imgStore.CheckBlob("test", "inexistent") So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob("test", "inexistent") + _, _, _, err = imgStore.StatBlob("test", "inexistent") So(err, ShouldNotBeNil) }) @@ -564,7 +581,7 @@ func TestStorageAPIs(t *testing.T) { indexContent, err := imgStore.GetIndexContent("test") So(err, ShouldBeNil) - if testcase.storageType == "fs" { + if testcase.storageType == storageConstants.LocalStorageDriverName { err = os.Chmod(path.Join(imgStore.RootDir(), "test", "index.json"), 0o000) So(err, ShouldBeNil) _, err = imgStore.GetIndexContent("test") @@ -737,7 +754,7 @@ func TestMandatoryAnnotations(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -748,13 +765,16 @@ func TestMandatoryAnnotations(t *testing.T) { testDir = path.Join("/oci-repo-test", uuid.String()) tdir = t.TempDir() - store, _, _ = createObjectsStore(testDir, tdir) - imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, + store, _, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + driver := s3.New(store) + imgStore = imagestore.NewImageStore(testDir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, store, nil) + }, driver, nil) defer cleanupStorage(store, testDir) } else { @@ -764,12 +784,14 @@ func TestMandatoryAnnotations(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, cacheDriver) + }, driver, cacheDriver) } Convey("Setup manifest", t, func() { @@ -815,27 +837,31 @@ func TestMandatoryAnnotations(t *testing.T) { }) Convey("Error on mandatory annotations", func() { - if testcase.storageType == "s3" { - imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, + if testcase.storageType == storageConstants.S3StorageDriverName { + driver := s3.New(store) + imgStore = imagestore.NewImageStore(testDir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, - }, store, nil) + }, driver, nil) } else { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: tdir, Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, - }, cacheDriver) + }, driver, cacheDriver) } _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) @@ -857,7 +883,7 @@ func TestDeleteBlobsInUse(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -868,7 +894,8 @@ func TestDeleteBlobsInUse(t *testing.T) { testDir = path.Join("/oci-repo-test", uuid.String()) tdir = t.TempDir() - store, imgStore, _ = createObjectsStore(testDir, tdir) + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(store, testDir) } else { @@ -878,8 +905,10 @@ func TestDeleteBlobsInUse(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, + true, log, metrics, nil, driver, cacheDriver) } Convey("Setup manifest", t, func() { @@ -961,12 +990,12 @@ func TestDeleteBlobsInUse(t *testing.T) { So(err, ShouldBeNil) }) - if testcase.storageType != "s3" { + if testcase.storageType != storageConstants.S3StorageDriverName { Convey("get image manifest error", func() { err := os.Chmod(path.Join(imgStore.RootDir(), "repo", "blobs", "sha256", manifestDigest.Encoded()), 0o000) So(err, ShouldBeNil) - ok, _ := storageCommon.IsBlobReferenced(imgStore, "repo", unusedDigest, log.Logger) + ok, _ := storageCommon.IsBlobReferenced(imgStore, "repo", unusedDigest, log) So(ok, ShouldBeFalse) err = os.Chmod(path.Join(imgStore.RootDir(), "repo", "blobs", "sha256", manifestDigest.Encoded()), 0o755) @@ -1011,6 +1040,7 @@ func TestDeleteBlobsInUse(t *testing.T) { var cdigest godigest.Digest var cblob []byte + //nolint: dupl for i := 0; i < 4; i++ { // upload image config blob upload, err = imgStore.NewBlobUpload(repoName) @@ -1067,6 +1097,12 @@ func TestDeleteBlobsInUse(t *testing.T) { indexManifestDigest, _, err := imgStore.PutImageManifest(repoName, "index", ispec.MediaTypeImageIndex, indexContent) So(err, ShouldBeNil) + Convey("Try to delete manifest being referenced by image index", func() { + // modifying multi arch images should not be allowed + err := imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldEqual, zerr.ErrManifestReferenced) + }) + Convey("Try to delete blob currently in use", func() { // layer blob err := imgStore.DeleteBlob("test", bdgst1) @@ -1103,13 +1139,13 @@ func TestDeleteBlobsInUse(t *testing.T) { So(err, ShouldBeNil) }) - if testcase.storageType != "s3" { + if testcase.storageType != storageConstants.S3StorageDriverName { Convey("repo not found", func() { // delete repo err := os.RemoveAll(path.Join(imgStore.RootDir(), repoName)) So(err, ShouldBeNil) - ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log.Logger) + ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log) So(err, ShouldNotBeNil) So(ok, ShouldBeFalse) }) @@ -1118,7 +1154,7 @@ func TestDeleteBlobsInUse(t *testing.T) { err := os.Remove(path.Join(imgStore.RootDir(), repoName, "index.json")) So(err, ShouldBeNil) - ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log.Logger) + ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log) So(err, ShouldNotBeNil) So(ok, ShouldBeFalse) }) @@ -1127,7 +1163,7 @@ func TestDeleteBlobsInUse(t *testing.T) { err := os.Remove(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexManifestDigest.Encoded())) So(err, ShouldBeNil) - ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, unusedDigest, log.Logger) + ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, unusedDigest, log) So(err, ShouldNotBeNil) So(ok, ShouldBeFalse) }) @@ -1148,22 +1184,25 @@ func TestStorageHandler(t *testing.T) { var secondRootDir string var thirdRootDir string - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) var firstStorageDriver driver.StorageDriver var secondStorageDriver driver.StorageDriver var thirdStorageDriver driver.StorageDriver firstRootDir = "/util_test1" - firstStorageDriver, firstStore, _ = createObjectsStore(firstRootDir, t.TempDir()) + firstStorageDriver, firstStore, _ = createObjectsStore(firstRootDir, t.TempDir(), + storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(firstStorageDriver, firstRootDir) secondRootDir = "/util_test2" - secondStorageDriver, secondStore, _ = createObjectsStore(secondRootDir, t.TempDir()) + secondStorageDriver, secondStore, _ = createObjectsStore(secondRootDir, t.TempDir(), + storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(secondStorageDriver, secondRootDir) thirdRootDir = "/util_test3" - thirdStorageDriver, thirdStore, _ = createObjectsStore(thirdRootDir, t.TempDir()) + thirdStorageDriver, thirdStore, _ = createObjectsStore(thirdRootDir, t.TempDir(), + storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(thirdStorageDriver, thirdRootDir) } else { // Create temporary directory @@ -1175,15 +1214,17 @@ func TestStorageHandler(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) + driver := local.New(true) + // Create ImageStore - firstStore = local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + firstStore = imagestore.NewImageStore(firstRootDir, firstRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, driver, nil) - secondStore = local.NewImageStore(secondRootDir, false, - storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + secondStore = imagestore.NewImageStore(secondRootDir, secondRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, driver, nil) - thirdStore = local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + thirdStore = imagestore.NewImageStore(thirdRootDir, thirdRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, driver, nil) } Convey("Test storage handler", t, func() { @@ -1226,3 +1267,1708 @@ func TestRoutePrefix(t *testing.T) { So(routePrefix, ShouldEqual, "/a") }) } + +func TestGarbageCollectImageManifest(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + Convey("Repo layout", t, func(c C) { + Convey("Garbage collect with default/long delay", func() { + var imgStore storageTypes.ImageStore + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + repoName := "gc-long" + + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + bdigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: digest, + Size: int64(len(manifestBuf)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + err = imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + }) + + Convey("Garbage collect with short delay", func() { + var imgStore storageTypes.ImageStore + + gcDelay := 1 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, gcDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, + true, log, metrics, nil, driver, cacheDriver) + } + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + // sleep so orphan blob can be GC'ed + time.Sleep(5 * time.Second) + + // upload blob + upload, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data2") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + bdigest := godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(manifestBuf)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push artifact manifest pointing to artifact above + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: artifactDigest, + Size: int64(len(artifactManifestBuf)), + } + artifactManifest.ArtifactType = "application/forArtifact" //nolint: goconst + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push orphan artifact (missing subject) + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromBytes([]byte("miss")), + Size: int64(30), + } + artifactManifest.ArtifactType = "application/orphan" //nolint: goconst + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + orphanArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push orphan artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push oras manifest pointing to manifest + orasArtifactManifest := artifactspec.Manifest{} + orasArtifactManifest.ArtifactType = "signature-example" //nolint: goconst + orasArtifactManifest.Subject = &artifactspec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(manifestBuf)), + } + orasArtifactManifest.Blobs = []artifactspec.Descriptor{} + + orasArtifactManifestBuf, err := json.Marshal(orasArtifactManifest) + So(err, ShouldBeNil) + + orasDigest := godigest.FromBytes(orasArtifactManifestBuf) + + // push oras manifest + _, _, err = imgStore.PutImageManifest(repoName, orasDigest.Encoded(), + artifactspec.MediaTypeArtifactManifest, orasArtifactManifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // sleep so orphan blob can be GC'ed + time.Sleep(5 * time.Second) + + Convey("Garbage collect blobs after manifest is removed", func() { + err = imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check artifacts are gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + + Convey("Garbage collect - don't gc manifests/blobs which are referenced by another image", func() { + // upload same image with another tag + _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, tag, false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // orphan artifact should be deleted + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // check artifacts manifests + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + }) + }) + + Convey("Garbage collect with dedupe", func() { + // garbage-collect is repo-local and dedupe is global and they can interact in strange ways + var imgStore storageTypes.ImageStore + + gcDelay := 5 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, gcDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + // first upload an image to the first repo and wait for GC timeout + + repo1Name := "gc1" + + // upload blob + upload, err := imgStore.NewBlobUpload(repo1Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + bdigest := godigest.FromBytes(content) + tdigest := bdigest + + blob, err := imgStore.PutBlobChunk(repo1Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo1Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repo1Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repo1Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // sleep so past GC timeout + time.Sleep(10 * time.Second) + + hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // upload another image into a second repo with the same blob contents so dedupe is triggered + + repo2Name := "gc2" + + upload, err = imgStore.NewBlobUpload(repo2Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + buf = bytes.NewBuffer(content) + buflen = buf.Len() + + blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap = make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // immediately upload any other image to second repo which should invoke GC inline, but expect layers to persist + + upload, err = imgStore.NewBlobUpload(repo2Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data-more") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + bdigest = godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap = make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repo2Name) + So(err, ShouldBeNil) + + // original blob should exist + hasBlob, _, err = imgStore.CheckBlob(repo2Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + _, _, _, err = imgStore.GetImageManifest(repo2Name, digest.String()) + So(err, ShouldBeNil) + }) + }) + }) + } +} + +func TestGarbageCollectImageIndex(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + Convey("Repo layout", t, func(c C) { + Convey("Garbage collect with default/long delay", func() { + var imgStore storageTypes.ImageStore + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + repoName := "gc-long" + + bdgst, digest, indexDigest, indexSize := pushRandomImageIndex(imgStore, repoName) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: indexSize, + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest referencing index image + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: indexSize, + } + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + Convey("delete index manifest, layers should be persisted", func() { + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + }) + }) + + Convey("Garbage collect with short delay", func() { + var imgStore storageTypes.ImageStore + + gcDelay := 5 * time.Second + imageRetentionDelay := 5 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay, imageRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, gcDelay, + imageRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + bdgst, digest, indexDigest, indexSize := pushRandomImageIndex(imgStore, repoName) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + // push artifact manifest pointing to index + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: indexSize, + }, + ArtifactType: "application/forIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(content)), + } + artifactManifest.ArtifactType = "application/forManifestInIndex" + + artifactManifestIndexBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestIndexDigest := godigest.FromBytes(artifactManifestIndexBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestIndexDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestIndexBuf) + So(err, ShouldBeNil) + + // push artifact manifest pointing to artifact above + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: artifactDigest, + Size: int64(len(artifactManifestBuf)), + } + artifactManifest.ArtifactType = "application/forArtifact" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push orphan artifact (missing subject) + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromBytes([]byte("miss")), + Size: int64(30), + } + artifactManifest.ArtifactType = "application/orphan" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + orphanArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push orphan artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push oras manifest pointing to index image + orasArtifactManifest := artifactspec.Manifest{} + orasArtifactManifest.ArtifactType = "signature-example" + orasArtifactManifest.Subject = &artifactspec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: indexSize, + } + orasArtifactManifest.Blobs = []artifactspec.Descriptor{} + + orasArtifactManifestBuf, err := json.Marshal(orasArtifactManifest) + So(err, ShouldBeNil) + + orasDigest := godigest.FromBytes(orasArtifactManifestBuf) + + // push oras manifest + _, _, err = imgStore.PutImageManifest(repoName, orasDigest.Encoded(), + artifactspec.MediaTypeArtifactManifest, orasArtifactManifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + time.Sleep(5 * time.Second) + + Convey("delete inner referenced manifest", func() { + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, artifactDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + }) + + Convey("delete index manifest, references should not be persisted", func() { + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // isn't yet gced because manifests part of index are removed after gcReferrers, + // so the artifacts pointing to manifest which are part of index are not removed after a first gcRepo + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + // orphan blob + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check referrer is gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + // this will remove refferers of manifests part of image index + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldNotBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + }) + }) + }) + } +} + +func TestGarbageCollectChainedImageIndexes(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + Convey("Garbage collect with short delay", t, func() { + var imgStore storageTypes.ImageStore + + gcDelay := 5 * time.Second + imageRetentionDelay := 5 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay, imageRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, gcDelay, + imageRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + content = []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + for i := 0; i < 4; i++ { //nolint: dupl + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + + // for each manifest inside index, push an artifact + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(content)), + }, + ArtifactType: "application/forManifestInInnerIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + } + + // also add a new image index inside this one + var innerIndex ispec.Index + innerIndex.SchemaVersion = 2 + innerIndex.MediaType = ispec.MediaTypeImageIndex + + for i := 0; i < 3; i++ { //nolint: dupl + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + innerIndex.Manifests = append(innerIndex.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload inner index image + innerIndexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + innerIndexDigest := godigest.FromBytes(innerIndexContent) + So(innerIndexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, innerIndexDigest.String(), + ispec.MediaTypeImageIndex, innerIndexContent) + So(err, ShouldBeNil) + + // add inner index into root index + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: innerIndexDigest, + MediaType: ispec.MediaTypeImageIndex, + Size: int64(len(innerIndexContent)), + }) + + // push root index + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: int64(len(indexContent)), + }, + ArtifactType: "application/forIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(content)), + } + artifactManifest.ArtifactType = "application/forManifestInIndex" + + artifactManifestIndexBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestIndexDigest := godigest.FromBytes(artifactManifestIndexBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestIndexDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestIndexBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: innerIndexDigest, + Size: int64(len(innerIndexContent)), + } + artifactManifest.ArtifactType = "application/forInnerIndex" + + artifactManifestInnerIndexBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestInnerIndexDigest := godigest.FromBytes(artifactManifestInnerIndexBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestInnerIndexDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestInnerIndexBuf) + So(err, ShouldBeNil) + + // push artifact manifest pointing to artifact above + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: artifactDigest, + Size: int64(len(artifactManifestBuf)), + } + artifactManifest.ArtifactType = "application/forArtifact" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push orphan artifact (missing subject) + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromBytes([]byte("miss")), + Size: int64(30), + } + artifactManifest.ArtifactType = "application/orphan" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + orphanArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push orphan artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push oras manifest pointing to index image + orasArtifactManifest := artifactspec.Manifest{} + orasArtifactManifest.ArtifactType = "signature-example" + orasArtifactManifest.Subject = &artifactspec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: int64(len(indexContent)), + } + orasArtifactManifest.Blobs = []artifactspec.Descriptor{} + + orasArtifactManifestBuf, err := json.Marshal(orasArtifactManifest) + So(err, ShouldBeNil) + + orasDigest := godigest.FromBytes(orasArtifactManifestBuf) + + // push oras manifest + _, _, err = imgStore.PutImageManifest(repoName, orasDigest.Encoded(), + artifactspec.MediaTypeArtifactManifest, orasArtifactManifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + time.Sleep(5 * time.Second) + + Convey("delete inner referenced manifest", func() { + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, artifactDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + }) + + Convey("delete index manifest, references should not be persisted", func() { + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + // this will remove artifacts pointing to root index which was remove + // it will also remove inner index because now although its referenced in index.json it has no tag + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // isn't yet gced because manifests part of index are removed after gcReferrers, + // so the artifacts pointing to manifest which are part of index are not removed after a single gcRepo + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + // orphan blob + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check artifact is gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + // this will remove manifests referenced in inner index because even if they are referenced in index.json + // they do not have tags + // it will also remove referrers pointing to inner manifest + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check inner index artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestInnerIndexDigest.String()) + So(err, ShouldNotBeNil) + + // this will remove referrers pointing to manifests referenced in inner index + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldNotBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + }) + }) + } +} + +func pushRandomImageIndex(imgStore storageTypes.ImageStore, repoName string, +) (godigest.Digest, godigest.Digest, godigest.Digest, int64) { + content := []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + + for i := 0; i < 4; i++ { //nolint: dupl + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + return bdgst, digest, indexDigest, int64(len(indexContent)) +} diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 1d66d7d369..e2a5a18bd4 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -4,6 +4,7 @@ import ( "io" "time" + storagedriver "github.com/docker/distribution/registry/storage/driver" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" @@ -38,7 +39,7 @@ type ImageStore interface { //nolint:interfacebloat DeleteBlobUpload(repo, uuid string) error BlobPath(repo string, digest godigest.Digest) string CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) - StatBlob(repo string, digest godigest.Digest) (bool, int64, error) + StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, ) (io.ReadCloser, int64, int64, error) @@ -52,4 +53,22 @@ type ImageStore interface { //nolint:interfacebloat RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) + GetAllBlobs(repo string) ([]string, error) +} + +type Driver interface { //nolint:interfacebloat + Name() string + EnsureDir(path string) error + DirExists(path string) bool + Reader(path string, offset int64) (io.ReadCloser, error) + ReadFile(path string) ([]byte, error) + Delete(path string) error + Stat(path string) (storagedriver.FileInfo, error) + Writer(filepath string, append bool) (storagedriver.FileWriter, error) //nolint: predeclared + WriteFile(filepath string, content []byte) (int, error) + Walk(path string, f storagedriver.WalkFn) error + List(fullpath string) ([]string, error) + Move(sourcePath string, destPath string) error + SameFile(path1, path2 string) bool + Link(src, dest string) error } diff --git a/pkg/test/common.go b/pkg/test/common.go index 5525101a6b..5e99f85862 100644 --- a/pkg/test/common.go +++ b/pkg/test/common.go @@ -2068,7 +2068,7 @@ func GetDefaultLayersBlobs() [][]byte { } func GetDefaultImageStore(rootDir string, log zLog.Logger) stypes.ImageStore { - return local.NewImageStore(rootDir, false, time.Hour, false, false, log, + return local.NewImageStore(rootDir, false, false, time.Hour, time.Hour, false, false, log, monitoring.NewMetricsServer(false, log), mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore stypes.ImageStore) (bool, error) { diff --git a/pkg/test/mocks/cache_mock.go b/pkg/test/mocks/cache_mock.go index 1d4e95e8f8..ef8dbf16b6 100644 --- a/pkg/test/mocks/cache_mock.go +++ b/pkg/test/mocks/cache_mock.go @@ -17,6 +17,16 @@ type CacheMock struct { // Delete a blob from the cachedb. DeleteBlobFn func(digest godigest.Digest, path string) error + + UsesRelativePathsFn func() bool +} + +func (cacheMock CacheMock) UsesRelativePaths() bool { + if cacheMock.UsesRelativePathsFn != nil { + return cacheMock.UsesRelativePaths() + } + + return false } func (cacheMock CacheMock) Name() string { diff --git a/pkg/test/mocks/image_store_mock.go b/pkg/test/mocks/image_store_mock.go index 63d05e85eb..7736d9464a 100644 --- a/pkg/test/mocks/image_store_mock.go +++ b/pkg/test/mocks/image_store_mock.go @@ -35,7 +35,7 @@ type MockedImageStore struct { DeleteBlobUploadFn func(repo string, uuid string) error BlobPathFn func(repo string, digest godigest.Digest) string CheckBlobFn func(repo string, digest godigest.Digest) (bool, int64, error) - StatBlobFn func(repo string, digest godigest.Digest) (bool, int64, error) + StatBlobFn func(repo string, digest godigest.Digest) (bool, int64, time.Time, error) GetBlobPartialFn func(repo string, digest godigest.Digest, mediaType string, from, to int64, ) (io.ReadCloser, int64, int64, error) GetBlobFn func(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) @@ -51,6 +51,7 @@ type MockedImageStore struct { RunDedupeBlobsFn func(interval time.Duration, sch *scheduler.Scheduler) RunDedupeForDigestFn func(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPathsFn func(lastDigests []godigest.Digest) (godigest.Digest, []string, error) + GetAllBlobsFn func(repo string) ([]string, error) } func (is MockedImageStore) Lock(t *time.Time) { @@ -142,6 +143,14 @@ func (is MockedImageStore) GetImageTags(name string) ([]string, error) { return []string{}, nil } +func (is MockedImageStore) GetAllBlobs(repo string) ([]string, error) { + if is.GetAllBlobsFn != nil { + return is.GetAllBlobsFn(repo) + } + + return []string{}, nil +} + func (is MockedImageStore) DeleteImageManifest(name string, reference string, detectCollision bool) error { if is.DeleteImageManifestFn != nil { return is.DeleteImageManifestFn(name, reference, detectCollision) @@ -252,12 +261,12 @@ func (is MockedImageStore) CheckBlob(repo string, digest godigest.Digest) (bool, return true, 0, nil } -func (is MockedImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { +func (is MockedImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) { if is.StatBlobFn != nil { return is.StatBlobFn(repo, digest) } - return true, 0, nil + return true, 0, time.Time{}, nil } func (is MockedImageStore) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, diff --git a/pkg/test/oci-layout/oci_layout_test.go b/pkg/test/oci-layout/oci_layout_test.go index 227be7b2d8..14c63fe1fa 100644 --- a/pkg/test/oci-layout/oci_layout_test.go +++ b/pkg/test/oci-layout/oci_layout_test.go @@ -346,7 +346,7 @@ func TestExtractImageDetails(t *testing.T) { Convey("extractImageDetails good workflow", t, func() { dir := t.TempDir() testLogger := log.NewLogger("debug", "") - imageStore := local.NewImageStore(dir, false, 0, false, false, + imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false, testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) storeController := storage.StoreController{ @@ -382,7 +382,7 @@ func TestExtractImageDetails(t *testing.T) { Convey("extractImageDetails bad ispec.ImageManifest", t, func() { dir := t.TempDir() testLogger := log.NewLogger("debug", "") - imageStore := local.NewImageStore(dir, false, 0, false, false, + imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false, testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) storeController := storage.StoreController{ @@ -402,7 +402,7 @@ func TestExtractImageDetails(t *testing.T) { Convey("extractImageDetails bad imageConfig", t, func() { dir := t.TempDir() testLogger := log.NewLogger("debug", "") - imageStore := local.NewImageStore(dir, false, 0, false, false, + imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false, testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) storeController := storage.StoreController{ diff --git a/test/blackbox/garbage_collect.bats b/test/blackbox/garbage_collect.bats new file mode 100644 index 0000000000..73d5f2a476 --- /dev/null +++ b/test/blackbox/garbage_collect.bats @@ -0,0 +1,159 @@ +load helpers_zot + +function verify_prerequisites { + if [ ! $(command -v curl) ]; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + return 0 +} + +function setup_file() { + # Verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + cat > ${zot_config_file}< signature.json + run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'signature/example' ./signature.json:application/json + [ "$status" -eq 0 ] + # attach sbom to image + echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > sbom.json + run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'sbom/example' ./sbom.json:application/json + [ "$status" -eq 0 ] + + # attach signature to index image + run oras attach --plain-http 127.0.0.1:8080/busybox:latest --image-spec v1.1-image --artifact-type 'signature/example' ./signature.json:application/json + [ "$status" -eq 0 ] + # attach sbom to index image + echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > sbom.json + run oras attach --plain-http 127.0.0.1:8080/busybox:latest --image-spec v1.1-image --artifact-type 'sbom/example' ./sbom.json:application/json + [ "$status" -eq 0 ] +} + +@test "push OCI artifact with regclient" { + run regctl registry set 127.0.0.1:8080 --tls disabled + [ "$status" -eq 0 ] + + run regctl artifact put --artifact-type application/vnd.example.artifact --subject 127.0.0.1:8080/golang:1.20 <