refactor(storage): refactor storage into a single ImageStore

unified both local and s3 ImageStore logic into a single ImageStore added a new driver interface for common file/dirs manipulations to be implemented by different storage types refactor(gc): drop umoci dependency, implemented internal gc added retentionDelay config option that specifies the garbage collect delay for images without tags this will also clean manifests which are part of an index image (multiarch) that no longer exist. fix(dedupe): skip blobs under .sync/ directory if startup dedupe is running while also syncing is running ignore blobs under sync's temporary storage fix(storage): do not allow image indexes modifications when deleting a manifest verify that it is not part of a multiarch image and throw a MethodNotAllowed error to the client if it is. we don't want to modify multiarch images Signed-off-by: Petu Eusebiu <[email protected]>
project-zot · Aug 31, 2023 · 5bb6c1b · 5bb6c1b
1 parent 41bbb23
commit 5bb6c1b
Show file tree

Hide file tree

Showing 58 changed files with 6,065 additions and 4,755 deletions.
diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml
@@ -17,6 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
       - uses: actions/setup-go@v4
         with:
           cache: false
@@ -98,6 +99,9 @@ jobs:
       - name: Run annotations tests
         run: |
             make test-annotations
+      - name: Run garbage collect tests
+        run: |
+            make test-garbage-collect
       - name: Install localstack
         run: |
           pip install --upgrade pyopenssl
@@ -129,4 +133,4 @@ jobs:
           sudo du -sh /var/
           sudo du -sh /var/lib/docker/
           du -sh /home/runner/work/
-          set +x
+          set +x
diff --git a/.github/workflows/gc-stress-test.yaml b/.github/workflows/gc-stress-test.yaml
@@ -12,8 +12,8 @@ on:
 permissions: read-all
 
 jobs:
-  client-tools:
-    name: GC with short interval
+  gc-stress-local:
+    name: GC on filesystem with short interval
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -27,11 +27,51 @@ jobs:
         run: |
             make binary
             make bench
-            ./bin/zot-linux-amd64 serve examples/config-gc-bench.json &
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-local.json &
             sleep 10
             bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
 
             killall -r zot-*
 
             # clean zot storage
             sudo rm -rf /tmp/zot
+  gc-stress-s3:
+    name: GC on S3 with short interval
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
+      - uses: actions/setup-go@v4
+        with:
+          cache: false
+          go-version: 1.20.x
+      - name: Setup localstack service
+        run: |
+            pip install localstack                    # Install LocalStack cli
+            docker pull localstack/localstack:1.3     # Make sure to pull the latest version of the image
+            localstack start -d                       # Start LocalStack in the background
+            
+            echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
+            localstack wait -t 30                     # to become ready before timing out 
+            echo "Startup complete"
+            
+            aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
+            aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+      - name: Run zb
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -74,3 +74,45 @@ jobs:
       - name: Run sync harness
         run: |
             make test-sync-harness
+  gc-stress-s3:
+    name: GC on S3 with short interval
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/clean-runner
+      - uses: actions/setup-go@v4
+        with:
+          cache: false
+          go-version: 1.20.x
+      - name: Setup localstack service
+        run: |
+            pip install localstack                    # Install LocalStack cli
+            docker pull localstack/localstack:1.3     # Make sure to pull the latest version of the image
+            localstack start -d                       # Start LocalStack in the background
+            
+            echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
+            localstack wait -t 30                     # to become ready before timing out 
+            echo "Startup complete"
+            
+            aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
+            aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+      - name: Run zb
+        run: |
+            make binary
+            make bench
+            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json &
+            sleep 10
+            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
+
+            killall -r zot-*
+
+            # clean zot storage
+            sudo rm -rf /tmp/zot
+        env:
+          AWS_ACCESS_KEY_ID: fake
+          AWS_SECRET_ACCESS_KEY: fake
+
+
diff --git a/Makefile b/Makefile
@@ -357,6 +357,14 @@ test-push-pull: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(H
 test-push-pull-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
 
+.PHONY: test-garbage-collect
+test-garbage-collect: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
+	$(BATS) --trace --print-output-on-failure test/blackbox/garbage_collect.bats
+
+.PHONY: test-garbage-collect-verbose
+test-garbage-collect-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
+	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/garbage_collect.bats
+
 .PHONY: test-push-pull-running-dedupe
 test-push-pull-running-dedupe: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
 	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats

diff --git a/errors/errors.go b/errors/errors.go
@@ -58,6 +58,7 @@ var (
 	ErrBadBlob                        = errors.New("blob: bad blob")
 	ErrBadBlobDigest                  = errors.New("blob: bad blob digest")
 	ErrBlobReferenced                 = errors.New("blob: referenced by manifest")
+	ErrManifestReferenced             = errors.New("manifest: referenced by index image")
 	ErrUnknownCode                    = errors.New("error: unknown error code")
 	ErrBadCACert                      = errors.New("tls: invalid ca cert")
 	ErrBadUser                        = errors.New("auth: non-existent user")
@@ -155,4 +156,7 @@ var (
 	ErrGQLEndpointNotFound            = errors.New("cli: the server doesn't have a gql endpoint")
 	ErrGQLQueryNotSupported           = errors.New("cli: query is not supported or has different arguments")
 	ErrBadHTTPStatusCode              = errors.New("cli: the response doesn't contain the expected status code")
+	ErrFileAlreadyCancelled           = errors.New("storageDriver: file already cancelled")
+	ErrFileAlreadyClosed              = errors.New("storageDriver: file already closed")
+	ErrFileAlreadyCommitted           = errors.New("storageDriver: file already committed")
 )
diff --git a/examples/config-gc.json b/examples/config-gc.json
@@ -3,7 +3,10 @@
     "storage": {
         "rootDirectory": "/tmp/zot",
         "gc": true,
-        "gcDelay": "1s"
+        "gcReferrers": true,
+        "gcDelay": "2h",
+        "untaggedImageRetentionDelay": "4h",
+        "gcInterval": "1h"
     },
     "http": {
         "address": "127.0.0.1",

diff --git a/go.mod b/go.mod
@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/99designs/gqlgen v0.17.35
 	github.com/Masterminds/semver v1.5.0
-	github.com/apex/log v1.9.0
+	github.com/apex/log v1.9.0 // indirect
 	github.com/aquasecurity/trivy-db v0.0.0-20230703082116-dc52e83376ce
 	github.com/bmatcuk/doublestar/v4 v4.6.0
 	github.com/briandowns/spinner v1.23.0
@@ -23,7 +23,6 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/hashicorp/golang-lru/v2 v2.0.5
 	github.com/json-iterator/go v1.1.12
-	github.com/minio/sha256-simd v1.0.1
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba
 	github.com/olekukonko/tablewriter v0.0.5
@@ -367,7 +366,6 @@ require (
 	github.com/jtolds/gls v4.20.0+incompatible // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
 	github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
 	github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect
 	github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422 // indirect

diff --git a/go.sum b/go.sum
@@ -1129,8 +1129,6 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e
 github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
 github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
-github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU=
-github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 h1:BcxbplxjtczA1a6d3wYoa7a0WL3rq9DKBMGHeKyjEF0=
 github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
@@ -1257,8 +1255,6 @@ github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU=
 github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/migueleliasweb/go-github-mock v0.0.19 h1:z/88f6wPqZVFnE7s9DbwXMhCtmV/0FofNxc4M7FuSdU=
 github.com/migueleliasweb/go-github-mock v0.0.19/go.mod h1:dBoCB3W9NjzyABhoGkfI0iSlFpzulAXhI7M+9A4ONYI=
-github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM=
-github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
@@ -2063,7 +2059,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go
@@ -23,15 +23,17 @@ var (
 )
 
 type StorageConfig struct {
-	RootDirectory string
-	Dedupe        bool
-	RemoteCache   bool
-	GC            bool
-	Commit        bool
-	GCDelay       time.Duration
-	GCInterval    time.Duration
-	StorageDriver map[string]interface{} `mapstructure:",omitempty"`
-	CacheDriver   map[string]interface{} `mapstructure:",omitempty"`
+	RootDirectory               string
+	Dedupe                      bool
+	RemoteCache                 bool
+	GC                          bool
+	Commit                      bool
+	GCDelay                     time.Duration
+	GCInterval                  time.Duration
+	GCReferrers                 bool
+	UntaggedImageRetentionDelay time.Duration
+	StorageDriver               map[string]interface{} `mapstructure:",omitempty"`
+	CacheDriver                 map[string]interface{} `mapstructure:",omitempty"`
 }
 
 type TLSConfig struct {
@@ -188,8 +190,9 @@ func New() *Config {
 		BinaryType:      BinaryType,
 		Storage: GlobalStorageConfig{
 			StorageConfig: StorageConfig{
-				GC: true, GCDelay: storageConstants.DefaultGCDelay,
-				GCInterval: storageConstants.DefaultGCInterval, Dedupe: true,
+				GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay,
+				UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay,
+				GCInterval:                  storageConstants.DefaultGCInterval, Dedupe: true,
 			},
 		},
 		HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}},