From 9aec28215b761d802392cb9a8dd24ec219269e74 Mon Sep 17 00:00:00 2001 From: Alexei Dodon Date: Tue, 19 Sep 2023 16:12:17 +0300 Subject: [PATCH] fix: reduce test run time Signed-off-by: Alexei Dodon --- .../{bug_report.yml => bug_report.yaml} | 0 ...ature_request.yml => feature_request.yaml} | 0 .github/actions/check-diskspace/action.yaml | 21 +++ .github/actions/setup-localstack/action.yaml | 20 +++ .../actions/teardown-localstack/action.yaml | 8 + .github/{dependabot.yml => dependabot.yaml} | 0 .../{branch-cov.yml => branch-cov.yaml} | 0 .github/workflows/build.yaml | 103 +++++++++++ .github/workflows/{cloc.yml => cloc.yaml} | 0 .github/workflows/cluster.yaml | 2 +- ...deql-analysis.yml => codeql-analysis.yaml} | 5 +- ...nary-size.yml => compare-binary-size.yaml} | 12 +- .github/workflows/{dco.yml => dco.yaml} | 1 - .github/workflows/golangci-lint.yaml | 5 +- ...action.yml => oci-conformance-action.yaml} | 0 .github/workflows/{ci-cd.yml => publish.yaml} | 167 +---------------- .../{scorecards.yml => scorecards.yaml} | 0 .github/workflows/test.yaml | 169 ++++++++++++++++++ .../workflows/{web-scan.yml => web-scan.yaml} | 0 Makefile | 68 ++++--- README.md | 2 +- build/Dockerfile-conformance | 6 +- pkg/cli/client/image_cmd_internal_test.go | 48 +---- 23 files changed, 388 insertions(+), 249 deletions(-) rename .github/ISSUE_TEMPLATE/{bug_report.yml => bug_report.yaml} (100%) rename .github/ISSUE_TEMPLATE/{feature_request.yml => feature_request.yaml} (100%) create mode 100644 .github/actions/check-diskspace/action.yaml create mode 100644 .github/actions/setup-localstack/action.yaml create mode 100644 .github/actions/teardown-localstack/action.yaml rename .github/{dependabot.yml => dependabot.yaml} (100%) rename .github/workflows/{branch-cov.yml => branch-cov.yaml} (100%) create mode 100644 .github/workflows/build.yaml rename .github/workflows/{cloc.yml => cloc.yaml} (100%) rename .github/workflows/{codeql-analysis.yml => codeql-analysis.yaml} (94%) rename .github/workflows/{compare-binary-size.yml => compare-binary-size.yaml} (82%) rename .github/workflows/{dco.yml => dco.yaml} (93%) rename .github/workflows/{oci-conformance-action.yml => oci-conformance-action.yaml} (100%) rename .github/workflows/{ci-cd.yml => publish.yaml} (62%) rename .github/workflows/{scorecards.yml => scorecards.yaml} (100%) create mode 100644 .github/workflows/test.yaml rename .github/workflows/{web-scan.yml => web-scan.yaml} (100%) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yaml similarity index 100% rename from .github/ISSUE_TEMPLATE/bug_report.yml rename to .github/ISSUE_TEMPLATE/bug_report.yaml diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yaml similarity index 100% rename from .github/ISSUE_TEMPLATE/feature_request.yml rename to .github/ISSUE_TEMPLATE/feature_request.yaml diff --git a/.github/actions/check-diskspace/action.yaml b/.github/actions/check-diskspace/action.yaml new file mode 100644 index 0000000000..cf6689d24c --- /dev/null +++ b/.github/actions/check-diskspace/action.yaml @@ -0,0 +1,21 @@ +name: 'Check disk space' +description: 'Show available disk space' +runs: + using: "composite" + steps: + - shell: bash + if: always() + run: | + cd $GITHUB_WORKSPACE + set -x + df -h + sudo ls -lRh /tmp/* || true + sudo du -sh /tmp || true + sudo du -sh /tmp/* || true + sudo find /tmp/ -size +5M | sudo xargs ls -lh + du -sh ./* || true + find ./ -size +5M | xargs ls -lh + sudo du -sh /var/ + sudo du -sh /var/lib/docker/ + du -sh /home/runner/work/ + set +x diff --git a/.github/actions/setup-localstack/action.yaml b/.github/actions/setup-localstack/action.yaml new file mode 100644 index 0000000000..d3d6cbe999 --- /dev/null +++ b/.github/actions/setup-localstack/action.yaml @@ -0,0 +1,20 @@ +name: 'Setup localstack service' +description: 'Download & run localstack container' +runs: + using: "composite" + steps: + - shell: bash + run: | + pip install localstack # Install LocalStack cli + docker pull localstack/localstack:2.2 # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + + aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" + aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake diff --git a/.github/actions/teardown-localstack/action.yaml b/.github/actions/teardown-localstack/action.yaml new file mode 100644 index 0000000000..5060f53f43 --- /dev/null +++ b/.github/actions/teardown-localstack/action.yaml @@ -0,0 +1,8 @@ +name: 'Stop localstack' +description: 'Stop localstack container' +runs: + using: "composite" + steps: + - shell: bash + if: always() + run: localstack stop diff --git a/.github/dependabot.yml b/.github/dependabot.yaml similarity index 100% rename from .github/dependabot.yml rename to .github/dependabot.yaml diff --git a/.github/workflows/branch-cov.yml b/.github/workflows/branch-cov.yaml similarity index 100% rename from .github/workflows/branch-cov.yml rename to .github/workflows/branch-cov.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000000..1c9d6576d0 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,103 @@ +on: + push: + branches: + - main + pull_request: + branches: + - main + release: + types: + - published +name: build + +permissions: read-all + +jobs: + build-arch: + name: Build ZOT multiarch + permissions: + contents: write + packages: write + runs-on: ubuntu-latest + strategy: + matrix: + os: [linux, darwin, freebsd] + arch: [amd64, arm64] + steps: + - name: Check out source code + uses: actions/checkout@v4 + - uses: ./.github/actions/clean-runner + - name: Install go + uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Cache go dependencies + id: cache-go-dependencies + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-mod- + - name: Cache go build output + id: cache-go-build + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + key: ${{ matrix.os }}-${{ matrix.arch }}-go-build-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ matrix.os }}-${{ matrix.arch }}-go-build- + - name: Install go dependencies + if: steps.cache-go-dependencies.outputs.cache-hit != 'true' + run: | + cd $GITHUB_WORKSPACE + go mod download + - name: Install other dependencies + run: | + cd $GITHUB_WORKSPACE + go install github.com/swaggo/swag/cmd/swag@v1.8.12 + sudo apt-get update + sudo apt-get install rpm + sudo apt-get install snapd + sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config + git clone https://github.com/containers/skopeo -b v1.12.0 $GITHUB_WORKSPACE/src/github.com/containers/skopeo + cd $GITHUB_WORKSPACE/src/github.com/containers/skopeo && \ + make bin/skopeo && \ + sudo cp bin/skopeo /usr/bin && \ + rm -rf $GITHUB_WORKSPACE/src/github.com/containers/skopeo + cd $GITHUB_WORKSPACE + curl -Lo notation.tar.gz https://github.com/notaryproject/notation/releases/download/v1.0.0-rc.4/notation_1.0.0-rc.4_linux_amd64.tar.gz + sudo tar xvzf notation.tar.gz -C /usr/bin notation + rm -f notation.tar.gz + - uses: ./.github/actions/check-diskspace + - name: Run build + timeout-minutes: 80 + run: | + echo "Building for $OS:$ARCH" + cd $GITHUB_WORKSPACE + make binary binary-minimal binary-debug cli bench exporter-minimal + env: + OS: ${{ matrix.os }} + ARCH: ${{ matrix.arch }} + - uses: ./.github/actions/check-diskspace + + - name: Generate GraphQL Introspection JSON on Release + if: github.event_name == 'release' && github.event.action == 'published' && matrix.os == 'linux' && matrix.arch == 'amd64' + run: | + bin/zot-linux-amd64 serve examples/config-search.json & + sleep 10 + curl -X POST -H "Content-Type: application/json" -d @.pkg/debug/githubWorkflows/introspection-query.json http://localhost:5000/v2/_zot/ext/search | jq > bin/zot-gql-introspection-result.json + pkill zot + + - if: github.event_name == 'release' && github.event.action == 'published' + name: Publish artifacts on releases + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: bin/z* + tag: ${{ github.ref }} + overwrite: true + file_glob: true diff --git a/.github/workflows/cloc.yml b/.github/workflows/cloc.yaml similarity index 100% rename from .github/workflows/cloc.yml rename to .github/workflows/cloc.yaml diff --git a/.github/workflows/cluster.yaml b/.github/workflows/cluster.yaml index 67f9be7244..22c5266dc1 100644 --- a/.github/workflows/cluster.yaml +++ b/.github/workflows/cluster.yaml @@ -14,7 +14,7 @@ permissions: read-all jobs: client-tools: name: Stateless zot with shared reliable storage - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest # services: # minio: # image: minio/minio:edge-cicd diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yaml similarity index 94% rename from .github/workflows/codeql-analysis.yml rename to .github/workflows/codeql-analysis.yaml index 511fc009fa..f3a612d257 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yaml @@ -25,7 +25,7 @@ permissions: read-all jobs: analyze: name: Analyze - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest permissions: actions: read contents: read @@ -56,9 +56,6 @@ jobs: cd $GITHUB_WORKSPACE go install github.com/swaggo/swag/cmd/swag@v1.8.12 go mod download - go install github.com/wadey/gocovmerge@latest - go get -u github.com/swaggo/swag/cmd/swag - go mod download # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/compare-binary-size.yml b/.github/workflows/compare-binary-size.yaml similarity index 82% rename from .github/workflows/compare-binary-size.yml rename to .github/workflows/compare-binary-size.yaml index 248f6463e8..652f08f41e 100644 --- a/.github/workflows/compare-binary-size.yml +++ b/.github/workflows/compare-binary-size.yaml @@ -36,11 +36,11 @@ jobs: [[ $BINSIZE -eq $BINSIZE_MAIN ]] && echo "zot-minimal binary size is not affected by PR" && exit 0 if [[ $BINSIZE -gt $BINSIZE_MAIN ]]; then \ - PERCENTAGE=$(echo "scale=2; (($BINSIZE-$BINSIZE_MAIN)*100)/$BINSIZE_MAIN" | bc); \ + PERCENTAGE=$(printf '%.*f\n' 2 $(echo "scale=2; (($BINSIZE-$BINSIZE_MAIN)*100)/$BINSIZE_MAIN" | bc)); \ echo "zot minimal binary increased by $PERCENTAGE% comparing with main"; \ if ((`bc <<< "$PERCENTAGE>=1.0"`)); then exit 1; fi; \ else \ - PERCENTAGE=$(echo "scale=2; (($BINSIZE_MAIN-$BINSIZE)*100)/$BINSIZE_MAIN" | bc); \ + PERCENTAGE=$(printf '%.*f\n' 2 $(echo "scale=2; (($BINSIZE_MAIN-$BINSIZE)*100)/$BINSIZE_MAIN" | bc)); \ echo "zot minimal binary decreased by $PERCENTAGE% comparing with main"; \ fi - if: always() @@ -61,11 +61,11 @@ jobs: [[ $BINSIZE -eq $BINSIZE_MAIN ]] && echo "zb binary size is not affected by PR" && exit 0 if [[ $BINSIZE -gt $BINSIZE_MAIN ]]; then \ - PERCENTAGE=$(echo "scale=2; (($BINSIZE-$BINSIZE_MAIN)*100)/$BINSIZE_MAIN" | bc); \ + PERCENTAGE=$(printf '%.*f\n' 2 $(echo "scale=2; (($BINSIZE-$BINSIZE_MAIN)*100)/$BINSIZE_MAIN" | bc)); \ echo "zb binary increased by $PERCENTAGE% comparing with main"; \ if ((`bc <<< "$PERCENTAGE>=1.0"`)); then exit 1; fi; \ else \ - PERCENTAGE=$(echo "scale=2; (($BINSIZE_MAIN-$BINSIZE)*100)/$BINSIZE_MAIN" | bc); \ + PERCENTAGE=$(printf '%.*f\n' 2 $(echo "scale=2; (($BINSIZE_MAIN-$BINSIZE)*100)/$BINSIZE_MAIN" | bc)); \ echo "zb binary decreased by $PERCENTAGE% comparing with main"; \ fi - if: always() @@ -86,10 +86,10 @@ jobs: [[ $BINSIZE -eq $BINSIZE_MAIN ]] && echo "zli binary size is not affected by PR" && exit 0 if [[ $BINSIZE -gt $BINSIZE_MAIN ]]; then \ - PERCENTAGE=$(echo "scale=2; (($BINSIZE-$BINSIZE_MAIN)*100)/$BINSIZE_MAIN" | bc); \ + PERCENTAGE=$(printf '%.*f\n' 2 $(echo "scale=2; (($BINSIZE-$BINSIZE_MAIN)*100)/$BINSIZE_MAIN" | bc)); \ echo "zli binary increased by $PERCENTAGE% comparing with main"; \ if ((`bc <<< "$PERCENTAGE>=1.0"`)); then exit 1; fi; \ else \ - PERCENTAGE=$(echo "scale=2; (($BINSIZE_MAIN-$BINSIZE)*100)/$BINSIZE_MAIN" | bc); \ + PERCENTAGE=$(printf '%.*f\n' 2 $(echo "scale=2; (($BINSIZE_MAIN-$BINSIZE)*100)/$BINSIZE_MAIN" | bc)); \ echo "zli binary decreased by $PERCENTAGE% comparing with main"; \ fi diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yaml similarity index 93% rename from .github/workflows/dco.yml rename to .github/workflows/dco.yaml index 268df50e05..faabf1d994 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yaml @@ -1,4 +1,3 @@ -# .github/workflows/dco.yml name: DCO on: pull_request: diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 78031a1cc0..631bcd7d65 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -20,7 +20,7 @@ jobs: cache: false go-version: '1.20' - uses: actions/checkout@v4 - - name: golangci-lint + - name: Run linter with GH action uses: golangci/golangci-lint-action@v3 with: # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version @@ -41,3 +41,6 @@ jobs: # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. # skip-build-cache: true + - name: Run linter from make target + run: | + make check diff --git a/.github/workflows/oci-conformance-action.yml b/.github/workflows/oci-conformance-action.yaml similarity index 100% rename from .github/workflows/oci-conformance-action.yml rename to .github/workflows/oci-conformance-action.yaml diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/publish.yaml similarity index 62% rename from .github/workflows/ci-cd.yml rename to .github/workflows/publish.yaml index 5979923be9..2e793e4055 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/publish.yaml @@ -1,178 +1,13 @@ on: - push: - branches: - - main - pull_request: - branches: - - main release: types: - published -name: build-test +name: publish permissions: read-all jobs: - build-test-arch: - name: Build and test ZOT - permissions: - contents: write - packages: write - runs-on: ubuntu-latest - strategy: - matrix: - os: [linux, darwin, freebsd] - arch: [amd64, arm64] - steps: - - name: Check out source code - uses: actions/checkout@v4 - - uses: ./.github/actions/clean-runner - - name: Install go - uses: actions/setup-go@v4 - with: - cache: false - go-version: 1.20.x - - name: Cache go dependencies - id: cache-go-dependencies - uses: actions/cache@v3 - with: - path: | - ~/go/pkg/mod - key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-go-mod- - - name: Cache go build output - id: cache-go-build - uses: actions/cache@v3 - with: - path: | - ~/.cache/go-build - key: ${{ matrix.os }}-${{ matrix.arch }}-go-build-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ matrix.os }}-${{ matrix.arch }}-go-build- - - name: Install go dependencies - if: steps.cache-go-dependencies.outputs.cache-hit != 'true' - run: | - cd $GITHUB_WORKSPACE - go mod download - - name: Install other dependencies - run: | - cd $GITHUB_WORKSPACE - go install github.com/swaggo/swag/cmd/swag@v1.8.12 - sudo apt-get update - sudo apt-get install rpm - sudo apt-get install snapd - sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config - git clone https://github.com/containers/skopeo -b v1.12.0 $GITHUB_WORKSPACE/src/github.com/containers/skopeo - cd $GITHUB_WORKSPACE/src/github.com/containers/skopeo && \ - make bin/skopeo && \ - sudo cp bin/skopeo /usr/bin && \ - rm -rf $GITHUB_WORKSPACE/src/github.com/containers/skopeo - cd $GITHUB_WORKSPACE - curl -Lo notation.tar.gz https://github.com/notaryproject/notation/releases/download/v1.0.0-rc.4/notation_1.0.0-rc.4_linux_amd64.tar.gz - sudo tar xvzf notation.tar.gz -C /usr/bin notation - rm -f notation.tar.gz - go install github.com/wadey/gocovmerge@latest - - - if: matrix.os == 'linux' && matrix.arch == 'amd64' - name: Setup localstack service - run: | - pip install localstack # Install LocalStack cli - docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image - localstack start -d # Start LocalStack in the background - - echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container - localstack wait -t 30 # to become ready before timing out - echo "Startup complete" - - aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" - aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 - env: - AWS_ACCESS_KEY_ID: fake - AWS_SECRET_ACCESS_KEY: fake - - - name: Check disk space before build - run: | - cd $GITHUB_WORKSPACE - set -x - df -h - sudo ls -lRh /tmp/* || true - sudo du -sh /tmp || true - sudo du -sh /tmp/* || true - sudo find /tmp/ -size +5M | sudo xargs ls -lh - du -sh ./* || true - find ./ -size +5M | xargs ls -lh - sudo du -sh /var/ - sudo du -sh /var/lib/docker/ - du -sh /home/runner/work/ - set +x - - - name: Run build and test - timeout-minutes: 80 - run: | - echo "Building for $OS:$ARCH" - cd $GITHUB_WORKSPACE - if [[ $OS == "linux" && $ARCH == "amd64" ]]; then - make - sudo env "PATH=$PATH" make privileged-test - else - make binary binary-minimal binary-debug cli bench exporter-minimal - fi - env: - S3MOCK_ENDPOINT: localhost:4566 - DYNAMODBMOCK_ENDPOINT: http://localhost:4566 - AWS_ACCESS_KEY_ID: fake - AWS_SECRET_ACCESS_KEY: fake - OS: ${{ matrix.os }} - ARCH: ${{ matrix.arch }} - - - if: matrix.os == 'linux' && matrix.arch == 'amd64' - name: Stop localstack - run: | - localstack stop - - - name: Check disk space after build - if: always() - run: | - cd $GITHUB_WORKSPACE - set -x - df -h - sudo ls -lRh /tmp/* || true - sudo du -sh /tmp || true - sudo du -sh /tmp/* || true - sudo find /tmp/ -size +5M | sudo xargs ls -lh - du -sh ./* || true - find ./ -size +5M | xargs ls -lh - sudo du -sh /var/ - sudo du -sh /var/lib/docker/ - du -sh /home/runner/work/ - set +x - - - name: Upload code coverage - uses: codecov/codecov-action@v3 - with: - token: ${{ secrets.CODECOV_TOKEN }} - - - name: Generate GraphQL Introspection JSON on Release - if: github.event_name == 'release' && github.event.action == 'published' && matrix.os == 'linux' && matrix.arch == 'amd64' - run: | - bin/zot-linux-amd64 serve examples/config-search.json & - sleep 10 - curl -X POST -H "Content-Type: application/json" -d @.pkg/debug/githubWorkflows/introspection-query.json http://localhost:5000/v2/_zot/ext/search | jq > bin/zot-gql-introspection-result.json - pkill zot - - - if: github.event_name == 'release' && github.event.action == 'published' - name: Publish artifacts on releases - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: bin/z* - tag: ${{ github.ref }} - overwrite: true - file_glob: true - push-image: - if: github.event_name == 'release' && github.event.action== 'published' name: Push OCI images to GitHub Packages runs-on: ubuntu-latest permissions: diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yaml similarity index 100% rename from .github/workflows/scorecards.yml rename to .github/workflows/scorecards.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..ef795122ca --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,169 @@ +name: "Running tests" +on: + push: + branches: + - main + pull_request: + branches: [main] + release: + types: + - published + +permissions: read-all + +jobs: + test-run-minimal: + name: Running zot without extensions tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/clean-runner + - name: Install go + uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Cache go dependencies + id: cache-go-dependencies + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-mod- + - name: Install go dependencies + if: steps.cache-go-dependencies.outputs.cache-hit != 'true' + run: | + cd $GITHUB_WORKSPACE + go mod download + - uses: ./.github/actions/setup-localstack + - uses: ./.github/actions/check-diskspace + - name: run zot minimal tests + run: | + cd $GITHUB_WORKSPACE + make test-minimal + env: + S3MOCK_ENDPOINT: localhost:4566 + DYNAMODBMOCK_ENDPOINT: http://localhost:4566 + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + - name: upload coverage + uses: actions/upload-artifact@v3 + with: + name: coverage-minimal + path: coverage-minimal.txt + - uses: ./.github/actions/check-diskspace + - uses: ./.github/actions/teardown-localstack + test-run-extensions: + name: Run zot with extensions tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Cache go dependencies + id: cache-go-dependencies + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-mod- + - name: Install go dependencies + if: steps.cache-go-dependencies.outputs.cache-hit != 'true' + run: | + cd $GITHUB_WORKSPACE + go mod download + - uses: ./.github/actions/setup-localstack + - uses: ./.github/actions/check-diskspace + - name: run zot extended tests + run: | + cd $GITHUB_WORKSPACE + make test-extended + env: + S3MOCK_ENDPOINT: localhost:4566 + DYNAMODBMOCK_ENDPOINT: http://localhost:4566 + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + - name: upload coverage + uses: actions/upload-artifact@v3 + with: + name: coverage-extended + path: coverage-extended.txt + - uses: ./.github/actions/check-diskspace + - uses: ./.github/actions/teardown-localstack + test-run-devmode: + name: Running privileged tests on Linux + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Cache go dependencies + id: cache-go-dependencies + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-mod- + - name: Install go dependencies + if: steps.cache-go-dependencies.outputs.cache-hit != 'true' + run: | + cd $GITHUB_WORKSPACE + go mod download + - uses: ./.github/actions/check-diskspace + - name: run zot development-mode unit tests (possibly using failure injection) + run: make test-devmode + - name: upload coverage + uses: actions/upload-artifact@v3 + with: + name: coverage-devmode + path: coverage-dev-minimal.txt + - name: upload coverage + uses: actions/upload-artifact@v3 + with: + name: coverage-devmode + path: coverage-dev-extended.txt + - name: run zot privileged tests + run: sudo env "PATH=$PATH" make privileged-test + - name: upload coverage + uses: actions/upload-artifact@v3 + with: + name: coverage-devmode + path: coverage-dev-needprivileges.txt + - uses: ./.github/actions/check-diskspace + test-coverage: + name: generate test coverage + runs-on: ubuntu-latest + if: always() + needs: [test-run-minimal,test-run-extensions,test-run-devmode] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: download all workflow coverage files + uses: actions/download-artifact@v3 + - name: merge code coverage + run: | + cd $GITHUB_WORKSPACE + cp coverage-minimal/* . + cp coverage-extended/* . + cp coverage-devmode/* . + make covhtml + - name: Upload code coverage + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} + diff --git a/.github/workflows/web-scan.yml b/.github/workflows/web-scan.yaml similarity index 100% rename from .github/workflows/web-scan.yml rename to .github/workflows/web-scan.yaml diff --git a/Makefile b/Makefile index 8bcc63ee59..ddd5449968 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ REGCLIENT_VERSION := v0.4.5 CRICTL := $(TOOLSDIR)/bin/crictl CRICTL_VERSION := v1.26.1 ACTION_VALIDATOR := $(TOOLSDIR)/bin/action-validator -ACTION_VALIDATOR_VERSION := v0.2.1 +ACTION_VALIDATOR_VERSION := v0.5.3 ZUI_VERSION := commit-b787273 SWAGGER_VERSION := v1.8.12 STACKER := $(TOOLSDIR)/bin/stacker @@ -125,24 +125,37 @@ exporter-minimal: EXTENSIONS= exporter-minimal: modcheck build-metadata env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zxp-$(OS)-$(ARCH) $(BUILDMODE_FLAGS) -tags containers_image_openpgp -v -trimpath ./cmd/zxp -.PHONY: test -test: $(if $(findstring ui,$(BUILD_LABELS)), ui) -test: check-skopeo $(TESTDATA) $(ORAS) - go test -failfast -tags $(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./... +.PHONY: test-prereq +test-prereq: check-skopeo $(TESTDATA) $(ORAS) + +.PHONY: test-extended +test-extended: $(if $(findstring ui,$(BUILD_LABELS)), ui) +test-extended: test-prereq + go test -failfast -tags $(BUILD_LABELS),containers_image_openpgp -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./... rm -rf /tmp/getter*; rm -rf /tmp/trivy* - go test -failfast -tags containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./... + +.PHONY: test-minimal +test-minimal: test-prereq + go test -failfast -tags containers_image_openpgp -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./... rm -rf /tmp/getter*; rm -rf /tmp/trivy* - # development-mode unit tests possibly using failure injection - go test -failfast -tags dev,$(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/test/... ./pkg/api/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject + +.PHONY: test-devmode +test-devmode: $(if $(findstring ui,$(BUILD_LABELS)), ui) +test-devmode: test-prereq + go test -failfast -tags dev,$(BUILD_LABELS),containers_image_openpgp -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/test/... ./pkg/api/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject rm -rf /tmp/getter*; rm -rf /tmp/trivy* - go test -failfast -tags dev,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/test/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject + go test -failfast -tags dev,containers_image_openpgp -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/test/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject rm -rf /tmp/getter*; rm -rf /tmp/trivy* - go test -failfast -tags stress,$(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m ./pkg/cli/server/stress_test.go + go test -failfast -tags stress,$(BUILD_LABELS),containers_image_openpgp -trimpath -race -timeout 15m ./pkg/cli/server/stress_test.go + +.PHONY: test +test: $(if $(findstring ui,$(BUILD_LABELS)), ui) +test: test-extended test-minimal test-devmode .PHONY: privileged-test privileged-test: $(if $(findstring ui,$(BUILD_LABELS)), ui) privileged-test: check-skopeo $(TESTDATA) - go test -failfast -tags needprivileges,$(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-needprivileges.txt -covermode=atomic ./pkg/storage/local/... ./pkg/cli/client/... -run ^TestElevatedPrivileges + go test -failfast -tags needprivileges,$(BUILD_LABELS),containers_image_openpgp -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-needprivileges.txt -covermode=atomic ./pkg/storage/local/... ./pkg/cli/client/... -run ^TestElevatedPrivileges $(TESTDATA): check-skopeo mkdir -p ${TESTDATA}; \ @@ -158,10 +171,13 @@ $(TESTDATA): check-skopeo .PHONY: run-bench run-bench: binary bench - bin/zot-$(OS)-$(ARCH) serve examples/config-bench.json & + bin/zot-$(OS)-$(ARCH) serve examples/config-bench.json & echo $$! > zot.PID sleep 5 bin/zb-$(OS)-$(ARCH) -c 10 -n 100 -o $(BENCH_OUTPUT) http://localhost:8080 - killall -r zot-* + @if [ -e zot.PID ]; then \ + kill -TERM $$(cat zot.PID) || true; \ + fi; \ + rm zot.PID .PHONY: check-skopeo check-skopeo: @@ -205,12 +221,12 @@ $(CRICTL): $(ACTION_VALIDATOR): mkdir -p $(TOOLSDIR)/bin - curl -Lo action-validator https://github.com/mpalmer/action-validator/releases/download/$(ACTION_VALIDATOR_VERSION)/action-validator_linux_amd64 + curl -Lo action-validator https://github.com/mpalmer/action-validator/releases/download/$(ACTION_VALIDATOR_VERSION)/action-validator_$(OS)_$(ARCH) mv action-validator $(TOOLSDIR)/bin/action-validator chmod +x $(TOOLSDIR)/bin/action-validator .PHONY: check-gh-actions -check-gh-actions: $(ACTION_VALIDATOR) +check-gh-actions: check-compatibility $(ACTION_VALIDATOR) for i in $$(ls .github/workflows/*); do $(ACTION_VALIDATOR) $$i; done .PHONY: covhtml @@ -227,13 +243,13 @@ $(GOLINTER): .PHONY: check check: $(if $(findstring ui,$(BUILD_LABELS)), ui) check: ./golangcilint.yaml $(GOLINTER) + $(GOLINTER) cache status mkdir -p pkg/extensions/build; touch pkg/extensions/build/.empty $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags containers_image_openpgp ./... - $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags $(BUILD_LABELS),containers_image_openpgp ./... - $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags $(BUILD_LABELS),containers_image_openpgp,debug ./... - $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags dev,containers_image_openpgp ./... - $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags dev,$(BUILD_LABELS),containers_image_openpgp ./... - $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags stress,$(BUILD_LABELS),containers_image_openpgp ./... + $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags $(BUILD_LABELS),containers_image_openpgp ./... + $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags debug ./pkg/debug/swagger/ ./pkg/debug/gqlplayground + $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags dev ./pkg/test/inject/ + $(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags stress ./pkg/cli/server/ rm pkg/extensions/build/.empty .PHONY: swagger @@ -472,3 +488,15 @@ check-linux: ifneq ($(shell go env GOOS),linux) $(error makefile target can be run only on linux) endif + +.PHONY: check-compatibility +check-compatibility: +ifeq ($(OS),freebsd) + $(error makefile target can't be run on freebsd) +endif +ifneq ($(OS),$(shell go env GOOS)) + $(error target can't be run on $(shell go env GOOS) as binary is compiled for $(OS)) +endif +ifneq ($(ARCH),$(shell go env GOARCH)) + $(error target can't be run on $(shell go env GOARCH) (binary is for $(ARCH))) +endif diff --git a/README.md b/README.md index 4a68e17edf..6ac5a7f862 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# zot [![build-test](https://github.com/project-zot/zot/actions/workflows/ci-cd.yml/badge.svg?branch=main)](https://github.com/project-zot/zot/actions/workflows/ci-cd.yml) [![codecov.io](http://codecov.io/github/project-zot/zot/coverage.svg?branch=main)](http://codecov.io/github/project-zot/zot?branch=main) [![Conformance Results](https://github.com/project-zot/zot/workflows/conformance/badge.svg)](https://github.com/project-zot/zot/actions?query=workflow%3Aconformance) [![CodeQL](https://github.com/project-zot/zot/workflows/CodeQL/badge.svg)](https://github.com/project-zot/zot/actions?query=workflow%3ACodeQL) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5425/badge)](https://bestpractices.coreinfrastructure.org/projects/5425) [![Go Reference](https://pkg.go.dev/badge/zotregistry.io/zot.svg)](https://pkg.go.dev/zotregistry.io/zot) +# zot [![build](https://github.com/project-zot/zot/actions/workflows/build.yaml/badge.svg?branch=main)](https://github.com/project-zot/zot/actions/workflows/build.yaml) [![test](https://github.com/project-zot/zot/actions/workflows/test.yaml/badge.svg?branch=main)](https://github.com/project-zot/zot/actions/workflows/test.yaml) [![codecov.io](http://codecov.io/github/project-zot/zot/coverage.svg?branch=main)](http://codecov.io/github/project-zot/zot?branch=main) [![Conformance Results](https://github.com/project-zot/zot/workflows/conformance/badge.svg)](https://github.com/project-zot/zot/actions?query=workflow%3Aconformance) [![CodeQL](https://github.com/project-zot/zot/workflows/CodeQL/badge.svg)](https://github.com/project-zot/zot/actions?query=workflow%3ACodeQL) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5425/badge)](https://bestpractices.coreinfrastructure.org/projects/5425) [![Go Reference](https://pkg.go.dev/badge/zotregistry.io/zot.svg)](https://pkg.go.dev/zotregistry.io/zot) **zot**: a production-ready vendor-neutral OCI image registry - images stored in [OCI image format](https://github.com/opencontainers/image-spec), [distribution specification](https://github.com/opencontainers/distribution-spec) on-the-wire, that's it! diff --git a/build/Dockerfile-conformance b/build/Dockerfile-conformance index e0e7f10bed..5378dcbcd9 100644 --- a/build/Dockerfile-conformance +++ b/build/Dockerfile-conformance @@ -15,7 +15,7 @@ http:\n\ storage:\n\ rootDirectory: /var/lib/registry\n\ gc: false\n\ - dedupe: false' > config.yml && cat config.yml + dedupe: false' > config.yaml && cat config.yaml # --- # Stage 2: Final image with nothing but certs, binary, and default config file @@ -23,8 +23,8 @@ storage:\n\ FROM gcr.io/distroless/base AS final COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /go/src/github.com/project-zot/zot/bin/zot-linux-amd64 /usr/bin/zot -COPY --from=builder /go/src/github.com/project-zot/zot/config.yml /etc/zot/config.yml +COPY --from=builder /go/src/github.com/project-zot/zot/config.yaml /etc/zot/config.yaml ENTRYPOINT ["/usr/bin/zot"] EXPOSE 5000 VOLUME ["/var/lib/registry"] -CMD ["serve", "/etc/zot/config.yml"] +CMD ["serve", "/etc/zot/config.yaml"] diff --git a/pkg/cli/client/image_cmd_internal_test.go b/pkg/cli/client/image_cmd_internal_test.go index cce35f795c..4a75e61364 100644 --- a/pkg/cli/client/image_cmd_internal_test.go +++ b/pkg/cli/client/image_cmd_internal_test.go @@ -681,8 +681,8 @@ func TestOutputFormat(t *testing.T) { ) So(err, ShouldBeNil) - Convey("Test yml", func() { - args := []string{"name", "dummyImageName", "--config", "imagetest", "-f", "yml"} + Convey("Test yaml", func() { + args := []string{"name", "dummyImageName", "--config", "imagetest", "-f", "yaml"} configPath := makeConfigFile( `{"configs":[{"_name":"imagetest",` + `"url":"https://test-url.com","showspinner":false}]}`, @@ -838,50 +838,6 @@ func TestOutputFormatGQL(t *testing.T) { So(err, ShouldBeNil) }) - Convey("Test yml", func() { - args := []string{"name", "repo7", "--config", "imagetest", "-f", "yml"} - configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url)) - defer os.Remove(configPath) - cmd := NewImageCommand(new(searchService)) - buff := bytes.NewBufferString("") - cmd.SetOut(buff) - cmd.SetErr(buff) - cmd.SetArgs(args) - err := cmd.Execute() - So(err, ShouldBeNil) - space := regexp.MustCompile(`\s+`) - str := space.ReplaceAllString(buff.String(), " ") - expectedStr := `--- reponame: repo7 tag: test:1.0 ` + - `digest: sha256:51e18f508fd7125b0831ff9a22ba74cd79f0b934e77661ff72cfb54896951a06 ` + - `mediatype: application/vnd.oci.image.manifest.v1+json manifests: - ` + - `digest: sha256:51e18f508fd7125b0831ff9a22ba74cd79f0b934e77661ff72cfb54896951a06 ` + - `configdigest: sha256:d14faead7d60053bad0d62e5ceb0031df28037d8c636d7911179b2f874ee004e ` + - `lastupdated: 2023-01-01T12:00:00Z size: "528" platform: os: linux arch: amd64 variant: "" ` + - `issigned: false downloadcount: 0 layers: - size: "15" ` + - `digest: sha256:b8781e8844f5b7bf6f2f8fa343de18ec471c3b278027355bc34c120585ff04f6 score: 0 ` + - `history: [] vulnerabilities: maxseverity: "" ` + - `count: 0 referrers: [] artifacttype: "" signatureinfo: [] ` + - `size: "528" downloadcount: 0 lastupdated: 2023-01-01T12:00:00Z description: "" ` + - `issigned: false licenses: "" labels: "" title: "" source: "" documentation: "" ` + - `authors: "" vendor: "" vulnerabilities: maxseverity: "" ` + - `count: 0 referrers: [] signatureinfo: [] ` + - `--- reponame: repo7 tag: test:2.0 ` + - `digest: sha256:51e18f508fd7125b0831ff9a22ba74cd79f0b934e77661ff72cfb54896951a06 ` + - `mediatype: application/vnd.oci.image.manifest.v1+json manifests: - ` + - `digest: sha256:51e18f508fd7125b0831ff9a22ba74cd79f0b934e77661ff72cfb54896951a06 ` + - `configdigest: sha256:d14faead7d60053bad0d62e5ceb0031df28037d8c636d7911179b2f874ee004e ` + - `lastupdated: 2023-01-01T12:00:00Z size: "528" platform: os: linux arch: amd64 variant: "" ` + - `issigned: false downloadcount: 0 layers: - size: "15" ` + - `digest: sha256:b8781e8844f5b7bf6f2f8fa343de18ec471c3b278027355bc34c120585ff04f6 score: 0 ` + - `history: [] vulnerabilities: maxseverity: "" ` + - `count: 0 referrers: [] artifacttype: "" signatureinfo: [] ` + - `size: "528" downloadcount: 0 lastupdated: 2023-01-01T12:00:00Z description: "" ` + - `issigned: false licenses: "" labels: "" title: "" source: "" documentation: "" ` + - `authors: "" vendor: "" vulnerabilities: maxseverity: "" count: 0 referrers: [] signatureinfo: []` - So(strings.TrimSpace(str), ShouldEqual, expectedStr) - So(err, ShouldBeNil) - }) - Convey("Test invalid", func() { args := []string{"name", "repo7", "--config", "imagetest", "-f", "random"} configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url))