From 059a1a29c234f27cf7f5eca5d130c42056d1fc3d Mon Sep 17 00:00:00 2001 From: Khaled Emara Date: Mon, 4 Nov 2024 16:48:22 +0200 Subject: [PATCH 1/2] feat(ci): enhance load testing (#11515) Signed-off-by: Khaled Emara --- .github/workflows/load-testing.yml | 300 ++++++++++++++---- Makefile | 17 +- .../standard-with-profiling/kyverno.yaml | 67 ++++ .../config/stress-with-profiling/kyverno.yaml | 19 -- test/load/k6/pull_request-matrix.json | 79 +++++ test/load/k6/release-matrix.json | 79 +++++ test/load/k6/reports-size-in-etcd.sh | 49 +++ test/load/k6/schedule-matrix.json | 79 +++++ 8 files changed, 610 insertions(+), 79 deletions(-) create mode 100644 scripts/config/standard-with-profiling/kyverno.yaml delete mode 100644 scripts/config/stress-with-profiling/kyverno.yaml create mode 100644 test/load/k6/pull_request-matrix.json create mode 100644 test/load/k6/release-matrix.json create mode 100755 test/load/k6/reports-size-in-etcd.sh create mode 100644 test/load/k6/schedule-matrix.json diff --git a/.github/workflows/load-testing.yml b/.github/workflows/load-testing.yml index 46bfcb1fc920..504f526f918e 100644 --- a/.github/workflows/load-testing.yml +++ b/.github/workflows/load-testing.yml @@ -1,23 +1,38 @@ -name: Baseline Load Tests +name: Load Tests permissions: {} on: + release: + types: [published] pull_request: branches: - "main" - "release*" + schedule: + - cron: "27 0 * * 0" concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + define-matrix: + runs-on: ubuntu-latest + outputs: + tests: ${{ steps.set-tests.outputs.tests }} + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Set Tests + id: set-tests + run: echo "tests=$(jq -c . < ./test/load/k6/${{ github.event_name }}-matrix.json)" >> $GITHUB_OUTPUT + prepare-images: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Setup caches uses: ./.github/actions/setup-caches timeout-minutes: 5 @@ -35,110 +50,277 @@ jobs: set -e VERSION=${{ github.ref_name }} make docker-save-image-all - name: upload images archive - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: name: kyverno.tar path: kyverno.tar retention-days: 1 if-no-files-found: error - load-test: + old-load-test: + if: github.event_name == 'pull_request' + needs: + - prepare-images + outputs: + p95: ${{ steps.extract-p95.outputs.p95 }} runs-on: ubuntu-latest permissions: packages: read strategy: fail-fast: false matrix: - k8s-version: - - name: v1.29 - version: v1.29.1 - kyverno-config: - - name: default - values: - - default-with-profiling - - name: stress - values: - - stress-with-profiling - test: - - kyverno-pss - - kyverno-mutate - k6-config: - - vus: 5 - iterations: 100 - - vus: 10 - iterations: 200 + k8s-version: [v1.31.0] + steps: + - name: Checkout kyverno/kyverno + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Checkout kyverno/load-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: kyverno/load-testing + path: load-testing + - name: Install Helm + id: helm + uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + - name: Create Kind cluster + uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0 + with: + node_image: kindest/node:${{ matrix.k8s-version }} + cluster_name: kind + config: ./scripts/config/kind/default.yaml + - name: Download kyverno images archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: kyverno.tar + - name: Load Kyverno images archive in Kind cluster + shell: bash + run: | + set -e + kind load image-archive kyverno.tar --name kind + - name: Install Kyverno + shell: bash + run: | + set -e + export HELM=${{ steps.helm.outputs.helm-path }} + export USE_CONFIG=default-with-profiling + $HELM repo add kyverno https://kyverno.github.io/kyverno/ + $HELM repo update + export INSTALL_VERSION=$($HELM search repo kyverno/kyverno -o json | jq -r '.[0].version') + export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=1 --set admissionController.resources.requests.cpu=100m --set admissionController.resources.limits.cpu=1500m --set admissionController.resources.requests.memory=128Mi --set admissionController.resources.limits.memory=384Mi' + make kind-install-kyverno-from-repo + - name: Wait for kyverno ready + uses: ./.github/actions/kyverno-wait-ready + - name: Install K6 + shell: bash + run: | + set -e + go install go.k6.io/xk6/cmd/xk6@latest + $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest + mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Run load tests using K6 + shell: bash + run: | + set -e + mkdir -p report + KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}') + curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=90" > report/cpu.pprof & + cd load-testing + ./k6/run.sh k6/tests/kyverno-pss.js -e SCENARIO=average --out dashboard=export=load-report.html + wait %1 || true + mv load-report.html ../report + - name: Extract P(95) + id: extract-p95 + shell: bash + run: | + set -e + echo "p95=$(grep http_req_duration load-testing/test-output.log | awk -F 'p\\(95\\)=' '{split($2,a,\"ms\"); print a[1]}')" >> $GITHUB_OUTPUT + echo $GITHUB_OUTPUT + - name: Debug failure + if: failure() + uses: ./.github/actions/kyverno-logs + + load-test: + if: github.event_name == 'pull_request' needs: - prepare-images - name: ${{ matrix.kyverno-config.name }} - ${{ matrix.test }} - ${{ matrix.k6-config.vus }} vus - ${{ matrix.k6-config.iterations }} iterations + - old-load-test + runs-on: ubuntu-latest + permissions: + packages: read + strategy: + fail-fast: false + matrix: + k8s-version: [v1.31.0] steps: - name: Checkout kyverno/kyverno - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Checkout kyverno/load-testing - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: kyverno/load-testing path: load-testing - - name: Install helm + - name: Install Helm id: helm - uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 + uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + - name: Create Kind cluster + uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0 with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Create kind cluster - uses: helm/kind-action@dda0770415bac9fc20092cacbc54aa298604d140 # v1.8.0 - with: - node_image: kindest/node:${{ matrix.k8s-version.version }} + node_image: kindest/node:${{ matrix.k8s-version }} cluster_name: kind config: ./scripts/config/kind/default.yaml - name: Download kyverno images archive - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: name: kyverno.tar - - name: Load kyverno images archive in kind cluster + - name: Load Kyverno images archive in Kind cluster shell: bash run: | set -e kind load image-archive kyverno.tar --name kind - - name: Install kyverno + - name: Install Kyverno shell: bash run: | set -e export HELM=${{ steps.helm.outputs.helm-path }} - export USE_CONFIG=${{ join(matrix.kyverno-config.values, ',') }} + export USE_CONFIG=default-with-profiling + export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=1 --set admissionController.resources.requests.cpu=100m --set admissionController.resources.limits.cpu=1500m --set admissionController.resources.requests.memory=128Mi --set admissionController.resources.limits.memory=384Mi' make kind-install-kyverno - bash load-testing/k8s/metrics-server/hack.sh - make kind-install-goldilocks - name: Wait for kyverno ready uses: ./.github/actions/kyverno-wait-ready + - name: Install K6 + shell: bash + run: | + set -e + go install go.k6.io/xk6/cmd/xk6@latest + $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest + mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH - name: Run load tests using K6 shell: bash run: | set -e + mkdir -p report KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}') - curl http://$KYVERNO_NODE_IP:30950/debug/pprof/heap > heap.pprof - curl "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=30" > cpu.pprof 2> curl.tmp & - cd load-testing/k6 - ./start.sh tests/${{ matrix.test }}.js ${{ matrix.k6-config.vus }} ${{ matrix.k6-config.iterations }} + curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=90" > report/cpu.pprof & + cd load-testing + ./k6/run.sh k6/tests/kyverno-pss.js -e SCENARIO=average --out dashboard=export=load-report.html wait %1 || true - # TODO: wait for VPA to stabilize and recommend - kubectl -n kyverno get vpa goldilocks-kyverno-admission-controller -o jsonpath='{.status.recommendation.containerRecommendations[*]}' - - name: Archive load test results - if: failure() - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + mv load-report.html ../report + - name: Compare P(95) + shell: bash + run: | + set -e + echo "Old P(95): ${{ needs.old-load-test.outputs.p95 }}" + OLD_NUM=${{ needs.old-load-test.outputs.p95 }} + NEW_NUM=$(grep http_req_duration load-testing/test-output.log | awk -F 'p\\(95\\)=' '{split($2,a,"ms"); print a[1]}') + echo "$OLD_NUM to $NEW_NUM" + if [ $(echo "$OLD_NUM < $NEW_NUM" | bc) -eq 1 ]; then + echo "P(95) increased from $OLD_NUM to $NEW_NUM" + exit 1 + fi + - name: Archive Report + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: - name: load-test-reports - path: load-testing/k6/${{ matrix.test }}.js-${{ matrix.k6-config.vus }}vu-${{ matrix.k6-config.iterations }}it-logs.txt - - name: Archive pprof CPU profiles + name: load-test-report.html + path: report + - name: Debug failure if: failure() - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + uses: ./.github/actions/kyverno-logs + + scale-test: + if: github.event_name == 'pull_request' + needs: + - define-matrix + - prepare-images + runs-on: ubuntu-latest + permissions: + packages: read + strategy: + fail-fast: false + matrix: + k8s-version: [v1.31.0] + test: ${{ fromJson(needs.define-matrix.outputs.tests) }} + steps: + - name: Checkout kyverno/kyverno + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name: Checkout kyverno/load-testing + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - name: pprof-cpu-profiles - path: cpu.pprof - - name: Archive pprof HEAP profiles - if: failure() - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + repository: kyverno/load-testing + path: load-testing + - name: Install Helm + id: helm + uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4.2.0 + - name: Create Kind cluster + uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0 + with: + node_image: kindest/node:${{ matrix.k8s-version }} + cluster_name: kind + config: ./scripts/config/kind/default.yaml + - name: Download kyverno images archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: kyverno.tar + - name: Load Kyverno images archive in Kind cluster + shell: bash + run: | + set -e + kind load image-archive kyverno.tar --name kind + - name: Install Metrics Server and Prometheus + shell: bash + run: | + set -e + export HELM=${{ steps.helm.outputs.helm-path }} + make dev-lab-metrics-server dev-lab-prometheus + - name: Install Kyverno + shell: bash + run: | + set -e + export HELM=${{ steps.helm.outputs.helm-path }} + export USE_CONFIG=default-with-profiling + export EXPLICIT_INSTALL_SETTINGS='--set admissionController.replicas=${{ matrix.test.replicas }} --set admissionController.serviceMonitor.enabled=true --set reportsController.serviceMonitor.enabled=true --set admissionController.container.resources.requests.cpu=${{ matrix.test.cpu_request }} --set admissionController.container.resources.requests.memory=${{ matrix.test.memory_request }} --set admissionController.container.resources.limits.memory=${{ matrix.test.memory_limit }} --set reportsController.resources.limits.memory=10Gi' + make kind-install-kyverno + - name: Wait for kyverno ready + uses: ./.github/actions/kyverno-wait-ready + - name: Install K6 + shell: bash + run: | + set -e + go install go.k6.io/xk6/cmd/xk6@latest + $(go env GOPATH)/bin/xk6 build --with github.com/grafana/xk6-dashboard@latest + mkdir -p $HOME/.local/bin && mv ./k6 $HOME/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH + - name: Run load tests using K6 + shell: bash + run: | + set -e + mkdir -p report + KYVERNO_NODE_IP=$(kubectl get nodes -o jsonpath='{.items[?(@.metadata.labels.kubernetes\.io/hostname=="kind-control-plane")].status.addresses[?(@.type=="InternalIP")].address}') + curl -s "http://$KYVERNO_NODE_IP:30950/debug/pprof/profile?seconds=30" > report/cpu.pprof & + cd load-testing + ./k6/run.sh k6/tests/${{ matrix.test.name }}.js -e SCENARIO=${{ matrix.test.scenario }} --vus ${{ matrix.test.concurrent_connections }} --iterations ${{ matrix.test.total_iterations }} ${{ matrix.test.extra_options }} --out dashboard=export=load-report.html + wait %1 || true + mv load-report.html ../report + - name: Collect Resource Metrics + shell: bash + run: | + set -e + kubectl port-forward --address 127.0.0.1 svc/kube-prometheus-stack-prometheus 9090:9090 -n monitoring & + sleep 3 + curl -s "http://127.0.0.1:9090/prometheus/api/v1/query?query=$(echo -n "rate(container_cpu_usage_seconds_total{image=\"$(make kind-admission-controller-image-name)\"}[1m])" | jq -sRr @uri)" > report/cpu-usage.json + curl -s "http://127.0.0.1:9090/prometheus/api/v1/query?query=$(echo -n "max_over_time(container_memory_working_set_bytes{image=\"$(make kind-admission-controller-image-name)\"}[1m])/(2^20)" | jq -sRr @uri)" > report/memory-usage.json + kill %1 || true + - name: Collect Report Metrics + shell: bash + run: | + set -e + sleep 60 + ./test/load/k6/reports-size-in-etcd.sh > report/reports-size-in-etcd.txt + - name: Archive Report + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: - name: pprof-heap-profiles - path: heap.pprof + name: report-${{ matrix.k8s-version }}-${{ matrix.test.name }}-${{ matrix.test.scenario }}-${{ matrix.test.replicas }}-${{ matrix.test.cpu_request }}-${{ matrix.test.memory_request }}-${{ matrix.test.memory_limit }}-${{ matrix.test.concurrent_connections }} + path: report - name: Debug failure - if: failure() + # if: failure() uses: ./.github/actions/kyverno-logs diff --git a/Makefile b/Makefile index 19fb771ad8c2..8fc0497df44d 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ REPO_CLEANUP := $(REGISTRY)/$(REPO)/$(CLEANUP_IMAGE) REPO_REPORTS := $(REGISTRY)/$(REPO)/$(REPORTS_IMAGE) REPO_BACKGROUND := $(REGISTRY)/$(REPO)/$(BACKGROUND_IMAGE) USE_CONFIG ?= standard +INSTALL_VERSION ?= 3.2.6 ######### # TOOLS # @@ -997,7 +998,17 @@ kind-install-kyverno: $(HELM) ## Install kyverno helm chart --set crds.migration.image.registry=$(LOCAL_REGISTRY) \ --set crds.migration.image.repository=$(LOCAL_CLI_REPO) \ --set crds.migration.image.tag=$(GIT_SHA) \ - $(foreach CONFIG,$(subst $(COMMA), ,$(USE_CONFIG)),--values ./scripts/config/$(CONFIG)/kyverno.yaml) + $(foreach CONFIG,$(subst $(COMMA), ,$(USE_CONFIG)),--values ./scripts/config/$(CONFIG)/kyverno.yaml) \ + $(EXPLICIT_INSTALL_SETTINGS) + +.PHONY: kind-install-kyverno-from-repo +kind-install-kyverno-from-repo: $(HELM) ## Install Kyverno Helm Chart from the Kyverno repo + @echo Install kyverno chart... >&2 + @$(HELM) upgrade --install kyverno --namespace kyverno --create-namespace --wait \ + --repo https://kyverno.github.io/kyverno/ kyverno \ + --version $(INSTALL_VERSION) \ + $(foreach CONFIG,$(subst $(COMMA), ,$(USE_CONFIG)),--values ./scripts/config/$(CONFIG)/kyverno.yaml) \ + $(EXPLICIT_INSTALL_SETTINGS) .PHONY: kind-install-goldilocks kind-install-goldilocks: $(HELM) ## Install goldilocks helm chart @@ -1029,6 +1040,10 @@ kind-deploy-reporter: $(HELM) ## Deploy policy-reporter helm chart --values ./scripts/config/standard/kyverno-reporter.yaml @kubectl port-forward -n policy-reporter services/policy-reporter-ui 8082:8080 +.PHONY: kind-admission-controller-image-name +kind-admission-controller-image-name: ## Print admission controller image name + @echo -n $(LOCAL_REGISTRY)/$(LOCAL_KYVERNO_REPO):$(GIT_SHA) + ########### # ROLLOUT # ########### diff --git a/scripts/config/standard-with-profiling/kyverno.yaml b/scripts/config/standard-with-profiling/kyverno.yaml new file mode 100644 index 000000000000..12b989f27f11 --- /dev/null +++ b/scripts/config/standard-with-profiling/kyverno.yaml @@ -0,0 +1,67 @@ +features: + policyExceptions: + enabled: true + omitEvents: + eventTypes: [] + +admissionController: + extraArgs: + v: 4 + rbac: + clusterRole: + extraResources: + - apiGroups: + - "*" + resources: + - secrets + verbs: + - create + - update + - patch + - delete + - get + - list + profiling: + enabled: true + serviceType: NodePort + nodePort: 30950 + +backgroundController: + extraArgs: + v: 4 + rbac: + clusterRole: + extraResources: + - apiGroups: + - "*" + resources: + - configmaps + - networkpolicies + - resourcequotas + - secrets + - roles + - rolebindings + - limitranges + - namespaces + - nodes + - nodes/status + - pods + verbs: + - create + - update + - patch + - delete + - get + - list + +cleanupController: + rbac: + clusterRole: + extraResources: + - apiGroups: + - "" + resources: + - pods + verbs: + - list + - delete diff --git a/scripts/config/stress-with-profiling/kyverno.yaml b/scripts/config/stress-with-profiling/kyverno.yaml deleted file mode 100644 index dfeb1358c7b2..000000000000 --- a/scripts/config/stress-with-profiling/kyverno.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# stress-with-profiling sets the most restricted configurations for the admission controller. It disables -# any additional feature that could affect the performance of the admission controller. -features: - policyExceptions: - enabled: true - omitEvents: - eventTypes: - - PolicyApplied - - PolicySkipped - - PolicyViolation - - PolicyError - admissionReports: - enabled: false - -admissionController: - profiling: - enabled: true - serviceType: NodePort - nodePort: 30950 \ No newline at end of file diff --git a/test/load/k6/pull_request-matrix.json b/test/load/k6/pull_request-matrix.json new file mode 100644 index 000000000000..8bc253320995 --- /dev/null +++ b/test/load/k6/pull_request-matrix.json @@ -0,0 +1,79 @@ +[ + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 20, + "total_iterations": 5000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 50, + "total_iterations": 10000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 20, + "total_iterations": 5000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 50, + "total_iterations": 10000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-generate", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "" + } +] diff --git a/test/load/k6/release-matrix.json b/test/load/k6/release-matrix.json new file mode 100644 index 000000000000..8bc253320995 --- /dev/null +++ b/test/load/k6/release-matrix.json @@ -0,0 +1,79 @@ +[ + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 20, + "total_iterations": 5000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 50, + "total_iterations": 10000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 20, + "total_iterations": 5000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 50, + "total_iterations": 10000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-generate", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "" + } +] diff --git a/test/load/k6/reports-size-in-etcd.sh b/test/load/k6/reports-size-in-etcd.sh new file mode 100755 index 000000000000..0a0f7c50b596 --- /dev/null +++ b/test/load/k6/reports-size-in-etcd.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Function to execute etcdctl commands +execute_etcdctl() { + local key=$1 + local options=$2 + kubectl -n kube-system exec etcd-kind-control-plane -- sh -c \ + "ETCDCTL_API=3 etcdctl --cacert /etc/kubernetes/pki/etcd/ca.crt \ + --key /etc/kubernetes/pki/etcd/server.key \ + --cert /etc/kubernetes/pki/etcd/server.crt \ + get $key $options" +} + +# Function to extract size and metadata +get_key_info() { + local key=$1 + + local size=$(execute_etcdctl "$key" "--print-value-only" | wc -c) + local count=$(execute_etcdctl "$key" "--write-out=fields" | grep "Count" | cut -f2 -d':') + + if [ "$count" -ne 0 ]; then + local versions=$(execute_etcdctl "$key" "--write-out=fields" | grep "Version" | cut -f2 -d':') + else + local versions=0 + fi + + # Return size, count, and versions as a string + echo "$size $count $versions" +} + +# Initialize sum +total_size=0 +output_file="/tmp/etcdkeys.txt" + +# Get list of policy report keys +keys=$(execute_etcdctl "/registry/wgpolicyk8s.io/policyreports" "--prefix --keys-only") + +# Process each key +for key in $keys; do + read size count versions <<< $(get_key_info "$key") + total=$((size * versions)) + total_size=$((total_size + total)) + + # Log to output file + echo "$total_size $total $size $versions $count $key" >> "$output_file" +done + +# Print final total size +echo "The total size for Policy Reports is $total_size bytes." diff --git a/test/load/k6/schedule-matrix.json b/test/load/k6/schedule-matrix.json new file mode 100644 index 000000000000..8bc253320995 --- /dev/null +++ b/test/load/k6/schedule-matrix.json @@ -0,0 +1,79 @@ +[ + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 20, + "total_iterations": 5000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 50, + "total_iterations": 10000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 20, + "total_iterations": 5000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-pss", + "scenario": "", + "replicas": 3, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 50, + "total_iterations": 10000, + "extra_options": "--no-teardown" + }, + { + "name": "kyverno-generate", + "scenario": "", + "replicas": 1, + "cpu_request": "100m", + "memory_request": "128Mi", + "memory_limit": "384Mi", + "concurrent_connections": 10, + "total_iterations": 1000, + "extra_options": "" + } +] From 4e3bde519a13bbf49c718f59b66751d1a9780fbb Mon Sep 17 00:00:00 2001 From: Killian ALLAIRE Date: Tue, 5 Nov 2024 08:59:50 +0100 Subject: [PATCH 2/2] feat(chart):add Helms' CustomLabels to the pods metadata of jobs and cronjobs k8s ressources (#11483) * all pods for the cronjobs will now act the same way as the pods in the deployments Signed-off-by: LazyKeru * updated the artifacthub.io/changes on the Chart.yaml Signed-off-by: LazyKeru * ran 'make codegen-manifest-all' to update install-latest-testing.yaml Signed-off-by: LazyKeru --------- Signed-off-by: LazyKeru Co-authored-by: kyverno-bot <104836976+kyverno-bot@users.noreply.github.com> --- charts/kyverno/Chart.yaml | 4 +--- .../cleanup/cleanup-admission-reports.yaml | 7 ++++--- .../cleanup-cluster-admission-reports.yaml | 7 ++++--- .../cleanup-cluster-ephemeral-reports.yaml | 7 ++++--- .../cleanup/cleanup-ephemeral-reports.yaml | 7 ++++--- .../cleanup/cleanup-update-requests.yaml | 7 ++++--- config/install-latest-testing.yaml | 20 +++++++++++++++++++ 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/charts/kyverno/Chart.yaml b/charts/kyverno/Chart.yaml index d0444b60ad0a..85e479f604f7 100644 --- a/charts/kyverno/Chart.yaml +++ b/charts/kyverno/Chart.yaml @@ -34,9 +34,7 @@ annotations: # valid kinds are: added, changed, deprecated, removed, fixed and security artifacthub.io/changes: | - kind: added - description: Add a key to preserve configmap settings during upgrade - - kind: added - description: Make admission reports breaker threshold configurable + description: Added customLabels to the pods label metadata of the cronjobs dependencies: - name: grafana version: 3.2.7 diff --git a/charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml b/charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml index 721918295422..1ddbea0b23c5 100644 --- a/charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml +++ b/charts/kyverno/templates/cleanup/cleanup-admission-reports.yaml @@ -20,10 +20,11 @@ spec: annotations: {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.cleanupJobs.admissionReports.podLabels }} labels: - {{- toYaml . | nindent 12 }} - {{- end }} + {{- include "kyverno.cleanup.labels" . | nindent 12 }} + {{- with .Values.cleanupJobs.admissionReports.podLabels }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} spec: serviceAccountName: {{ template "kyverno.name" . }}-cleanup-jobs {{- with .Values.cleanupJobs.admissionReports.podSecurityContext }} diff --git a/charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml b/charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml index 76d19ad40a94..869df5f4e8c9 100644 --- a/charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml +++ b/charts/kyverno/templates/cleanup/cleanup-cluster-admission-reports.yaml @@ -20,10 +20,11 @@ spec: annotations: {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.cleanupJobs.clusterAdmissionReports.podLabels }} labels: - {{- toYaml . | nindent 12 }} - {{- end }} + {{- include "kyverno.cleanup.labels" . | nindent 12 }} + {{- with .Values.cleanupJobs.clusterAdmissionReports.podLabels }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} spec: serviceAccountName: {{ template "kyverno.name" . }}-cleanup-jobs {{- with .Values.cleanupJobs.clusterAdmissionReports.podSecurityContext }} diff --git a/charts/kyverno/templates/cleanup/cleanup-cluster-ephemeral-reports.yaml b/charts/kyverno/templates/cleanup/cleanup-cluster-ephemeral-reports.yaml index f5fb679d97ff..3619b7634efd 100644 --- a/charts/kyverno/templates/cleanup/cleanup-cluster-ephemeral-reports.yaml +++ b/charts/kyverno/templates/cleanup/cleanup-cluster-ephemeral-reports.yaml @@ -23,10 +23,11 @@ spec: annotations: {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.cleanupJobs.clusterEphemeralReports.podLabels }} labels: - {{- toYaml . | nindent 12 }} - {{- end }} + {{- include "kyverno.cleanup.labels" . | nindent 12 }} + {{- with .Values.cleanupJobs.clusterEphemeralReports.podLabels }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} spec: serviceAccountName: {{ template "kyverno.name" . }}-cleanup-jobs {{- with .Values.cleanupJobs.clusterEphemeralReports.podSecurityContext }} diff --git a/charts/kyverno/templates/cleanup/cleanup-ephemeral-reports.yaml b/charts/kyverno/templates/cleanup/cleanup-ephemeral-reports.yaml index 2ec6469ad5cc..ee4eee6299a9 100644 --- a/charts/kyverno/templates/cleanup/cleanup-ephemeral-reports.yaml +++ b/charts/kyverno/templates/cleanup/cleanup-ephemeral-reports.yaml @@ -23,10 +23,11 @@ spec: annotations: {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.cleanupJobs.ephemeralReports.podLabels }} labels: - {{- toYaml . | nindent 12 }} - {{- end }} + {{- include "kyverno.cleanup.labels" . | nindent 12 }} + {{- with .Values.cleanupJobs.ephemeralReports.podLabels }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} spec: serviceAccountName: {{ template "kyverno.name" . }}-cleanup-jobs {{- with .Values.cleanupJobs.ephemeralReports.podSecurityContext }} diff --git a/charts/kyverno/templates/cleanup/cleanup-update-requests.yaml b/charts/kyverno/templates/cleanup/cleanup-update-requests.yaml index ac564f9377ef..940d6279b49d 100644 --- a/charts/kyverno/templates/cleanup/cleanup-update-requests.yaml +++ b/charts/kyverno/templates/cleanup/cleanup-update-requests.yaml @@ -23,10 +23,11 @@ spec: annotations: {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.cleanupJobs.updateRequests.podLabels }} labels: - {{- toYaml . | nindent 12 }} - {{- end }} + {{- include "kyverno.cleanup.labels" . | nindent 12 }} + {{- with .Values.cleanupJobs.updateRequests.podLabels }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} spec: serviceAccountName: {{ template "kyverno.name" . }}-cleanup-jobs {{- with .Values.cleanupJobs.updateRequests.podSecurityContext }} diff --git a/config/install-latest-testing.yaml b/config/install-latest-testing.yaml index f522eb969f36..3dd5acfd7e7d 100644 --- a/config/install-latest-testing.yaml +++ b/config/install-latest-testing.yaml @@ -49410,6 +49410,11 @@ spec: backoffLimit: 3 template: metadata: + labels: + app.kubernetes.io/component: cleanup + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno + app.kubernetes.io/version: latest spec: serviceAccountName: kyverno-cleanup-jobs containers: @@ -49460,6 +49465,11 @@ spec: backoffLimit: 3 template: metadata: + labels: + app.kubernetes.io/component: cleanup + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno + app.kubernetes.io/version: latest spec: serviceAccountName: kyverno-cleanup-jobs containers: @@ -49510,6 +49520,11 @@ spec: backoffLimit: 3 template: metadata: + labels: + app.kubernetes.io/component: cleanup + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno + app.kubernetes.io/version: latest spec: serviceAccountName: kyverno-cleanup-jobs containers: @@ -49560,6 +49575,11 @@ spec: backoffLimit: 3 template: metadata: + labels: + app.kubernetes.io/component: cleanup + app.kubernetes.io/instance: kyverno + app.kubernetes.io/part-of: kyverno + app.kubernetes.io/version: latest spec: serviceAccountName: kyverno-cleanup-jobs containers: