diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index bd258389f9..45b5f82033 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -4,39 +4,47 @@ on: workflow_dispatch: inputs: runStandalone: - description: 'Run the benchmarks against standalone APM Server with Moxy' + description: 'Benchmark against standalone APM Server with Moxy' required: false type: boolean default: false enableTailSampling: - description: 'Enable tail-based sampling on the APM server' + description: 'Enable tail-based sampling on APM server' required: false type: boolean default: false tailSamplingStorageLimit: - description: 'Storage size limit of tail-based sampling on the APM server, defaults to 10GB' + description: 'Storage size limit of tail-based sampling on APM server' required: false type: string default: "10GB" profile: - description: 'The system profile used to run the benchmarks' + description: 'System profile for benchmark, e.g. system-profiles/8GBx1zone.tfvars' required: false type: string + # defaults set below (TFVARS_SOURCE) runOnStable: - description: 'Run the benchmarks on the latest stable version' + description: 'Benchmark on latest stable version' required: false type: boolean default: false benchmarkAgents: - description: 'Set the number of agents to send data to the APM Server' + description: 'Number of benchmark agents sending data to APM Server' required: false type: string + # defaults set in Makefile benchmarkRun: - description: 'Set the expression that matches the benchmark scenarios to run' + description: 'Benchmark scenarios that only match regex, e.g. BenchmarkAgentAll' required: false type: string + # defaults set in Makefile + warmupTime: + description: 'Benchmark warmup time for APM Server e.g. 3s, 5m, 2h' + required: false + type: string + # defaults set below (Set up env) pgoExport: - description: 'Run the benchmarks with PGO export' + description: 'Benchmark with PGO export' required: false type: boolean default: false @@ -71,7 +79,7 @@ jobs: TF_VAR_apm_server_tail_sampling_storage_limit: ${{ inputs.tailSamplingStorageLimit || '10GB' }} # set the default again otherwise schedules won't work RUN_STANDALONE: ${{ inputs.runStandalone || github.event.schedule=='0 0 1 * *' }} PGO_EXPORT: ${{ inputs.pgoExport || github.event.schedule=='0 0 1 * *' }} - TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # // Default to use an 8gb profile + TFVARS_SOURCE: ${{ inputs.profile || 'system-profiles/8GBx1zone.tfvars' }} # set the default again otherwise schedules won't work TF_VAR_BUILD_ID: ${{ github.run_id }} TF_VAR_ENVIRONMENT: ci TF_VAR_REPO: ${{ github.repository }} @@ -102,6 +110,15 @@ jobs: if [ ! -z "${{ inputs.benchmarkRun }}" ]; then echo "BENCHMARK_RUN=${{ inputs.benchmarkRun }}" >> "$GITHUB_ENV" fi + if [ ! -z "${{ inputs.warmupTime }}" ]; then + echo "BENCHMARK_WARMUP_TIME=${{ inputs.warmupTime }}" >> "$GITHUB_ENV" + elif [ "${{github.event_name}}" == 'schedule' ] && [ "${{env.RUN_STANDALONE}}" == 'false' ]; then + echo "BENCHMARK_WARMUP_TIME=10m" >> "$GITHUB_ENV" + elif [ "${{github.event_name}}" == 'schedule' ] && [ "${{env.RUN_STANDALONE}}" == 'true' ]; then + echo "BENCHMARK_WARMUP_TIME=3m" >> "$GITHUB_ENV" + else + echo "BENCHMARK_WARMUP_TIME=5m" >> "$GITHUB_ENV" + fi # exclude BenchmarkTraces* from scheduled benchmarks - if: github.event_name == 'schedule' diff --git a/testing/benchmark/README.md b/testing/benchmark/README.md index 62139f6e88..2ababbf392 100644 --- a/testing/benchmark/README.md +++ b/testing/benchmark/README.md @@ -65,7 +65,8 @@ The main commands are: - `BENCHMARK_TIME`: Set the amount of time to run each benchmark scenario for. Defaults to `2m`. - `BENCHMARK_RUN`: Set the expression that matches the benchmark scenarios to run. Defaults to `Benchmark` (all). - `BENCHMARK_RESULT`: Set the output file where the results of the benchmark will be written. Defaults to `benchmark-result.txt` - - `BENCHMARK_DETAILED`: Sets the `-detailed` when running `apmbench`, displaying extra metrics for each benchmark. Defaults to `false`. + - `BENCHMARK_DETAILED`: Set the `-detailed` when running `apmbench`, displaying extra metrics for each benchmark. Defaults to `false`. + - `BENCHMARK_EVENT_RATE`: Set the rate of events sent to the APM Server. Defaults to `0/s`, which means unlimited. - `index-benchmark-result`: Indexes `$(BENCHMARK_RESULT)` to an Elasticsearch cluster. Can be configured with: - `GOBENCH_INDEX`: Set the Elasticsearch index where the benchmark results will be stored. Defaults to `gobench`. - `GOBENCH_USERNAME`: Set the Elasticsearch username to use for authentication. Defaults to `admin`.