From a7ffd4143f6eef6d092174ac871213d3cb91471a Mon Sep 17 00:00:00 2001 From: Samuel Burnham <45365069+samuelburnham@users.noreply.github.com> Date: Fri, 1 Nov 2024 10:01:20 -0400 Subject: [PATCH] ci: Refactor LC bench (#311) * ci: Improve automated benchmark report * ci: Refactor automated benches into composite action * Test e2e * Prep for review --- .github/actions/bench/action.yml | 87 ++++++++++ .github/workflows/bench.yml | 154 ++++-------------- .../light-client/benches/committee_change.rs | 4 +- ethereum/light-client/benches/inclusion.rs | 4 +- kadena/light-client/benches/longest_chain.rs | 4 +- kadena/light-client/benches/spv.rs | 4 +- 6 files changed, 123 insertions(+), 134 deletions(-) create mode 100644 .github/actions/bench/action.yml diff --git a/.github/actions/bench/action.yml b/.github/actions/bench/action.yml new file mode 100644 index 00000000..03e33510 --- /dev/null +++ b/.github/actions/bench/action.yml @@ -0,0 +1,87 @@ +name: Run a light client benchmark + +description: Run benchmark and output a Markdown report + +inputs: + light-client: + description: 'Light client to benchmark' + required: true + benchmark: + description: 'Benchmark to run' + required: true + +outputs: + report: + description: "Markdown report" + value: ${{ steps.prep-report.outputs.report }} + +runs: + using: "composite" + steps: + - name: Install jtbl + shell: bash + run: | + sudo apt-get update && sudo apt-get install -y python3-pip + pip3 --version + pip3 install --break-system-packages jtbl + echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile + source ~/.profile + which jtbl + - name: Run benchmarks + shell: bash + run: | + make bench-ci BENCH=${{ inputs.benchmark }} 2>&1 | tee out.txt + working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client + - name: Create report + shell: bash + run: | + grep 'cycles=' out.txt > cycles.txt + grep 'proving_time' out.txt > timings.txt + + while IFS=$'\t' read -r f1 f2 + do + num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }') + timings=$(echo "$f2" | jq ' + to_entries | + map( + if .key == "proving_time" then + {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")} + elif .key == "verifying_time" then + {key, value: ((.value / 1000 * 1000 | floor) / 1000 | tostring + "s")} + else + . + end + ) | + from_entries + ') + + echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json + done < <(paste cycles.txt timings.txt) + + COMMIT_SHORT=$(git rev-parse --short HEAD) + + echo '# `${{ inputs.light-client }}` Benchmark Results' | tee -a summary.md + echo "Commit: \`$COMMIT_SHORT\`" | tee -a summary.md + echo '## `${{ inputs.benchmark }}` Proof' | tee -a summary.md + cat summary.json | jtbl -m | tee -a summary.md + echo "" | tee -a summary.md + + echo "[Workflow URL](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" | tee -a summary.md + working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client + - name: Write bench on commit comment + uses: peter-evans/commit-comment@v3 + id: commit-comment + with: + body-path: ${{ github.workspace }}/${{ inputs.light-client }}/light-client/summary.md + - name: Prep report for Zulip + id: prep-report + shell: bash + run: | + COMMIT=$(git rev-parse HEAD) + ID=${{ steps.commit-comment.outputs.comment-id }} + echo "[Commit comment](https://github.com/${{ github.repository }}/commit/$COMMIT#commitcomment-$ID)" | tee -a summary.md + + echo "report<> $GITHUB_OUTPUT + cat summary.md >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index fe016535..ccbae067 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,4 +1,4 @@ -# Runs benchmarks on self-hosted infra via `workflow_dispatch` and scheduled on Mondays at 9am EST +# Runs benchmarks on self-hosted infra via `workflow_dispatch` and scheduled on Thursdays at 2pm UTC # # `workflow_dispatch` trigger # The workflow can be run at https://github.com/argumentcomputer/zk-light-clients/actions/workflows/bench.yml @@ -6,7 +6,7 @@ # The report can also be sent as a Zulip message to https://zulip.argument.xyz # # `schedule` trigger -# The workflow runs every week on Monday at 9am EST +# The workflow runs every week on Thursday at 2pm UTC # It runs all light client benchmarks and sends them to Zulip name: Light client benchmark on: @@ -69,7 +69,7 @@ concurrency: jobs: benchmark-manual: - name: Light client benchmark (manual) + name: LC bench (manual) if: github.event_name == 'workflow_dispatch' runs-on: warp-custom-r7iz-metal-32xl steps: @@ -80,14 +80,6 @@ jobs: - uses: actions/checkout@v4 - name: Setup CI uses: ./.github/actions/setup - - name: Install extra deps - run: | - sudo apt-get update && sudo apt-get install -y python3-pip - pip3 --version - pip3 install --break-system-packages jtbl - echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile - source ~/.profile - which jtbl - name: Set env run: | # Default benchmark settings optimized for light clients, can be overwritten with `env` input @@ -128,49 +120,12 @@ jobs: echo "STREAM=$STREAM" | tee -a $GITHUB_ENV echo "TOPIC=$TOPIC" | tee -a $GITHUB_ENV fi - - name: Run benchmarks - run: | - make bench-ci BENCH=${{ inputs.bench-name }} 2>&1 | tee out.txt - working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client - - name: Create report - id: run-benchmarks - run: | - grep 'cycles=' out.txt > cycles.txt - grep 'proving_time' out.txt > timings.txt - - while IFS=$'\t' read -r f1 f2 - do - num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }') - timings=$(echo "$f2" | jq ' - to_entries | - map( - if .key == "proving_time" or .key == "verifying_time" then - {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")} - else - . - end - ) | - from_entries - ') - - echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json - done < <(paste cycles.txt timings.txt) - - echo '# `${{ inputs.light-client }}` Benchmark Results' | tee -a summary.md - echo '## `${{ inputs.bench-name }}` Prove' | tee -a summary.md - cat summary.json | jtbl -m | tee -a summary.md - echo "" | tee -a summary.md - - echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md - - echo "report<> $GITHUB_OUTPUT - cat summary.md >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - working-directory: ${{ github.workspace }}/${{ inputs.light-client }}/light-client - - name: Write bench on commit comment - uses: peter-evans/commit-comment@v3 + - name: Run benchmark and create report + id: run-benchmark + uses: ./.github/actions/bench with: - body-path: ${{ github.workspace }}/${{ inputs.light-client }}/light-client/summary.md + light-client: ${{ inputs.light-client }} + benchmark: ${{ inputs.bench-name }} - name: Send report to Zulip if: inputs.zulip uses: zulip/github-actions-zulip/send-message@v1 @@ -182,29 +137,28 @@ jobs: type: "${{ env.TYPE }}" # Ignored if `type: private` topic: "${{ env.TOPIC }}" - content: "${{ steps.run-benchmarks.outputs.report }}" + content: "${{ steps.run-benchmark.outputs.report }}" benchmark-scheduled: - name: Light client bench (scheduled) + name: LC bench (scheduled) if: github.event_name == 'schedule' runs-on: warp-custom-r7iz-metal-32xl strategy: fail-fast: false matrix: - light-client: [ aptos, ethereum, kadena ] include: - - benchmark: inclusion - light-client: aptos - - benchmark: epoch_change - light-client: aptos - - benchmark: inclusion - light-client: ethereum - - benchmark: committee_change - light-client: ethereum - - benchmark: spv - light-client: kadena - - benchmark: longest_chain - light-client: kadena + - light-client: aptos + benchmark: inclusion + - light-client: aptos + benchmark: epoch_change + - light-client: ethereum + benchmark: inclusion + - light-client: ethereum + benchmark: committee_change + - light-client: kadena + benchmark: spv + - light-client: kadena + benchmark: longest_chain steps: - uses: actions/checkout@v4 with: @@ -213,14 +167,6 @@ jobs: - uses: actions/checkout@v4 - name: Setup CI uses: ./.github/actions/setup - - name: Install extra deps - run: | - sudo apt-get update && sudo apt-get install -y python3-pip - pip3 --version - pip3 install --break-system-packages jtbl - echo 'PATH="$HOME/.local/bin:$PATH"' >> ~/.profile - source ~/.profile - which jtbl - name: Set env run: | # Default benchmark settings optimized for light clients, can be overwritten with `env` input @@ -230,61 +176,17 @@ jobs: echo "RECONSTRUCT_COMMITMENTS=false" | tee -a $GITHUB_ENV echo "SHARD_CHUNKING_MULTIPLIER=1" | tee -a $GITHUB_ENV echo "MODE=SNARK" | tee -a $GITHUB_ENV - - IFS=',' read -ra ENV_VARS <<< "ethereum" - for VAR in "${ENV_VARS[@]}"; do - VAR_NAME="${VAR%%=*}" - VAR_VALUE="${VAR#*=}" - echo "${VAR_NAME}=${VAR_VALUE}" | tee -a $GITHUB_ENV - done - name: Set Zulip env run: | echo "TYPE=stream" | tee -a $GITHUB_ENV echo "STREAM=light-client" | tee -a $GITHUB_ENV echo "TOPIC=Benchmark Reports" | tee -a $GITHUB_ENV - - name: Run benchmarks - run: | - make bench-ci BENCH=${{ matrix.benchmark }} 2>&1 | tee out.txt - working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client - - name: Create report - id: run-benchmarks - run: | - grep 'cycles=' out.txt > cycles.txt - grep 'proving_time' out.txt > timings.txt - - while IFS=$'\t' read -r f1 f2 - do - num_cycles=$(echo "$f1" | grep -o 'cycles=[0-9]*' | awk -F'=' '{ print $2 }') - timings=$(echo "$f2" | jq ' - to_entries | - map( - if .key == "proving_time" or .key == "verifying_time" then - {key, value: (.value / 1000 | floor as $s | "\(($s / 60 | floor) | tostring)min\(($s % 60) | tostring)s")} - else - . - end - ) | - from_entries - ') - - echo "$timings" | jq -c --argjson cycles "$num_cycles" '. += {cycles: $cycles}' >> summary.json - done < <(paste cycles.txt timings.txt) - - echo '# `${{ matrix.light-client }}` Benchmark Results' | tee -a summary.md - echo '## `${{ matrix.benchmark }}` Prove' | tee -a summary.md - cat summary.json | jtbl -m | tee -a summary.md - echo "" | tee -a summary.md - - echo "Workflow URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | tee -a summary.md - - echo "report<> $GITHUB_OUTPUT - cat summary.md >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - working-directory: ${{ github.workspace }}/${{ matrix.light-client }}/light-client - - name: Write bench on commit comment - uses: peter-evans/commit-comment@v3 + - name: Run benchmark and create report + id: run-benchmark + uses: ./.github/actions/bench with: - body-path: ${{ github.workspace }}/${{ matrix.light-client }}/light-client/summary.md + light-client: ${{ matrix.light-client }} + benchmark: ${{ matrix.benchmark }} - name: Send report to Zulip uses: zulip/github-actions-zulip/send-message@v1 with: @@ -295,4 +197,4 @@ jobs: type: "${{ env.TYPE }}" # Ignored if `type: private` topic: "${{ env.TOPIC }}" - content: "${{ steps.run-benchmarks.outputs.report }}" \ No newline at end of file + content: "${{ steps.run-benchmark.outputs.report }}" diff --git a/ethereum/light-client/benches/committee_change.rs b/ethereum/light-client/benches/committee_change.rs index 8346dd0b..45c3cf55 100644 --- a/ethereum/light-client/benches/committee_change.rs +++ b/ethereum/light-client/benches/committee_change.rs @@ -71,7 +71,7 @@ impl BenchmarkAssets { #[derive(Debug, Clone, Serialize)] struct BenchResults { proving_time: u128, - verification_time: u128, + verifying_time: u128, } fn main() { @@ -114,7 +114,7 @@ fn main() { // Print results let results = BenchResults { proving_time: proving_time.as_millis(), - verification_time: verifying_time.as_millis(), + verifying_time: verifying_time.as_millis(), }; let json_output = serde_json::to_string(&results).unwrap(); diff --git a/ethereum/light-client/benches/inclusion.rs b/ethereum/light-client/benches/inclusion.rs index 9fa7e5d9..3160eb1b 100644 --- a/ethereum/light-client/benches/inclusion.rs +++ b/ethereum/light-client/benches/inclusion.rs @@ -84,7 +84,7 @@ impl BenchmarkAssets { #[derive(Debug, Clone, Serialize)] struct BenchResults { proving_time: u128, - verification_time: u128, + verifying_time: u128, } fn main() { @@ -125,7 +125,7 @@ fn main() { // Print results let results = BenchResults { proving_time: proving_time.as_millis(), - verification_time: verifying_time.as_millis(), + verifying_time: verifying_time.as_millis(), }; let json_output = serde_json::to_string(&results).unwrap(); diff --git a/kadena/light-client/benches/longest_chain.rs b/kadena/light-client/benches/longest_chain.rs index 9ec9c547..4c0a120d 100644 --- a/kadena/light-client/benches/longest_chain.rs +++ b/kadena/light-client/benches/longest_chain.rs @@ -29,7 +29,7 @@ impl BenchmarkAssets { #[derive(Debug, Clone, Serialize)] struct BenchResults { proving_time: u128, - verification_time: u128, + verifying_time: u128, } fn main() { @@ -65,7 +65,7 @@ fn main() { // Print results let results = BenchResults { proving_time: proving_time.as_millis(), - verification_time: verifying_time.as_millis(), + verifying_time: verifying_time.as_millis(), }; println!("{}", serde_json::to_string(&results).unwrap()); diff --git a/kadena/light-client/benches/spv.rs b/kadena/light-client/benches/spv.rs index c6ad48c1..7670ffa6 100644 --- a/kadena/light-client/benches/spv.rs +++ b/kadena/light-client/benches/spv.rs @@ -35,7 +35,7 @@ impl BenchmarkAssets { #[derive(Debug, Clone, Serialize)] struct BenchResults { proving_time: u128, - verification_time: u128, + verifying_time: u128, } fn main() { @@ -75,7 +75,7 @@ fn main() { // Print results let results = BenchResults { proving_time: proving_time.as_millis(), - verification_time: verifying_time.as_millis(), + verifying_time: verifying_time.as_millis(), }; println!("{}", serde_json::to_string(&results).unwrap());