From 13e36f86a51a7a56f7a25a6be62c49fa3449c591 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 14 Jan 2025 21:18:20 +0100 Subject: [PATCH] feat: initial setup for the ethereum hive github action (#1) Creating a reusable github action to run ethereum hive. --- .github/actions/install-deps/action.yaml | 18 ++ .github/workflows/pr-title-checker.yaml | 28 +++ .github/workflows/release-please.yaml | 20 ++ .github/workflows/test-custom-config.yaml | 26 +++ .github/workflows/test-simple.yaml | 19 ++ README.md | 143 ++++++++++++- action.yaml | 232 ++++++++++++++++++++++ example-workflows/pectra.yaml | 126 ++++++++++++ scripts/json-result-to-md-summary.sh | 39 ++++ 9 files changed, 649 insertions(+), 2 deletions(-) create mode 100644 .github/actions/install-deps/action.yaml create mode 100644 .github/workflows/pr-title-checker.yaml create mode 100644 .github/workflows/release-please.yaml create mode 100644 .github/workflows/test-custom-config.yaml create mode 100644 .github/workflows/test-simple.yaml create mode 100644 action.yaml create mode 100644 example-workflows/pectra.yaml create mode 100755 scripts/json-result-to-md-summary.sh diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml new file mode 100644 index 0000000..17cbe95 --- /dev/null +++ b/.github/actions/install-deps/action.yaml @@ -0,0 +1,18 @@ +name: Install dependencies + +runs: + using: 'composite' + steps: + - name: Add gh-cli to apt + shell: bash + run: | + (type -p wget >/dev/null || (apt update && apt-get install wget -y)) \ + && mkdir -p -m 755 /etc/apt/keyrings \ + && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null + - name: Install apt packages + shell: bash + run: | + apt update && apt install -y unzip gh yq diff --git a/.github/workflows/pr-title-checker.yaml b/.github/workflows/pr-title-checker.yaml new file mode 100644 index 0000000..e537e6c --- /dev/null +++ b/.github/workflows/pr-title-checker.yaml @@ -0,0 +1,28 @@ +name: pr-title-checker + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + - reopened + merge_group: + +concurrency: + group: "title-checker-${{ github.head_ref || github.ref }}" + cancel-in-progress: true + +permissions: + pull-requests: read + +jobs: + pr_title_checker: + name: validate + # Skip merge-queue PRs + if: contains(github.ref_name, 'gh-readonly-queue/main/') != true + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml new file mode 100644 index 0000000..1108efd --- /dev/null +++ b/.github/workflows/release-please.yaml @@ -0,0 +1,20 @@ +name: release-please + +on: + push: + branches: + - master + +permissions: + contents: write + pull-requests: write + +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - uses: googleapis/release-please-action@d1a8f221d7723166f48a584aebba00ef3f6febec + with: + token: ${{ secrets.PAT }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json diff --git a/.github/workflows/test-custom-config.yaml b/.github/workflows/test-custom-config.yaml new file mode 100644 index 0000000..11ed33a --- /dev/null +++ b/.github/workflows/test-custom-config.yaml @@ -0,0 +1,26 @@ +name: Test - Custom Config +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./ + with: + client: go-ethereum + simulator: ethereum/rpc-compat + workflow_artifact_upload: true + client_config: | + - client: go-ethereum + nametag: go-ethereum-main + dockerfile: git + build_args: + github: ethereum/go-ethereum + tag: master diff --git a/.github/workflows/test-simple.yaml b/.github/workflows/test-simple.yaml new file mode 100644 index 0000000..cea5a05 --- /dev/null +++ b/.github/workflows/test-simple.yaml @@ -0,0 +1,19 @@ +name: Test - Simple +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./ + with: + client: go-ethereum + simulator: ethereum/sync + workflow_artifact_upload: true diff --git a/README.md b/README.md index 29f8925..b8be5d0 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,141 @@ -# hive-github-action -GitHub action to run [Ethereum Hive](https://github.com/ethereum/hive) +# Ethereum Hive 🐝 Github Action + +This action is a wrapper around [Ethereum Hive](https://github.com/ethereum/hive). It allows you to run tests with different clients and simulators. It also supports uploading test results to S3 and/or as a workflow artifact. + +> ⚠️ **Note:** This action is still under development and may introduce breaking changes. If you want to use it in your workflows, make sure to reference to a specific commit hash. +## Inputs + +### Test Configuration + +| Input | Description | Required | Default | +|-------|-------------|----------|---------| +| `simulator` | Simulator to run (e.g. ethereum/sync) | Yes | `ethereum/sync` | +| `client` | Client to test | Yes | `go-ethereum` | +| `client_config` | Client configuration in YAML format | No | - | +| `extra_flags` | Additional flags to pass to hive | No | - | + +### Hive Configuration + +| Input | Description | Required | Default | +|-------|-------------|----------|---------| +| `hive_repository` | Hive repository to use | No | `ethereum/hive` | +| `hive_version` | Hive version/branch to use | No | `master` | +| `go_version` | Go version used to build hive | No | `1.21` | + +### S3 Upload Configuration + +| Input | Description | Required | Default | +|-------|-------------|----------|---------| +| `s3_upload` | Upload test results to S3 | No | `false` | +| `s3_bucket` | S3 bucket name | No* | - | +| `s3_path` | Path prefix in S3 bucket | No | `hive-results` | +| `rclone_version` | Rclone version to use | No | `latest` | +| `rclone_config` | Base64 encoded rclone config file | No | - | + +*Required if `s3_upload` is `true` + +### GitHub Workflow Artifact Configuration + +| Input | Description | Required | Default | +|-------|-------------|----------|---------| +| `workflow_artifact_upload` | Upload test results as a workflow artifact | No | `false` | +| `workflow_artifact_prefix` | Name of the workflow prefix. If not provided, the prefix will be the simulator and client name. | No | - | + +## Examples + +### Simple example doing sync tests with the latest go-ethereum + +```yaml +- uses: ./.github/actions/hive + with: + client: go-ethereum + simulator: ethereum/sync +``` + +### With custom client config + +You can customize the [client configuration](https://github.com/ethereum/hive/blob/master/docs/commandline.md#client-build-parameters). This allows using different docker files that are available for each client and their respective build arguments. + +```yaml +env: + CLIENT_CONFIG: | + - client: go-ethereum + nametag: prague-devnet-5 + dockerfile: git + build_args: + github: ethpandaops/go-ethereum + tag: my-custom-branch +``` + +Then you can use the `CLIENT_CONFIG` environment variable in your workflow. + +```yaml +- uses: ./.github/actions/hive + with: + client: go-ethereum + simulator: ethereum/sync + client_config: ${{ env.CLIENT_CONFIG }} +``` + +### Uploading the results directory to S3 + +You'll need to create an rclone config and base64 encode it. Then store it as a github actions secret on your repository. + +An example rclone config could look like this: + +```toml +# Content of rclone.conf +[s3] +type = s3 +provider = Cloudflare +region = auto +endpoint = https://your-r2-account-id.r2.cloudflarestorage.com +access_key_id = your-access-key-id +secret_access_key = your-secret-access-key +no_check_bucket = true +``` + +Then you can run `base64 -w 0 rclone.conf` and store the output as a github actions secret. + +Afterwards you just need to reference the secret for the `rclone_config` input. + +```yaml +- uses: ./.github/actions/hive + with: + client: go-ethereum + simulator: ethereum/sync + client_config: ${{ env.CLIENT_CONFIG }} + s3_upload: true + s3_bucket: my-bucket + s3_path: my-path + rclone_config: ${{ secrets.RCLONE_CONFIG }} +``` + +### Uploading the results directory as an GitHub workflow artifact + +This will upload the test results as a workflow artifact. By default the artifact prefix will be the simulator and client name. You can override this by providing a `workflow_artifact_prefix` input. +```yaml +- uses: ./.github/actions/hive + with: + client: go-ethereum + simulator: ethereum/sync + workflow_artifact_upload: true + # workflow_artifact_prefix: my-custom-prefix +``` +## Local testing and development + +To test the action locally, you can run [act](https://github.com/nektos/act) with the following command: + +```bash +act -s GITHUB_TOKEN="$(gh auth token)" -W '.github/workflows/test.yaml' +``` + +Or if you configured a custom runner, you can run `act` with the following command: + +```bash +act -s GITHUB_TOKEN="$(gh auth token)" -W '.github/workflows/your-workflow.yaml' -P your-self-hosted-runner=catthehacker/ubuntu:act-latest +``` + +## License + +Refer to the repository's license file for information on the licensing of this GitHub Action and the associated software. diff --git a/action.yaml b/action.yaml new file mode 100644 index 0000000..0cb9ec8 --- /dev/null +++ b/action.yaml @@ -0,0 +1,232 @@ +name: Ethereum Hive + +description: Run Ethereum Hive tests with a given client and simulator + +inputs: + # Test configuration + simulator: + description: 'Simulator to run (e.g. ethereum/sync)' + required: true + type: string + default: 'ethereum/sync' + client: + description: 'Client to test' + required: true + type: string + default: 'go-ethereum' + client_config: + description: 'Client configuration in YAML format' + required: false + type: string + extra_flags: + description: 'Additional flags for hive' + required: false + type: string + # Hive configuration + hive_repository: + description: 'Hive repository to use' + required: false + type: string + default: 'ethereum/hive' + hive_version: + description: 'Hive version/branch to use' + required: false + type: string + default: 'master' + go_version: + description: 'Go version used to build hive' + required: false + type: string + default: '1.21' + ## S3 upload using rclone + s3_upload: + description: 'Upload test results to S3' + required: false + default: 'false' + s3_bucket: + description: 'S3 bucket name' + required: false + type: string + s3_path: + description: 'Path prefix in S3 bucket' + required: false + type: string + default: '' + s3_public_url: + description: 'Public URL prefix for Hive View. Used to generate links to detailed results in the summary.' + required: false + type: string + default: '' + rclone_config: + description: 'Rclone config file' + required: false + type: string + default: '' # Should be base64 encoded. Example: base64 -w 0 rclone.conf + rclone_version: + description: 'Rclone version to use' + required: false + type: string + default: 'latest' + # Workflow artifact upload + workflow_artifact_upload: + description: 'Upload test results as an workflow artifact' + required: false + default: 'false' + workflow_artifact_prefix: + description: 'Prefix for the workflow artifacts. If not provided, the prefix will be the simulator and client name.' + required: false + type: string + default: '' + +runs: + using: 'composite' + steps: + - name: Install go + uses: actions/setup-go@v4 + with: + go-version: ${{ inputs.go_version }} + cache: false + + - name: Checkout hive + uses: actions/checkout@v4 + with: + repository: ethereum/hive + ref: ${{ inputs.hive_version }} + path: ./src + + - name: Build hive and hiveview + working-directory: ./src + shell: bash + run: | + go build -o hive . + go build -o hiveview ./cmd/hiveview + + - name: Create results directory + working-directory: ./src + shell: bash + run: mkdir -p results + + - name: Setup client config + if: inputs.client_config != '' + working-directory: ./src + shell: bash + run: | + echo "${{ inputs.client_config }}" > client-config.yaml + echo "CLIENT_CONFIG_FLAG=--client-file=client-config.yaml" >> $GITHUB_ENV + + - name: Run hive tests + working-directory: ./src + shell: bash + run: | + set -x + (./hive \ + --sim "${{ inputs.simulator }}" \ + --client "${{ inputs.client }}" \ + --results-root results \ + ${CLIENT_CONFIG_FLAG:-} \ + ${{ inputs.extra_flags }} \ + 2>&1 || true ) | tee hive.log + + # Check the last 10 lines of the log for the success message + if tail -n 10 hive.log | grep -q "simulation .* finished"; then + echo "RUN_SUCCESS=true" >> $GITHUB_ENV + exit 0 + else + echo "RUN_SUCCESS=false" >> $GITHUB_ENV + exit 1 + fi + + - name: Set artifact name + if: ${{ inputs.workflow_artifact_upload == 'true'}} + shell: bash + run: | + if [ -n "${{ inputs.workflow_artifact_prefix }}" ]; then + echo "WORKFLOW_ARTIFACT_PREFIX=${{ inputs.workflow_artifact_prefix }}" >> $GITHUB_ENV + else + SIMULATOR_NAME=${{ inputs.simulator }} + # replace / in simulator name with - because artifact names cannot contain / + SIMULATOR_NAME="${SIMULATOR_NAME//\//-}" + CLIENT_NAME=${{ inputs.client }} + echo "WORKFLOW_ARTIFACT_PREFIX=$SIMULATOR_NAME-$CLIENT_NAME" >> $GITHUB_ENV + fi + + - name: Upload test results as workflow artifact + if: ${{ inputs.workflow_artifact_upload == 'true'}} + uses: actions/upload-artifact@v4 + with: + name: ${{ env.WORKFLOW_ARTIFACT_PREFIX }}-results.zip + path: src/results + + - name: Upload client config as workflow artifact + if: ${{ inputs.workflow_artifact_upload == 'true' && inputs.client_config != '' }} + uses: actions/upload-artifact@v4 + with: + name: ${{ env.WORKFLOW_ARTIFACT_PREFIX }}-client-config.yaml + path: src/client-config.yaml + + - name: Setup Rclone for S3 upload + if: ${{ inputs.s3_upload == 'true' && env.RUN_SUCCESS == 'true' }} + uses: AnimMouse/setup-rclone@e4c62ff5f942e489edceaffb563832d970253322 + with: + rclone_config: ${{ inputs.rclone_config }} + version: ${{ inputs.rclone_version }} + + - name: Upload results to S3 + if: ${{ inputs.s3_upload == 'true' && env.RUN_SUCCESS == 'true' }} + shell: bash + run: | + rclone copy --no-traverse --exclude "hive.json" src/results s3:${{ inputs.s3_bucket }}/${{ inputs.s3_path }}/results + + - name: Generate hive view website assets and upload to S3 + if: ${{ inputs.s3_upload == 'true' && env.RUN_SUCCESS == 'true' }} + working-directory: ./src + shell: bash + run: | + ./hiveview --deploy hive-www + rclone copy --no-traverse hive-www s3:${{ inputs.s3_bucket }}/${{ inputs.s3_path }} + + - name: Update hive view index and upload to S3 + if: ${{ inputs.s3_upload == 'true' && env.RUN_SUCCESS == 'true' }} + working-directory: ./src + shell: bash + run: | + rclone copy --progress --transfers=100 --include "*.json" s3://${{ inputs.s3_bucket }}/${{ inputs.s3_path }}/results tmp_results + ./hiveview --listing --logdir tmp_results > listing.jsonl + rclone copy listing.jsonl s3:${{ inputs.s3_bucket }}/${{ inputs.s3_path }}/ + + - name: Generate summary + shell: bash + run: | + # Add the command executed + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Command" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "hive --sim \"${{ inputs.simulator }}\" --client \"${{ inputs.client }}\" --results-root results ${CLIENT_CONFIG_FLAG:-} ${{ inputs.extra_flags }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + + - name: Generate summary for client config + if: ${{ inputs.client_config != '' }} + shell: bash + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + echo "client-config.yaml" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`yaml" >> $GITHUB_STEP_SUMMARY + echo "${{ inputs.client_config }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "
" >> $GITHUB_STEP_SUMMARY + + - name: Convert JSON results to markdown summaries + if: env.RUN_SUCCESS == 'true' + shell: bash + run: | + set -x + ls -lah $GITHUB_ACTION_PATH/ + for json_file in src/results/*.json; do + if [ -f "$json_file" ] && [[ "$json_file" != *"hive.json" ]]; then + $GITHUB_ACTION_PATH/scripts/json-result-to-md-summary.sh "$json_file" "${{ inputs.s3_public_url }}" + cat "${json_file%.*}.md" >> $GITHUB_STEP_SUMMARY + fi + done diff --git a/example-workflows/pectra.yaml b/example-workflows/pectra.yaml new file mode 100644 index 0000000..5eed7f6 --- /dev/null +++ b/example-workflows/pectra.yaml @@ -0,0 +1,126 @@ +name: Pectra Devnet 5 +on: + push: + branches: + - main + +env: + S3_BUCKET: hive-results + S3_PATH: pectra-devnet-5 + S3_PUBLIC_URL: https://hive.ethpandaops.io/pectra-devnet-5 + INSTALL_RCLONE_VERSION: v1.68.2 + GLOBAL_EXTRA_FLAGS: >- + --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/pectra-devnet-5%40v1.1.0/fixtures_pectra-devnet-5.tar.gz + --sim.buildarg branch=pectra-devnet-5 + --client.checktimelimit=60s + --sim.parallelism=4 + CLIENT_CONFIG: | + - client: go-ethereum + nametag: prague-devnet-5 + dockerfile: git + build_args: + github: s1na/go-ethereum + tag: prague-devnet-5 + - client: besu + nametag: prague-devnet-5 + dockerfile: git + build_args: + github: siladu/besu + tag: pectra-devnet-5-interop + - client: reth + nametag: main + dockerfile: git + build_args: + github: paradigmxyz/reth + tag: main + - client: nethermind + nametag: main + dockerfile: git + build_args: + github: NethermindEth/nethermind + tag: pectra-devnet-5 + - client: erigon + nametag: pectra5 + build_args: + baseimage: erigontech/erigon + tag: pectra5 + +jobs: + rpc-compat: + runs-on: self-hosted-ghr-size-m-x64 + concurrency: + group: >- + ${{ github.head_ref }}-${{ matrix.client }}-rpc-compat + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + client: [go-ethereum, besu, reth, nethermind, erigon] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-deps + - uses: ./ + with: + client: ${{ matrix.client }} + simulator: ethereum/rpc-compat + client_config: ${{ env.CLIENT_CONFIG }} + extra_flags: ${{ env.GLOBAL_EXTRA_FLAGS }} + s3_upload: true + s3_bucket: ${{ env.S3_BUCKET }} + s3_path: ${{ env.S3_PATH }} + s3_public_url: ${{ env.S3_PUBLIC_URL }} + rclone_config: ${{ secrets.RCLONE_CONFIG }} + rclone_version: ${{ env.INSTALL_RCLONE_VERSION }} + workflow_artifact_upload: true + + eest-c-engine: + runs-on: self-hosted-ghr-size-l-x64 + concurrency: + group: >- + ${{ github.head_ref }}-${{ matrix.client }}-eest-consume-engine + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + client: [go-ethereum, besu, reth, nethermind, erigon] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-deps + - uses: ./ + with: + client: ${{ matrix.client }} + simulator: ethereum/eest/consume-engine + client_config: ${{ env.CLIENT_CONFIG }} + extra_flags: ${{ env.GLOBAL_EXTRA_FLAGS }} + s3_upload: true + s3_bucket: ${{ env.S3_BUCKET }} + s3_path: ${{ env.S3_PATH }} + rclone_config: ${{ secrets.RCLONE_CONFIG }} + rclone_version: ${{ env.INSTALL_RCLONE_VERSION }} + workflow_artifact_upload: true + + eest-c-rlp: + runs-on: self-hosted-ghr-size-l-x64 + concurrency: + group: >- + ${{ github.head_ref }}-${{ matrix.client }}-eest-consume-rlp + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + client: [go-ethereum, besu, reth, nethermind, erigon] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-deps + - uses: ./ + with: + client: ${{ matrix.client }} + simulator: ethereum/eest/consume-rlp + client_config: ${{ env.CLIENT_CONFIG }} + extra_flags: ${{ env.GLOBAL_EXTRA_FLAGS }} + s3_upload: true + s3_bucket: ${{ env.S3_BUCKET }} + s3_path: ${{ env.S3_PATH }} + rclone_config: ${{ secrets.RCLONE_CONFIG }} + rclone_version: ${{ env.INSTALL_RCLONE_VERSION }} + workflow_artifact_upload: true diff --git a/scripts/json-result-to-md-summary.sh b/scripts/json-result-to-md-summary.sh new file mode 100755 index 0000000..f436747 --- /dev/null +++ b/scripts/json-result-to-md-summary.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Check if file is provided +if [ $# -lt 1 ] || [ $# -gt 2 ]; then + echo "Usage: $0 [hiveViewURLPrefix]" + exit 1 +fi + +# Dependencies check +if ! command -v jq &> /dev/null; then + echo "Error: jq is required but not installed" + exit 1 +fi + +json_file="$1" +hive_view_prefix="${2:-}" +hive_view_prefix="${hive_view_prefix%/}" # Remove trailing slash if present + +# Convert JSON to markdown using jq +{ + echo "## Test: $(jq -r .name "$json_file")" + echo "
Description$(jq -r .description "$json_file")
" + echo + echo "### Client Versions" + jq -r '.clientVersions | to_entries[] | "- **\(.key)**: \(.value)"' "$json_file" + echo + echo "### Test Summary" + echo + echo "| Total | ✅ | ❌ |" + echo "|-------------|-----------|-----------|" + echo "| $(jq -r '.testCases | length' "$json_file") | $(jq -r '[.testCases[].summaryResult.pass] | map(select(. == true)) | length' "$json_file") | $(jq -r '[.testCases[].summaryResult.pass] | map(select(. == false)) | length' "$json_file") |" + echo + if [ -n "$hive_view_prefix" ]; then + echo "Detailed results: [$hive_view_prefix/suite.html?suiteid=$(basename "$json_file")]($hive_view_prefix/suite.html?suiteid=$(basename "$json_file"))" + fi + echo +} > "${json_file%.*}.md" + +echo "Markdown file created: ${json_file%.*}.md"