diff --git a/.github/workflows/ccip-client-compatibility-tests.yml b/.github/workflows/ccip-client-compatibility-tests.yml index b21ef82bab..40861de3b7 100644 --- a/.github/workflows/ccip-client-compatibility-tests.yml +++ b/.github/workflows/ccip-client-compatibility-tests.yml @@ -11,7 +11,7 @@ on: inputs: chainlinkVersion: description: commit SHA or tag of the Chainlink version to test - required: true + required: false type: string evmImplementations: description: comma separated list of EVM implementations to test (ignored if base64TestList is used) @@ -238,14 +238,14 @@ jobs: matrix: mirror: - name: ethereum/client-go - expression: '^(alltools-v|v)[0-9]\.[0-9]+\.[0-9]+$' + expression: '^(alltools-v|v)?[0-9]\.[0-9]+\.[0-9]+$' - name: hyperledger/besu - expression: '^[0-9]+\.[0-9]+(\.[0-9]+)?$' + expression: 'v?^[0-9]+\.[0-9]+(\.[0-9]+)?$' page_size: 300 - name: thorax/erigon - expression: '^v[0-9]+\.[0-9]+\.[0-9]+$' + expression: '^v?[0-9]+\.[0-9]+\.[0-9]+$' - name: nethermind/nethermind - expression: '^[0-9]+\.[0-9]+\.[0-9]+$' + expression: '^v?[0-9]+\.[0-9]+\.[0-9]+$' - name: tofelb/ethereum-genesis-generator expression: '^[0-9]+\.[0-9]+\.[0-9]+(\-slots\-per\-epoch)?' steps: @@ -330,7 +330,7 @@ jobs: - name: Set Up ecrimagefetcher shell: bash run: | - go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.0.1 + go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.50.2 - name: Get latest docker images from ECR if: ${{ github.event.inputs.base64TestList == '' }} env: @@ -360,7 +360,7 @@ jobs: if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then # 2.60.0 and 2.60.1 are ignored as they stopped working with CL node - erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "v2.60.2") echo "ERIGON_IMAGES=$erigon_images" >> $GITHUB_ENV echo "Erigon latest images: $erigon_images" fi @@ -463,20 +463,26 @@ jobs: run-client-compatibility-matrix: name: CCIP Compatibility with ${{ matrix.evm_node.visible_name }} - if: always() && needs.should-run.outputs.should_run == 'true' + if: always() && needs.should-run.outputs.should_run == 'true' && (needs.build-chainlink.result == 'success' || needs.build-chainlink.result == 'skipped') && needs.prepare-compatibility-matrix.outputs.matrix != '' environment: integration permissions: checks: write pull-requests: write id-token: write contents: read - needs: [build-chainlink, prepare-compatibility-matrix, should-run, select-versions] + needs: + - build-chainlink + - prepare-compatibility-matrix + - should-run + - select-versions env: + SELECTED_NETWORKS: SIMULATED,SIMULATED_1,SIMULATED_2 CHAINLINK_COMMIT_SHA: ${{ needs.select-versions.outputs.chainlink_version }} CHAINLINK_ENV_USER: ${{ github.actor }} TEST_LOG_LEVEL: debug strategy: fail-fast: false + max-parallel: 10 matrix: evm_node: ${{fromJson(needs.prepare-compatibility-matrix.outputs.matrix)}} runs-on: ubuntu-latest @@ -486,24 +492,15 @@ jobs: with: repository: smartcontractkit/ccip ref: ${{ needs.select-versions.outputs.chainlink_version }} - - name: Prepare Base64 TOML override - uses: ./.github/actions/setup-create-base64-config + - name: Setup GAP for Grafana + uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 with: - runId: ${{ github.run_id }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - selectedNetworks: ${{ matrix.evm_node.networks }} - chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - - name: Prepare Base64 TOML override for CCIP secrets - uses: ./.github/actions/setup-create-base64-config-ccip - id: setup_create_base64_config_ccip - with: - runId: ${{ github.run_id }} - selectedNetworks: ${{ matrix.evm_node.networks }} - testLogCollect: ${{ vars.TEST_LOG_COLLECT }} - chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} - logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - customEvmNodes: ${{ matrix.evm_node.docker_image }} + # aws inputs + aws-region: ${{ secrets.AWS_REGION }} + aws-role-arn: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }} + api-gateway-host: ${{ secrets.AWS_API_GW_HOST_GRAFANA }} + # other inputs + duplicate-authorization-header: "true" - name: Prepare test log name run: | replace_special_chars() { @@ -525,27 +522,46 @@ jobs: echo "$modified_string" } - echo "TEST_LOG_NAME=$(replace_special_chars "ccip-${{ matrix.evm_node.name }}-test-logs")" >> $GITHUB_ENV - - name: Print Test details - ${{ matrix.evm_node.docker_image }} - run: | - echo "EVM Implementation Docker Image: ${{ matrix.evm_node.docker_image }}" - echo "EVM Implementation Networks: ${{ matrix.evm_node.networks }}" - echo "Test identifier: ${{ matrix.evm_node.name }}" + echo "TEST_LOG_NAME=$(replace_special_chars "${{ matrix.evm_node.product }}-${{ matrix.evm_node.docker_image }}-test-logs")" >> $GITHUB_ENV + # - name: Collect Workflow Telemetry + # uses: catchpoint/workflow-telemetry-action@v2 + # with: + # comment_on_pr: false + # theme: 'dark' + - name: Prepare Base64 CCIP TOML secrets + uses: ./.github/actions/setup-create-base64-config-ccip + id: setup_create_base64_config_ccip + with: + runId: ${{ github.run_id }} + testLogCollect: ${{ vars.TEST_LOG_COLLECT }} + selectedNetworks: SIMULATED_1,SIMULATED_2 + chainlinkVersion: ${{ needs.select-versions.outputs.chainlink_version }} + logstreamLogTargets: ${{ vars.LOGSTREAM_LOG_TARGETS }} - name: Run Tests - uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@d38226be720c5ccc1ff4d3cee40608ebf264cd59 # v2.3.26 + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/run-tests@aa8eea635029ab8d95abd3c206f56dae1e22e623 # v2.3.28 env: BASE64_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} TEST_BASE64_CONFIG_OVERRIDE: ${{ steps.setup_create_base64_config_ccip.outputs.base64_config }} + DEBUG_RESTY: false with: - test_command_to_run: cd ./integration-tests && go test -timeout 30m -count=1 -json -test.parallel=2 ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci + test_command_to_run: cd ./integration-tests && touch .root_dir && go test -timeout 30m -count=1 -json ${{ matrix.evm_node.run }} 2>&1 | tee /tmp/gotest.log | gotestloghelper -ci -singlepackage -hidepassingtests=false -hidepassinglogs test_download_vendor_packages_command: cd ./integration-tests && go mod download + test_config_chainlink_version: ${{ needs.select-versions.outputs.chainlink_version }} + test_config_selected_networks: ${{ env.SELECTED_NETWORKS}} + test_config_logging_run_id: ${{ github.run_id }} + test_config_test_log_collect: ${{ vars.TEST_LOG_COLLECT }} + test_config_logstream_log_targets: "file" + test_config_private_ethereum_network_execution_layer: ${{ matrix.evm_node.eth_implementation || 'geth' }} + test_config_private_ethereum_network_custom_docker_image: ${{ matrix.evm_node.docker_image }} + cl_repo: ${{ env.CHAINLINK_IMAGE }} + cl_image_tag: ${{ needs.select-versions.outputs.chainlink_version }} aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }} - artifacts_name: ${{ env.TEST_LOG_NAME }} + artifacts_name: ${{ env.TEST_LOG_NAME }} artifacts_location: | ./integration-tests/smoke/logs/ ./integration-tests/ccip-tests/smoke/logs/* /tmp/gotest.log - publish_check_name: ${{ matrix.evm_node.name }} + publish_check_name: ${{ matrix.evm_node.product }}-${{ matrix.evm_node.eth_implementation }} token: ${{ secrets.GITHUB_TOKEN }} go_mod_path: ./integration-tests/go.mod cache_key_id: core-e2e-${{ env.MOD_CACHE_VERSION }} @@ -554,16 +570,16 @@ jobs: QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }} QA_KUBECONFIG: "" should_tidy: "false" - DEFAULT_LOKI_TENANT_ID: ${{ vars.LOKI_TENANT_ID }} - DEFAULT_LOKI_ENDPOINT: ${{ secrets.LOKI_URL_CI }} - DEFAULT_LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }} + go_coverage_src_dir: /var/tmp/go-coverage + go_coverage_dest_dir: ${{ github.workspace }}/.covdata DEFAULT_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} - DEFAULT_GRAFANA_BASE_URL: ${{ vars.GRAFANA_URL }} + DEFAULT_GRAFANA_BASE_URL: "http://localhost:8080/primary" DEFAULT_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" - DEFAULT_PYROSCOPE_SERVER_URL: ${{ !startsWith(github.ref, 'refs/tags/') && '' || secrets.QA_PYROSCOPE_INSTANCE }} # Avoid sending blank envs https://github.com/orgs/community/discussions/25725 - DEFAULT_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - DEFAULT_PYROSCOPE_ENVIRONMENT: ci-ccip-bidirectional-lane-${{ matrix.evm_node.name }} - DEFAULT_PYROSCOPE_ENABLED: 'true' + DEFAULT_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} + + - name: Print failed test summary + if: always() + uses: smartcontractkit/chainlink-github-actions/chainlink-testing-framework/show-test-summary@75a9005952a9e905649cfb5a6971fd9429436acd # v2.3.25 - name: Print failed test summary if: always() diff --git a/.github/workflows/client-compatibility-tests.yml b/.github/workflows/client-compatibility-tests.yml index 2e27f49a39..96697ede1b 100644 --- a/.github/workflows/client-compatibility-tests.yml +++ b/.github/workflows/client-compatibility-tests.yml @@ -29,7 +29,7 @@ on: type: string env: - CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-ccip + CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com MOD_CACHE_VERSION: 2 @@ -283,14 +283,14 @@ jobs: matrix: mirror: - name: ethereum/client-go - expression: '^(alltools-v|v)[0-9]\.[0-9]+\.[0-9]+$' + expression: '^(alltools-v|v)?[0-9]\.[0-9]+\.[0-9]+$' - name: hyperledger/besu - expression: '^[0-9]+\.[0-9]+(\.[0-9]+)?$' + expression: 'v?^[0-9]+\.[0-9]+(\.[0-9]+)?$' page_size: 300 - name: thorax/erigon - expression: '^v[0-9]+\.[0-9]+\.[0-9]+$' + expression: '^v?[0-9]+\.[0-9]+\.[0-9]+$' - name: nethermind/nethermind - expression: '^[0-9]+\.[0-9]+\.[0-9]+$' + expression: '^v?[0-9]+\.[0-9]+\.[0-9]+$' - name: tofelb/ethereum-genesis-generator expression: '^[0-9]+\.[0-9]+\.[0-9]+(\-slots\-per\-epoch)?' - name: ghcr.io/paradigmxyz/reth @@ -386,7 +386,7 @@ jobs: - name: Set Up ecrimagefetcher shell: bash run: | - go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.0.1 + go install github.com/smartcontractkit/chainlink-testing-framework/tools/ecrimagefetcher@v1.50.2 - name: Get latest docker images from ECR if: ${{ github.event.inputs.base64TestList == '' }} env: @@ -416,7 +416,7 @@ jobs: if [[ "$ETH_IMPLEMENTATIONS" == *"erigon"* ]]; then # 2.60.0 and 2.60.1 are ignored as they stopped working with CL node - erigon_images=$(ecrimagefetcher 'thorax/erigon' '^v[0-9]+\.[0-9]+\.[0-9]+$' ${{ env.LATEST_IMAGE_COUNT }} "v2.60.2") echo "ERIGON_IMAGES=$erigon_images" >> $GITHUB_ENV echo "Erigon latest images: $erigon_images" fi @@ -605,7 +605,7 @@ jobs: - name: Checkout the repo uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: - repository: smartcontractkit/chainlink + repository: smartcontractkit/ccip ref: ${{ needs.select-versions.outputs.chainlink_version }} - name: Setup GAP for Grafana uses: smartcontractkit/.github/actions/setup-gap@d316f66b2990ea4daa479daa3de6fc92b00f863e # setup-gap@0.3.2 @@ -669,18 +669,11 @@ jobs: E2E_TEST_SELECTED_NETWORK: ${{ env.SELECTED_NETWORKS}} E2E_TEST_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }} E2E_TEST_CHAINLINK_VERSION: ${{ needs.select-versions.outputs.chainlink_image_version }} - E2E_TEST_LOKI_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }} - E2E_TEST_LOKI_ENDPOINT: https://${{ secrets.GRAFANA_INTERNAL_HOST }}/loki/api/v1/push - E2E_TEST_LOKI_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }} E2E_TEST_GRAFANA_DASHBOARD_URL: "/d/ddf75041-1e39-42af-aa46-361fe4c36e9e/ci-e2e-tests-logs" E2E_TEST_GRAFANA_BEARER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }} - E2E_TEST_PYROSCOPE_SERVER_URL: ${{ secrets.QA_PYROSCOPE_INSTANCE }} - E2E_TEST_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }} - E2E_TEST_PYROSCOPE_ENVIRONMENT: ci-client-compatability-${{ matrix.eth_client }}-testnet - E2E_TEST_PYROSCOPE_ENABLED: "true" E2E_TEST_LOGGING_RUN_ID: ${{ github.run_id }} E2E_TEST_LOG_COLLECT: ${{ vars.TEST_LOG_COLLECT }} - E2E_TEST_LOG_STREAM_LOG_TARGETS: ${{ vars.LOGSTREAM_LOG_TARGETS }} + E2E_TEST_LOG_STREAM_LOG_TARGETS: "file" E2E_TEST_PRIVATE_ETHEREUM_EXECUTION_LAYER: ${{ matrix.evm_node.eth_implementation || 'geth' }} E2E_TEST_PRIVATE_ETHEREUM_ETHEREUM_VERSION: auto_fill # Auto fill the version based on the docker image E2E_TEST_PRIVATE_ETHEREUM_CUSTOM_DOCKER_IMAGE: ${{ matrix.evm_node.docker_image }}