improve discord notifications #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #name: Run assertoor tests on kurtosis with kubernetes backend | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| id: | ||
| type: string | ||
| description: "The test identifier." | ||
| required: true | ||
| name: | ||
| type: string | ||
| description: "The test name." | ||
| required: true | ||
| pairs: | ||
| type: string | ||
| description: "The client pairs to use (format: consensus1-execution1,consensus2-execution2,...)." | ||
| required: true | ||
| worker: | ||
| type: string | ||
| description: "The worker to use (default: ubuntu-latest)." | ||
| default: "ubuntu-latest" | ||
| backend: | ||
| type: string | ||
| description: "The backend type to use (docker, kubernetes)." | ||
| default: "docker" | ||
| kubeCluster: | ||
| type: string | ||
| description: "The name of the cluster to run the test on." | ||
| required: true | ||
| kubeStorageClass: | ||
| type: string | ||
| description: "The kubernetes storage class to run the kurtosis engine with." | ||
| required: true | ||
| clients: | ||
| type: string | ||
| description: "The clients config to use (default: clients/latest.yaml)." | ||
| default: "clients/latest.yaml" | ||
| kurtosis: | ||
| type: string | ||
| description: "The kurtosis network config to use (default: kurtosis-config/default.yaml)." | ||
| default: "kurtosis-config/default.yaml" | ||
| kurtosis_branch: | ||
| type: string | ||
| description: "The branch name for the kurtosis ethereum package to use (default: )." | ||
| default: "" | ||
| assertoor_tests: | ||
| type: string | ||
| description: "The list of assertoor test files to run." | ||
| required: true | ||
| send_notification: | ||
| type: string | ||
| description: "Send discord notification on test failure (default: false)." | ||
| default: "false" | ||
| secrets: | ||
| KUBECONFIG: | ||
| description: 'Kubernetes config' | ||
| required: true | ||
| DISCORD_HOOK: | ||
| description: 'Discord hook' | ||
| required: false | ||
| GITHUB_TOKEN: | ||
| description: 'Github hook' | ||
| required: false | ||
| jobs: | ||
| run_test: | ||
| name: "Run ${{ inputs.id }}" | ||
| runs-on: ${{ fromJson(inputs.worker) }} | ||
| timeout-minutes: 1440 | ||
| steps: | ||
| - name: Checkout Repository | ||
| uses: actions/checkout@v4 | ||
| - name: "Install shell dependencies" | ||
| shell: bash | ||
| run: | | ||
| if [ -z "$(which yq)" ]; then | ||
| sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq | ||
| fi | ||
| if [ -z "$(which envsubst)" ]; then | ||
| sudo apt-get update | ||
| sudo apt-get install gettext-base | ||
| fi | ||
| - name: "Generate kurtosis config with client pairs: ${{ inputs.pairs }}" | ||
| shell: bash | ||
| id: clients | ||
| run: | | ||
| clients_file="${{ inputs.clients }}" | ||
| kurtosis_file="${{ inputs.kurtosis }}" | ||
| pairs="${{ inputs.pairs }}" | ||
| mkdir -p ./temp | ||
| touch ./temp/participants.yaml | ||
| echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT | ||
| if [ ! -z "$pairs" ]; then | ||
| resolve_image() { | ||
| image="$1" | ||
| if [ ! -z "$(echo "$1" | grep -E '^.*::.*$')" ]; then | ||
| repository=$(echo "$1" | sed 's/^\(.*\)::\(.*\)$/\1/') | ||
| tag_pattern=$(echo "$1" | sed 's/^\(.*\)::\(.*\)$/\2/') | ||
| while read tag; do | ||
| if [ ! -z "$(echo "$tag" | grep -E $tag_pattern)" ]; then | ||
| image="${repository}:${tag}" | ||
| break | ||
| fi | ||
| done <<< $(curl --silent "https://hub.docker.com/v2/repositories/${repository}/tags?page_size=1000" | jq -r ".results[].name") | ||
| fi | ||
| echo "$image" | ||
| } | ||
| echo "participants:" >> ./temp/participants.yaml | ||
| touch ./temp/clients_summary.txt | ||
| client_idx=0 | ||
| for pair in $(echo $pairs | tr "," "\n") | ||
| do | ||
| client_idx=$(expr $client_idx + 1) | ||
| if [ "$pair" == "random" ]; then | ||
| cl_type="random" | ||
| el_type="random" | ||
| else | ||
| cl_type=$(echo "$pair" | sed 's/^\(.*\)-\(.*\)$/\1/') | ||
| el_type=$(echo "$pair" | sed 's/^\(.*\)-\(.*\)$/\2/') | ||
| fi | ||
| if [[ "$cl_type" == *"/"* ]]; then | ||
| vc_type=$(echo "$cl_type" | sed 's/^\(.*\)\/\(.*\)$/\2/') | ||
| cl_type=$(echo "$cl_type" | sed 's/^\(.*\)\/\(.*\)$/\1/') | ||
| else | ||
| vc_type="" | ||
| fi | ||
| if [ "$cl_type" == "random" ]; then | ||
| cl_type=$(cat $clients_file | yq -r ".consensus | keys" | shuf | tr -d ' -' | head -n 1) | ||
| fi | ||
| if [ "$vc_type" == "random" ]; then | ||
| vc_type=$(cat $clients_file | yq -r ".consensus | keys" | shuf | tr -d ' -' | head -n 1) | ||
| fi | ||
| if [ "$el_type" == "random" ]; then | ||
| el_type=$(cat $clients_file | yq -r ".execution | keys" | shuf | tr -d ' -' | head -n 1) | ||
| fi | ||
| cl_image="$(cat $clients_file | yq ".consensus.${cl_type}.image")" | ||
| if [ "$cl_image" == "null" ]; then | ||
| echo "Unknown CL client type: $cl_type" | ||
| exit 1 | ||
| fi | ||
| cl_image=$(resolve_image "$cl_image") | ||
| if [ -z "$vc_type" ]; then | ||
| vc_image="" | ||
| else | ||
| vc_image="$(cat $clients_file | yq ".consensus.${vc_type}.vc_image // .consensus.${vc_type}.image")" | ||
| if [ "$vc_image" == "null" ]; then | ||
| echo "Unknown VC client type: $vc_type" | ||
| exit 1 | ||
| fi | ||
| vc_image=$(resolve_image "$vc_image") | ||
| fi | ||
| el_image="$(cat $clients_file | yq ".execution.${el_type}.image")" | ||
| if [ "$el_image" == "null" ]; then | ||
| echo "Unknown EL client type: $el_type" | ||
| exit 1 | ||
| fi | ||
| el_image=$(resolve_image "$el_image") | ||
| echo " - el_type: $el_type" >> ./temp/participants.yaml | ||
| echo " el_image: $el_image" >> ./temp/participants.yaml | ||
| el_extra_params="$(cat $clients_file | yq ".execution.${el_type}.params")" | ||
| if [ "$el_extra_params" != "null" ]; then | ||
| echo " el_extra_params: $el_extra_params" >> ./temp/participants.yaml | ||
| fi | ||
| echo " cl_type: $cl_type" >> ./temp/participants.yaml | ||
| echo " cl_image: $cl_image" >> ./temp/participants.yaml | ||
| cl_extra_params="$(cat $clients_file | yq ".consensus.${cl_type}.params")" | ||
| if [ "$cl_extra_params" != "null" ]; then | ||
| echo " cl_extra_params: $cl_extra_params" >> ./temp/participants.yaml | ||
| fi | ||
| if [ ! -z "$vc_type" ]; then | ||
| echo " vc_type: $vc_type" >> ./temp/participants.yaml | ||
| echo " vc_image: $vc_image" >> ./temp/participants.yaml | ||
| vc_extra_params="$(cat $clients_file | yq ".consensus.${cl_type}.vc_params")" | ||
| if [ "$vc_extra_params" != "null" ]; then | ||
| echo " vc_extra_params: $vc_extra_params" >> ./temp/participants.yaml | ||
| fi | ||
| fi | ||
| echo " count: 1" >> ./temp/participants.yaml | ||
| echo "Client $client_idx" | ||
| echo "CL: $cl_type ($cl_image)" | ||
| if [ ! -z "$vc_type" ]; then | ||
| echo "VC: $vc_type ($vc_image)" | ||
| fi | ||
| echo "EL: $el_type ($el_image)" | ||
| participant="${el_type}-${cl_type}" | ||
| if [ ! -z "$vc_type" ]; then | ||
| participant="${participant}-${vc_type}" | ||
| fi | ||
| echo "participant $client_idx: $participant" >> ./temp/clients_summary.txt | ||
| done | ||
| fi | ||
| export PUBURL="https://raw.githubusercontent.com/${{ github.repository }}/${{ github.sha }}" | ||
| cp ./temp/participants.yaml ./temp/test-network.yaml | ||
| cat $kurtosis_file | envsubst >> ./temp/test-network.yaml | ||
| kurtosis_cfg="$(cat ./temp/test-network.yaml | yq '.assertoor_params.tests=[]')" | ||
| tests_json=$( | ||
| cat <<"EOF" | ||
| ${{ inputs.assertoor_tests }} | ||
| EOF | ||
| ) | ||
| test_index=0 | ||
| while read assertoorTest; do | ||
| kurtosis_cfg="$(echo "$kurtosis_cfg" | yq ".assertoor_params.tests[$test_index]=\"$PUBURL/$assertoorTest\"")" | ||
| test_index=$(expr $test_index + 1) | ||
| done <<< $(echo "$tests_json" | jq -r '.[]') | ||
| echo "$kurtosis_cfg" > ./temp/test-network.yaml | ||
| echo "" | ||
| cat ./temp/test-network.yaml | ||
| - name: Setup Kurtosis | ||
| shell: bash | ||
| run: | | ||
| echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list | ||
| sudo apt update | ||
| sudo apt install kurtosis-cli | ||
| kurtosis analytics disable | ||
| - name: Setup kubectl | ||
| if: ${{ inputs.backend == 'kubernetes' }} | ||
| uses: tale/kubectl-action@v1 | ||
| with: | ||
| base64-kube-config: ${{ secrets.KUBECONFIG }} | ||
| - name: Check Kubernetes engine | ||
| if: ${{ inputs.backend == 'kubernetes' }} | ||
| shell: bash | ||
| run: | | ||
| set -e | ||
| kubectl config use-context ${{ inputs.kubeCluster }} | ||
| kubectl get nodes | ||
| echo "Kubernetes engine is ready!" | ||
| - name: Configure kurtosis to use kubernetes backend | ||
| if: ${{ inputs.backend == 'kubernetes' }} | ||
| shell: bash | ||
| run: | | ||
| kurtosis_config=$(kurtosis config path) | ||
| echo "config-version: 2" > $kurtosis_config | ||
| echo "should-send-metrics: false" >> $kurtosis_config | ||
| echo "kurtosis-clusters:" >> $kurtosis_config | ||
| echo " docker:" >> $kurtosis_config | ||
| echo " type: \"docker\"" >> $kurtosis_config | ||
| echo " cloud:" >> $kurtosis_config | ||
| echo " type: \"kubernetes\"" >> $kurtosis_config | ||
| echo " config:" >> $kurtosis_config | ||
| echo " kubernetes-cluster-name: \"${{ inputs.kubeCluster }}\"" >> $kurtosis_config | ||
| echo " storage-class: \"${{ inputs.kubeStorageClass }}\"" >> $kurtosis_config | ||
| echo " enclave-size-in-megabytes: 200" >> $kurtosis_config | ||
| cat $kurtosis_config | ||
| kurtosis cluster set cloud | ||
| kurtosis gateway & | ||
| kurtosis engine status | ||
| kurtosis enclave ls || ( kurtosis engine restart && kurtosis enclave ls ) | ||
| - name: Run Kurtosis | ||
| shell: bash | ||
| id: services | ||
| run: | | ||
| if [ "${{ inputs.backend }}" == "kubernetes" ]; then | ||
| kurtosis gateway & | ||
| fi | ||
| kurtosis_package="github.com/kurtosis-tech/ethereum-package" | ||
| if [ ! -z "${{ inputs.kurtosis_branch }}" ]; then | ||
| kurtosis_package="${kurtosis_package}@${{ inputs.kurtosis_branch }}" | ||
| fi | ||
| kurtosis run $kurtosis_package --enclave assertoor-${{ github.run_id }}-${{ inputs.id }} --args-file ./temp/test-network.yaml --image-download always --non-blocking-tasks --verbosity DETAILED | ||
| enclave_dump=$(kurtosis enclave inspect assertoor-${{ github.run_id }}-${{ inputs.id }}) | ||
| assertoor_url=$(echo "$enclave_dump" | grep assertoor | grep http | sed 's/.*\(http:\/\/[0-9.:]\+\).*/\1/') | ||
| echo "assertoor_url: ${assertoor_url}" | ||
| echo "assertoor_url=${assertoor_url}" >> $GITHUB_OUTPUT | ||
| - name: Assertoor Status Check | ||
| id: test_result | ||
| uses: ethpandaops/assertoor-github-action@v1 | ||
| with: | ||
| kurtosis_enclave_name: "assertoor-${{ github.run_id }}-${{ inputs.id }}" | ||
| - name: Generate enclave dump | ||
| shell: bash | ||
| run: | | ||
| mkdir -p ./temp/dump | ||
| cd ./temp/dump | ||
| cp ../test-network.yaml ./kurtosis-params.yaml | ||
| kurtosis enclave dump assertoor-${{ github.run_id }}-${{ inputs.id }} | ||
| - name: Upload dump artifact | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: "kurtosis-enclave-dump-${{ inputs.id }}" | ||
| path: ./temp/dump | ||
| - name: Remove kurtosis enclave | ||
| if: ${{ always() }} | ||
| shell: bash | ||
| run: | | ||
| kurtosis enclave rm -f assertoor-${{ github.run_id }}-${{ inputs.id }} || true | ||
| - name: Send discord notification (on failure) | ||
| if: ${{ steps.test_result.outputs.result == 'failure' && inputs.send_notification == 'true' }} | ||
| shell: bash | ||
| run: | | ||
| if [ -z "$DISCORD_HOOK" ]; then | ||
| echo "discord hook missing" | ||
| exit 1 | ||
| fi | ||
| hookjson="{}" | ||
| hookjson=$(echo "$hookjson" | jq -c ".username|=$(echo -n "Assertoor Test Notification" | jq -R -s '.')") | ||
| #hookjson=$(echo "$hookjson" | jq -c ".username|=$(echo -n "Assertoor Test Notification" | jq -R -s '.')") | ||
| touch ./temp/notification_text.txt | ||
| echo "test id: ${{ inputs.id }}" >> ./temp/notification_text.txt | ||
| start_time="${{ steps.clients.outputs.start_time }}" | ||
| end_time="$(date +%s)" | ||
| runtime=$((end_time-start_time)) | ||
| echo "run time: $(printf '%02dh:%02dm:%02ds\n' $((runtime/3600)) $((runtime%3600/60)) $((runtime%60)))" >> ./temp/notification_text.txt | ||
| cat ./temp/clients_summary.txt >> ./temp/notification_text.txt | ||
| echo "" > ./temp/notification_text.txt | ||
| test_status=$( | ||
| cat <<"EOF" | ||
| ${{ steps.test_result.outputs.test_overview }} | ||
| EOF | ||
| ) | ||
| echo "$test_status" > ./temp/notification_text.txt | ||
| msgjson="{}" | ||
| msgjson=$(echo "$msgjson" | jq -c ".title|=$(echo -n "Test Failure: ${{ inputs.name }}" | jq -R -s '.')") | ||
| msgjson=$(echo "$msgjson" | jq -c ".description|=$(cat ./temp/notification_text.txt | jq -R -s '.')") | ||
| jobs=$(gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id}}/attempts/${{ github.run_attempt }}/jobs) | ||
| job_id=$(echo $jobs | jq -r '.jobs[] | select(.runner_name=="${{ runner.name }}") | .id') | ||
| msgjson=$(echo "$msgjson" | jq -c ".url|=$(echo -n "https://github.com/${{ github.repository_owner }}/${{ github.repository }}/actions/runs/${{ github.run_id}}/job/${job_id}" | jq -R -s '.')") | ||
| hookjson=$(echo "$hookjson" | jq -c ".embeds|=[${msgjson}]") | ||
| echo $hookjson | jq | ||
| curl -X POST -H 'Content-Type: application/json' -d "$hookjson" $DISCORD_HOOK | ||
| env: | ||
| DISCORD_HOOK: ${{ secrets.DISCORD_HOOK }} | ||
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Return test result | ||
| shell: bash | ||
| run: | | ||
| test_result="${{ steps.test_result.outputs.result }}" | ||
| test_status=$( | ||
| cat <<"EOF" | ||
| ${{ steps.test_result.outputs.test_overview }} | ||
| EOF | ||
| ) | ||
| failed_test_status=$( | ||
| cat <<"EOF" | ||
| ${{ steps.test_result.outputs.failed_test_details }} | ||
| EOF | ||
| ) | ||
| echo "Test Result: $test_result" | ||
| echo "$test_status" | ||
| if ! [ "$test_result" == "success" ]; then | ||
| echo "" | ||
| echo "Failed Test Task Status:" | ||
| echo "$failed_test_status" | ||
| echo "" | ||
| echo "See 'Await test completion' task for detailed logs about this failure!" | ||
| echo "" | ||
| exit 1 # fail action | ||
| fi | ||
