Skip to content
Stable Components Certification Tests

Stable Components Certification Tests #22148

Sign in to view logs

Stable Components Certification Tests

Stable Components Certification Tests #22148

Workflow file for this run

.github/workflows/certification.yml at b05e19a
# ------------------------------------------------------------
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------
name: Stable Components Certification Tests
on:
repository_dispatch:
types: [certification-test]
workflow_dispatch:
schedule:
- cron: '25 */8 * * *'
push:
branches:
- 'release-*'
pull_request:
branches:
- 'main'
- 'release-*'
jobs:
# Based on whether this is a PR or a scheduled run, we will run a different
# subset of the certification tests. This allows all the tests not requiring
# secrets to be executed on pull requests.
generate-matrix:
runs-on: ubuntu-22.04
steps:
- name: Parse repository_dispatch payload
if: github.event_name == 'repository_dispatch'
working-directory: ${{ github.workspace }}
run: |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
fi
- name: Check out code
uses: actions/checkout@v4
with:
repository: ${{ env.CHECKOUT_REPO }}
ref: ${{ env.CHECKOUT_REF }}
- name: Generate test matrix
id: generate-matrix
env:
VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
run: |
if [ -z "$VAULT_NAME" ]; then
# Do not include cloud tests when credentials are not available
node .github/scripts/test-info.mjs certification false
else
# Include cloud tests
node .github/scripts/test-info.mjs certification true
fi
- name: Create PR comment
if: env.PR_NUMBER != ''
uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba
with:
header: ${{ github.run_id }}
number: ${{ env.PR_NUMBER }}
GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
message: |
# Components certification test
🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**
Commit ref: ${{ env.CHECKOUT_REF }}
outputs:
test-matrix: ${{ steps.generate-matrix.outputs.test-matrix }}
certification:
name: ${{ matrix.component }} certification
# Add "id-token" with the intended permissions.
# Needed by the 'Authenticate to Google Cloud' step.
permissions:
contents: 'read'
id-token: 'write'
runs-on: ubuntu-22.04
env:
UNIQUE_ID: ${{github.run_id}}-${{github.run_attempt}}
GOCOV_VER: "v1.1.0"
GOTESTSUM_VER: "v1.9.0"
defaults:
run:
shell: bash
needs:
- generate-matrix
strategy:
fail-fast: false # Keep running even if one component fails
matrix:
include: ${{ fromJson(needs.generate-matrix.outputs.test-matrix) }}
steps:
- name: Set default payload repo and ref
run: |
echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
- name: Parse repository_dispatch payload
if: github.event_name == 'repository_dispatch'
run: |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
fi
- name: Check out code
uses: actions/checkout@v4
with:
repository: ${{ env.CHECKOUT_REPO }}
ref: ${{ env.CHECKOUT_REF }}
- name: Configure environment
run: |
# Output file
echo "TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report" >> $GITHUB_ENV
# Certification test and source path
TEST_COMPONENT=$(echo "${{ matrix.component }}" | sed -E 's/\./\//g')
echo "TEST_PATH=tests/certification/${TEST_COMPONENT}" >> $GITHUB_ENV
SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}"
echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV
# converts slashes to dots in this string, so that it doesn't consider them sub-folders
SOURCE_PATH_LINEAR=$(echo "$SOURCE_PATH" |sed 's#/#\.#g')
echo "SOURCE_PATH_LINEAR=$SOURCE_PATH_LINEAR" >> $GITHUB_ENV
# Current time (used by Terraform)
echo "CURRENT_TIME=$(date --rfc-3339=date)" >> ${GITHUB_ENV}
- uses: Azure/login@v1
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
if: matrix.required-secrets != ''
# Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal:
# az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID
# Using az cli to query keyvault as Azure/get-keyvault-secrets@v1 is deprecated
- name: Setup secrets
if: matrix.required-secrets != ''
env:
VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
run: |
secrets="${{ matrix.required-secrets }}"
for secretName in $(echo -n $secrets | tr ',' ' '); do
value=$(az keyvault secret show \
--name $secretName \
--vault-name $VAULT_NAME \
--query value \
--output tsv)
echo "::add-mask::$value"
echo "$secretName=$value" >> $GITHUB_OUTPUT
echo "$secretName=$value" >> $GITHUB_ENV
done
# Authenticate with GCP Workload Identity Pool
# Exports GCP ENV Vars:
# - GCP_PROJECT
# - GOOGLE_APPLICATION_CREDENTIALS
- id: 'auth'
if: matrix.require-gcp-credentials == 'true'
name: 'Authenticate to Google Cloud'
uses: 'google-github-actions/auth@v1'
with:
token_format: 'access_token'
workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_NAME }}
service_account: ${{ secrets.GCP_WIF_SA_EMAIL }}
create_credentials_file: true
export_environment_variables: true
cleanup_credentials: true
# Download the required certificates into files, and set env var pointing to their names
- name: Setup certs
if: matrix.required-certs != ''
working-directory: ${{ env.TEST_PATH }}
run: |
for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
CERT_FILE=$(mktemp --suffix .pfx)
echo "Downloading cert $CERT_NAME into file $CERT_FILE"
rm $CERT_FILE && \
az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE
echo 'Setting $CERT_NAME to' "$CERT_FILE"
echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV
done
- name: Setup Terraform
uses: hashicorp/[email protected]
if: matrix.require-terraform == 'true'
- name: Set Cloudflare env vars
if: matrix.require-cloudflare-credentials == 'true'
run: |
echo "CLOUDFLARE_ACCOUNT_ID=${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" >> $GITHUB_ENV
echo "CLOUDFLARE_API_TOKEN=${{ secrets.CLOUDFLARE_API_TOKEN }}" >> $GITHUB_ENV
- name: Set AWS env vars
if: matrix.require-aws-credentials == 'true'
run: |
echo "AWS_REGION=us-west-1" >> $GITHUB_ENV
echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> $GITHUB_ENV
echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> $GITHUB_ENV
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
if: matrix.require-aws-credentials == 'true'
with:
aws-access-key-id: "${{ secrets.AWS_ACCESS_KEY }}"
aws-secret-access-key: "${{ secrets.AWS_SECRET_KEY }}"
aws-region: "${{ env.AWS_REGION }}"
- name: Set up Go
id: setup-go
uses: actions/setup-go@v4
with:
go-version-file: 'go.mod'
cache: 'false'
- name: Download Go dependencies
working-directory: ${{ env.TEST_PATH }}
run: |
go mod download
go install github.com/axw/gocov/gocov@${{ env.GOCOV_VER }}
go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VER }}
- name: Run setup script
if: matrix.setup-script != ''
run: .github/scripts/components-scripts/${{ matrix.setup-script }}
- name: Catch setup failures
if: failure()
run: |
echo "CERTIFICATION_FAILURE=true" >> $GITHUB_ENV
- name: Run tests
continue-on-error: false
working-directory: ${{ env.TEST_PATH }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY }}
AWS_REGION: "${{ env.AWS_REGION }}"
run: |
echo "Running certification tests for ${{ matrix.component }} ... "
echo "Source Package: " ${{ matrix.source-pkg }}
export GOLANG_PROTOBUF_REGISTRATION_CONFLICT=ignore
set +e
gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json \
--junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.xml --format standard-quiet -- \
-coverprofile=cover.out -covermode=set -tags=certtests,unit -timeout=30m -coverpkg=${{ matrix.source-pkg }}
status=$?
echo "Completed certification tests for ${{ matrix.component }} ... "
if test $status -ne 0; then
echo "Setting CERTIFICATION_FAILURE"
export CERTIFICATION_FAILURE=true
fi
set -e
mkdir -p tmp/cert_code_cov_files
cat cover.out >> tmp/cert_code_cov_files/${{ env.SOURCE_PATH_LINEAR }}.out
cat tmp/cert_code_cov_files/${{ env.SOURCE_PATH_LINEAR }}.out
# Fail the step if we found no test to run
if grep -q "\[no test files\]" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.json ; then
echo "::error:: No certification test file was found for component ${{ matrix.component }}"
exit -1
fi
for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
CERT_FILE=$(printenv $CERT_NAME)
echo "Cleaning up the certificate file $CERT_FILE..."
rm $CERT_FILE || true
done
if [[ -v CERTIFICATION_FAILURE ]]; then
echo "CERTIFICATION_FAILURE=true" >> $GITHUB_ENV
exit 1
else
echo "CERTIFICATION_FAILURE=false" >> $GITHUB_ENV
fi
- name: Upload Cert Coverage Report File
uses: actions/upload-artifact@v3
if: github.event_name == 'schedule'
with:
name: cert_code_cov
path: ${{ env.TEST_PATH }}/tmp/cert_code_cov_files
retention-days: 7
- name: Prepare test result info
if: always()
run: |
mkdir -p tmp/result_files
echo "Writing to tmp/result_files/${{ matrix.component }}.txt"
if [[ "${{ env.CERTIFICATION_FAILURE }}" == "true" ]]; then
echo "0" >> "tmp/result_files/${{ matrix.component }}.txt"
else
echo "1" >> "tmp/result_files/${{ matrix.component }}.txt"
fi
- name: Upload result files
uses: actions/upload-artifact@v3
if: always()
with:
name: result_files
path: tmp/result_files
retention-days: 1
# Upload logs for test analytics to consume
- name: Upload test results
if: always()
uses: actions/upload-artifact@master
with:
name: ${{ matrix.component }}_certification_test
path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_certification.*
- name: Run destroy script
if: always() && matrix.destroy-script != ''
run: .github/scripts/components-scripts/${{ matrix.destroy-script }}
post_job:
name: Post-completion
runs-on: ubuntu-22.04
if: always()
needs:
- certification
- generate-matrix
steps:
- name: Parse repository_dispatch payload
if: github.event_name == 'repository_dispatch'
working-directory: ${{ github.workspace }}
run: |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
fi
- name: Download test result artifact
if: always() && env.PR_NUMBER != ''
uses: actions/download-artifact@v3
continue-on-error: true
id: testresults
with:
name: result_files
path: tmp/result_files
- name: Build message
if: always() && env.PR_NUMBER != ''
# Abusing of the github-script action to be able to write this in JS
uses: actions/github-script@v6
with:
script: |
const allComponents = JSON.parse('${{ needs.generate-matrix.outputs.test-matrix }}')
const basePath = '${{ steps.testresults.outputs.download-path }}'
const testType = 'certification'
const fs = require('fs')
const path = require('path')
let message = `# Components ${testType} test
🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**
Commit ref: ${{ env.CHECKOUT_REF }}`
let allSuccess = true
let allFound = true
let notSuccess = []
let notFound = []
for (let i = 0; i < allComponents.length; i++) {
let component = allComponents[i]
if (!component) {
continue
}
if (typeof component == 'object') {
component = component.component
}
let found = false
let success = false
try {
let read = fs.readFileSync(path.join(basePath, component + '.txt'), 'utf8')
read = read.split('\n')[0]
switch (read) {
case '1':
found = true
success = true
break
case '0':
found = true
success = false
}
} catch (e) {
// ignore errors, leave found = false
}
if (!found) {
allFound = false
notFound.push(component)
}
if (!success) {
allSuccess = false
notSuccess.push(component)
}
}
if (allSuccess) {
if (allFound) {
message += '\n\n' + `# ✅ All ${testType} tests passed
All tests have reported a successful status` + '\n\n'
} else {
message += '\n\n' + `# ⚠️ Some ${testType} tests did not report status
Although there were no failures reported, some tests did not report a status:` + '\n\n'
for (let i = 0; i < notFound.length; i++) {
message += '- ' + notFound[i] + '\n'
}
message += '\n'
}
} else {
message += '\n\n' + `# ❌ Some ${testType} tests failed
These tests failed:` + '\n\n'
for (let i = 0; i < notSuccess.length; i++) {
message += '- ' + notSuccess[i] + '\n'
}
message += '\n'
if (!allFound) {
message += 'Additionally, some tests did not report a status:\n\n'
for (let i = 0; i < notFound.length; i++) {
message += '- ' + notFound[i] + '\n'
}
message += '\n'
}
}
fs.writeFileSync('message.txt', message)
- name: Replace PR comment
if: always() && env.PR_NUMBER != ''
uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba
with:
header: ${{ github.run_id }}
number: ${{ env.PR_NUMBER }}
GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
path: message.txt