Skip to content
Components Conformance Tests

conformance-test #40977

Sign in to view logs

conformance-test

conformance-test #40977

Workflow file for this run

.github/workflows/conformance.yml at f0a99c1
# ------------------------------------------------------------
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ------------------------------------------------------------
name: Components Conformance Tests
on:
repository_dispatch:
types: [conformance-test]
workflow_dispatch:
schedule:
- cron: '0 */8 * * *'
push:
branches:
- 'release-*'
- 'gh-readonly-queue/main/*'
pull_request:
branches:
- 'main'
- 'release-*'
- 'gh-readonly-queue/main/*'
merge_group:
jobs:
# Based on whether this is a PR or a scheduled run, we will run a different
# subset of the conformance tests. This allows all the tests not requiring
# secrets to be executed on pull requests.
generate-matrix:
runs-on: ubuntu-22.04
steps:
- name: Parse repository_dispatch payload
if: github.event_name == 'repository_dispatch'
working-directory: ${{ github.workspace }}
run: |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
fi
- name: Check out code
uses: actions/checkout@v4
with:
repository: ${{ env.CHECKOUT_REPO }}
ref: ${{ env.CHECKOUT_REF }}
- name: Generate test matrix
id: generate-matrix
env:
VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
run: |
if [ -z "$VAULT_NAME" ]; then
# Do not include cloud tests when credentials are not available
node .github/scripts/test-info.mjs conformance false
else
# Include cloud tests
node .github/scripts/test-info.mjs conformance true
fi
- name: Create PR comment
if: env.PR_NUMBER != ''
uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba
with:
header: ${{ github.run_id }}
number: ${{ env.PR_NUMBER }}
GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
message: |
# Components conformance test
🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**
Commit ref: ${{ env.CHECKOUT_REF }}
outputs:
test-matrix: ${{ steps.generate-matrix.outputs.test-matrix }}
conformance:
name: ${{ matrix.component }} conformance
# Add "id-token" with the intended permissions.
# Needed by the 'Authenticate to Google Cloud' step.
permissions:
contents: 'read'
id-token: 'write'
runs-on: ubuntu-22.04
env:
UNIQUE_ID: ${{github.run_id}}-${{github.run_attempt}}
GOCOV_VER: "v1.1.0"
GOTESTSUM_VER: "v1.9.0"
defaults:
run:
shell: bash
needs:
- generate-matrix
strategy:
fail-fast: false # Keep running even if one component fails
matrix:
include: ${{ fromJson(needs.generate-matrix.outputs.test-matrix) }}
steps:
- name: Set default payload repo and ref
working-directory: ${{ github.workspace }}
run: |
echo "CHECKOUT_REPO=${{ github.repository }}" >> $GITHUB_ENV
echo "CHECKOUT_REF=${{ github.ref }}" >> $GITHUB_ENV
- name: Parse repository_dispatch payload
if: github.event_name == 'repository_dispatch'
working-directory: ${{ github.workspace }}
run: |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
echo "CHECKOUT_REPO=${{ github.event.client_payload.pull_head_repo }}" >> $GITHUB_ENV
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
fi
- name: Check out code
uses: actions/checkout@v4
with:
repository: ${{ env.CHECKOUT_REPO }}
ref: ${{ env.CHECKOUT_REF }}
- name: Setup test environment
run: |
# Output file
echo "TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report" >> $GITHUB_ENV
# Current time (used by Terraform)
echo "CURRENT_TIME=$(date --rfc-3339=date)" >> ${GITHUB_ENV}
- name: Configure conformance test and source path
run: |
TEST_COMPONENT=$(echo ${{ matrix.component }} | sed -E 's/\./\//g')
export SOURCE_PATH="github.com/dapr/components-contrib/${TEST_COMPONENT}"
echo "SOURCE_PATH=$SOURCE_PATH" >> $GITHUB_ENV
# converts slashes to dots in this string, so that it doesn't consider them sub-folders
export SOURCE_PATH_LINEAR=$(echo "$SOURCE_PATH" |sed 's#/#\.#g')
echo "SOURCE_PATH_LINEAR=$SOURCE_PATH_LINEAR" >> $GITHUB_ENV
- uses: Azure/login@v1
if: matrix.required-secrets != ''
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
# Set this GitHub secret to your KeyVault, and grant the KeyVault policy to your Service Principal:
# az keyvault set-policy -n $AZURE_KEYVAULT --secret-permissions get list --spn $SPN_CLIENT_ID
# Using az cli to query keyvault as Azure/get-keyvault-secrets@v1 is deprecated
- name: Setup secrets
if: matrix.required-secrets != ''
env:
VAULT_NAME: ${{ secrets.AZURE_KEYVAULT }}
run: |
secrets="${{ matrix.required-secrets }}"
for secretName in $(echo -n $secrets | tr ',' ' '); do
value=$(az keyvault secret show \
--name $secretName \
--vault-name $VAULT_NAME \
--query value \
--output tsv)
echo "::add-mask::$value"
echo "$secretName=$value" >> $GITHUB_OUTPUT
echo "$secretName=$value" >> $GITHUB_ENV
done
# Authenticate with GCP Workload Identity Pool
# Exports GCP ENV Vars:
# - GCP_PROJECT
# - GOOGLE_APPLICATION_CREDENTIALS
- id: 'auth'
if: matrix.require-gcp-credentials == 'true'
name: 'Authenticate to Google Cloud'
uses: 'google-github-actions/auth@v1'
with:
token_format: 'access_token'
workload_identity_provider: ${{ secrets.GCP_WIF_PROVIDER_NAME }}
service_account: ${{ secrets.GCP_WIF_SA_EMAIL }}
create_credentials_file: true
export_environment_variables: true
cleanup_credentials: true
# Download the required certificates into files, and set env var pointing to their names
- name: Setup certs
if: matrix.required-certs != ''
run: |
for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
CERT_FILE=$(mktemp --suffix .pfx)
echo "Downloading cert $CERT_NAME into file $CERT_FILE"
rm $CERT_FILE && \
az keyvault secret download --vault-name ${{ secrets.AZURE_KEYVAULT }} --name $CERT_NAME --encoding base64 --file $CERT_FILE
echo 'Setting $CERT_NAME to' "$CERT_FILE"
echo "$CERT_NAME=$CERT_FILE" >> $GITHUB_ENV
done
- name: Setup Terraform
if: matrix.require-terraform == 'true'
uses: hashicorp/[email protected]
- name: Set Cloudflare env vars
if: matrix.require-cloudflare-credentials == 'true'
run: |
echo "CLOUDFLARE_ACCOUNT_ID=${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" >> $GITHUB_ENV
echo "CLOUDFLARE_API_TOKEN=${{ secrets.CLOUDFLARE_API_TOKEN }}" >> $GITHUB_ENV
- name: Set AWS env vars
if: matrix.require-aws-credentials == 'true'
run: |
echo "AWS_ACCESS_KEY=${{ secrets.AWS_ACCESS_KEY }}" >> $GITHUB_ENV
echo "AWS_SECRET_KEY=${{ secrets.AWS_SECRET_KEY }}" >> $GITHUB_ENV
- name: Configure AWS Credentials
if: matrix.require-aws-credentials == 'true'
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_KEY }}
aws-region: us-west-1
- name: Start MongoDB
if: matrix.mongodb-version != ''
uses: supercharge/[email protected]
with:
mongodb-version: ${{ matrix.mongodb-version }}
mongodb-replica-set: test-rs
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version-file: 'go.mod'
cache: 'false'
- name: Install Node.js ${{ matrix.nodejs-version }}
if: matrix.nodejs-version != ''
uses: actions/setup-node@v3
with:
node-version: ${{ matrix.nodejs-version }}
- name: Start KinD
uses: helm/[email protected]
if: matrix.require-kind == 'true'
- name: Download Go dependencies
run: |
go mod download
go install github.com/axw/gocov/gocov@${{ env.GOCOV_VER }}
go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VER }}
- name: Run setup script
if: matrix.setup-script != ''
run: .github/scripts/components-scripts/${{ matrix.setup-script }}
- name: Catch setup failures
if: failure()
run: |
echo "CONFORMANCE_FAILURE=true" >> $GITHUB_ENV
- name: Run tests
continue-on-error: true
run: |
set -e
KIND=$(echo ${{ matrix.component }} | cut -d. -f1)
NAME=$(echo ${{ matrix.component }} | cut -d. -f2-)
KIND_UPPER="$(tr '[:lower:]' '[:upper:]' <<< ${KIND:0:1})${KIND:1}"
if [ "${KIND}" = "secretstores" ]; then
KIND_UPPER=SecretStore
fi
echo "Running tests for Test${KIND_UPPER}Conformance/${KIND}/${NAME} ... "
echo "Source Package: " ${{ matrix.source-pkg }}
set +e
gotestsum --jsonfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.json \
--junitfile ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.xml --format standard-verbose -- \
-p 2 -count=1 -timeout=15m -tags=conftests ./tests/conformance --run="Test${KIND_UPPER}Conformance/${NAME}" -coverprofile=cover.out \
-covermode=set -coverpkg=${{ matrix.source-pkg }}
status=$?
echo "Completed tests for Test${KIND_UPPER}Conformance/${KIND}/${NAME} ... "
if test $status -ne 0; then
echo "Setting CONFORMANCE_FAILURE"
echo "CONFORMANCE_FAILURE=true" >> $GITHUB_ENV
fi
set -e
# Fail the step if we found no test to run
if grep -q "warning: no tests to run" ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.json ; then
echo "::error:: No test was found for component ${{ matrix.component }}"
exit -1
fi
- name: Delete downloaded up certs
if: always() && matrix.required-certs != ''
run: |
for CERT_NAME in $(echo "${{ matrix.required-certs }}" | sed 's/,/ /g'); do
CERT_FILE=$(printenv $CERT_NAME)
echo "Cleaning up the certificate file $CERT_FILE..."
rm $CERT_FILE || true
done
- name: Check conformance test passed
continue-on-error: false
run: |
echo "CONFORMANCE_FAILURE=$CONFORMANCE_FAILURE"
if [[ -v CONFORMANCE_FAILURE ]]; then
exit 1
fi
- name: Prepare test result info
if: always()
run: |
mkdir -p tmp/result_files
echo "Writing to tmp/result_files/${{ matrix.component }}.txt"
if [[ "${{ env.CONFORMANCE_FAILURE }}" == "true" ]]; then
echo "0" >> "tmp/result_files/${{ matrix.component }}.txt"
else
echo "1" >> "tmp/result_files/${{ matrix.component }}.txt"
fi
- name: Upload result files
uses: actions/upload-artifact@v3
if: always()
with:
name: result_files
path: tmp/result_files
retention-days: 1
- name: Prepare coverage report file to upload
if: github.event_name == 'schedule'
run: |
mkdir -p tmp/conf_code_cov
cp cover.out tmp/conf_code_cov/${{ env.SOURCE_PATH_LINEAR }}.out
- name: Upload coverage report file
uses: actions/upload-artifact@v3
if: github.event_name == 'schedule'
with:
name: conf_code_cov
path: tmp/conf_code_cov
retention-days: 7
# Upload logs for test analytics to consume
- name: Upload test results
if: always()
uses: actions/upload-artifact@main
with:
name: ${{ matrix.component }}_conformance_test
path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_conformance.*
- name: Run destroy script
if: always() && matrix.destroy-script != ''
run: .github/scripts/components-scripts/${{ matrix.destroy-script }}
post_job:
name: Post-completion
runs-on: ubuntu-22.04
if: always()
needs:
- conformance
- generate-matrix
steps:
- name: Parse repository_dispatch payload
if: github.event_name == 'repository_dispatch'
working-directory: ${{ github.workspace }}
run: |
if [ ${{ github.event.client_payload.command }} = "ok-to-test" ]; then
echo "CHECKOUT_REF=${{ github.event.client_payload.pull_head_ref }}" >> $GITHUB_ENV
echo "PR_NUMBER=${{ github.event.client_payload.issue.number }}" >> $GITHUB_ENV
fi
- name: Download test result artifact
if: always() && env.PR_NUMBER != ''
uses: actions/download-artifact@v3
continue-on-error: true
id: testresults
with:
name: result_files
path: tmp/result_files
- name: Build message
if: always() && env.PR_NUMBER != ''
# Abusing of the github-script action to be able to write this in JS
uses: actions/github-script@v6
with:
script: |
const allComponents = JSON.parse('${{ needs.generate-matrix.outputs.test-matrix }}')
const basePath = '${{ steps.testresults.outputs.download-path }}'
const testType = 'conformance'
const fs = require('fs')
const path = require('path')
let message = `# Components ${testType} test
🔗 **[Link to Action run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})**
Commit ref: ${{ env.CHECKOUT_REF }}`
let allSuccess = true
let allFound = true
let notSuccess = []
let notFound = []
for (let i = 0; i < allComponents.length; i++) {
let component = allComponents[i]
if (!component) {
continue
}
if (typeof component == 'object') {
component = component.component
}
let found = false
let success = false
try {
let read = fs.readFileSync(path.join(basePath, component + '.txt'), 'utf8')
read = read.split('\n')[0]
switch (read) {
case '1':
found = true
success = true
break
case '0':
found = true
success = false
}
} catch (e) {
// ignore errors, leave found = false
}
if (!found) {
allFound = false
notFound.push(component)
}
if (!success) {
allSuccess = false
notSuccess.push(component)
}
}
if (allSuccess) {
if (allFound) {
message += '\n\n' + `# ✅ All ${testType} tests passed
All tests have reported a successful status` + '\n\n'
} else {
message += '\n\n' + `# ⚠️ Some ${testType} tests did not report status
Although there were no failures reported, some tests did not report a status:` + '\n\n'
for (let i = 0; i < notFound.length; i++) {
message += '- ' + notFound[i] + '\n'
}
message += '\n'
}
} else {
message += '\n\n' + `# ❌ Some ${testType} tests failed
These tests failed:` + '\n\n'
for (let i = 0; i < notSuccess.length; i++) {
message += '- ' + notSuccess[i] + '\n'
}
message += '\n'
if (!allFound) {
message += 'Additionally, some tests did not report a status:\n\n'
for (let i = 0; i < notFound.length; i++) {
message += '- ' + notFound[i] + '\n'
}
message += '\n'
}
}
fs.writeFileSync('message.txt', message)
- name: Replace PR comment
if: env.PR_NUMBER != ''
uses: artursouza/sticky-pull-request-comment@da9e86aa2a80e4ae3b854d251add33bd6baabcba
with:
header: ${{ github.run_id }}
number: ${{ env.PR_NUMBER }}
GITHUB_TOKEN: ${{ secrets.DAPR_BOT_TOKEN }}
path: message.txt