Merge pull request #54 from cyber-dojo/update-ci-workflow-setup #127
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Main - reports Trails to https://app.kosli.com | |
| on: | |
| push: | |
| branches: | |
| - main | |
| env: | |
| # KOSLI_DRY_RUN: true # Repo override | |
| KOSLI_HOST: ${{ vars.KOSLI_HOST }} # https://app.kosli.com | |
| KOSLI_ORG: ${{ vars.KOSLI_ORG }} # cyber-dojo | |
| KOSLI_FLOW: ${{ vars.KOSLI_FLOW }} # runner-ci | |
| KOSLI_API_TOKEN: ${{ secrets.KOSLI_API_TOKEN }} | |
| KOSLI_TRAIL: ${{ github.sha }} | |
| SERVICE_NAME: ${{ github.event.repository.name }} # runner | |
| AWS_ACCOUNT_ID: ${{ vars.AWS_ACCOUNT_ID }} | |
| AWS_ECR_ID: ${{ vars.AWS_ECR_ID }} | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| IMAGE_TAR_FILENAME: /tmp/${{ github.event.repository.name }}:${{ github.sha }}.tar | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| aws_account_id: ${{ steps.vars.outputs.aws_account_id }} | |
| ecr_registry: ${{ steps.vars.outputs.ecr_registry }} | |
| aws_region: ${{ steps.vars.outputs.aws_region }} | |
| gh_actions_iam_role_name: ${{ steps.vars.outputs.gh_actions_iam_role_name }} | |
| service_name: ${{ steps.vars.outputs.service_name }} | |
| image_tag: ${{ steps.vars.outputs.image_tag }} | |
| image_name: ${{ steps.vars.outputs.image_name }} | |
| image_name_previous: ${{ steps.vars.outputs.image_name_previous }} | |
| kosli_dry_run: ${{ steps.vars.outputs.kosli_dry_run }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 2 | |
| - name: Prepare outputs for workflow jobs | |
| id: vars | |
| run: | | |
| IMAGE_TAG=${GITHUB_SHA:0:7} | |
| IMAGE_TAG_PREVIOUS=$(git rev-parse --short=7 HEAD^) | |
| ECR_REGISTRY="${AWS_ECR_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com" | |
| IMAGE_NAME="${ECR_REGISTRY}/${{ env.SERVICE_NAME }}:${IMAGE_TAG}" | |
| IMAGE_NAME_PREVIOUS="${ECR_REGISTRY}/${{ env.SERVICE_NAME }}:${IMAGE_TAG_PREVIOUS}" | |
| echo "aws_account_id=${AWS_ACCOUNT_ID}" >> ${GITHUB_OUTPUT} | |
| echo "ecr_registry=${ECR_REGISTRY}" >> ${GITHUB_OUTPUT} | |
| echo "aws_region=${AWS_REGION}" >> ${GITHUB_OUTPUT} | |
| echo "gh_actions_iam_role_name=gh_actions_services" >> ${GITHUB_OUTPUT} | |
| echo "service_name=${{ env.SERVICE_NAME }}" >> ${GITHUB_OUTPUT} | |
| echo "image_tag=${IMAGE_TAG}" >> ${GITHUB_OUTPUT} | |
| echo "image_name=${IMAGE_NAME}" >> ${GITHUB_OUTPUT} | |
| echo "image_name_previous=${IMAGE_NAME_PREVIOUS}" >> ${GITHUB_OUTPUT} | |
| if [ "${{ vars.KOSLI_DRY_RUN }}" == "true" ] ; then # Org override | |
| value=true | |
| elif [ "${{ env.KOSLI_DRY_RUN }}" != "" ] ; then # Repo override | |
| value="${{ env.KOSLI_DRY_RUN }}" | |
| elif [ "${{ github.ref }}" == 'refs/heads/main' ] ; # Branch check | |
| value=false | |
| else | |
| value=true # Default | |
| fi | |
| echo "kosli_dry_run=${value}" >> ${GITHUB_OUTPUT} | |
| pull-request: | |
| needs: [setup] | |
| env: | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: read | |
| pull-requests: read | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Setup Kosli CLI | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Attest pull-request evidence to Kosli Trail | |
| run: | |
| kosli attest pullrequest github | |
| --github-token=${{ secrets.GITHUB_TOKEN }} | |
| --name=pull-request | |
| lint: | |
| needs: [setup] | |
| env: | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Setup Kosli CLI | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Run Rubocop lint on source, attest evidence to Kosli Trail | |
| env: | |
| KOSLI_ATTACHMENTS: /tmp/kosli_attachments | |
| run: | | |
| mkdir -p "${KOSLI_ATTACHMENTS}" | |
| set +e | |
| make lint | tee "${KOSLI_ATTACHMENTS}"/rubocop.log | |
| STATUS=${PIPESTATUS[0]} | |
| set -e | |
| KOSLI_COMPLIANT=$([ ${STATUS} = 0 ] && echo true || echo false) | |
| cp .rubocop.yml "${KOSLI_ATTACHMENTS}" | |
| kosli attest generic \ | |
| --attachments="${KOSLI_ATTACHMENTS}" \ | |
| --compliant="${KOSLI_COMPLIANT}" \ | |
| --name=runner.lint | |
| exit ${STATUS} | |
| build-image: | |
| runs-on: ubuntu-latest | |
| needs: [setup] | |
| env: | |
| IMAGE_NAME: ${{ needs.setup.outputs.image_name }} | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| permissions: | |
| id-token: write | |
| contents: write | |
| outputs: | |
| artifact_digest: ${{ steps.variables.outputs.artifact_digest }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ needs.setup.outputs.aws_region }} | |
| role-duration-seconds: 900 | |
| role-session-name: ${{ github.event.repository.name }} | |
| role-to-assume: arn:aws:iam::${{ needs.setup.outputs.aws_account_id }}:role/${{ needs.setup.outputs.gh_actions_iam_role_name }} | |
| mask-aws-account-id: 'no' | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - name: Build and push Docker image to ECR | |
| id: docker_build | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ env.IMAGE_NAME }} | |
| cache-from: type=registry,ref=${{ needs.setup.outputs.image_name_previous }} | |
| cache-to: type=inline,mode=max | |
| build-args: | |
| COMMIT_SHA=${{ github.sha }} | |
| - name: Tar Docker image | |
| run: | | |
| docker pull ${{ env.IMAGE_NAME }} | |
| docker image save ${{ env.IMAGE_NAME }} --output ${{ env.IMAGE_TAR_FILENAME }} | |
| - name: Cache Docker image | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.IMAGE_TAR_FILENAME }} | |
| key: ${{ env.IMAGE_NAME }} | |
| - name: Setup Kosli CLI | |
| if: success() || failure() | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Attest image evidence to Kosli Trail and make Digest available to following steps and jobs | |
| if: success() || failure() | |
| id: variables | |
| run: | | |
| DIGEST=$(echo ${{ steps.docker_build.outputs.digest }} | sed 's/.*://') | |
| kosli attest artifact "${{ env.IMAGE_NAME }}" \ | |
| --fingerprint="${DIGEST}" \ | |
| --name=runner | |
| echo "artifact_digest=${DIGEST}" >> ${GITHUB_OUTPUT} | |
| unit-tests: | |
| runs-on: ubuntu-latest | |
| needs: [setup, build-image] | |
| env: | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Retrieve Docker image from cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.IMAGE_TAR_FILENAME }} | |
| key: ${{ needs.setup.outputs.image_name }} | |
| - name: Load Docker image | |
| run: | |
| docker image load --input ${{ env.IMAGE_TAR_FILENAME }} | |
| - name: Run unit tests with coverage | |
| run: | |
| make unit_test | |
| - name: Setup Kosli CLI | |
| if: success() || failure() | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Attest results to Kosli | |
| if: success() || failure() | |
| env: | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| run: | |
| kosli attest junit | |
| --name=runner.unit-test | |
| --results-dir=./test/server/reports/junit | |
| - name: Attest coverage to Kosli | |
| if: success() || failure() | |
| env: | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| REPORTS_DIR: "./test/server/reports" | |
| run: | |
| kosli attest generic | |
| --attachments="${REPORTS_DIR}/coverage" | |
| --name=runner.unit-test-branch-coverage | |
| --user-data="${REPORTS_DIR}/coverage/summary.json" | |
| integration-tests: | |
| runs-on: ubuntu-latest | |
| needs: [setup, build-image] | |
| env: | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Retrieve Docker image from cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.IMAGE_TAR_FILENAME }} | |
| key: ${{ needs.setup.outputs.image_name }} | |
| - name: Load Docker image | |
| run: | |
| docker image load --input ${{ env.IMAGE_TAR_FILENAME }} | |
| - name: Run integration tests with coverage | |
| run: | |
| make integration_test | |
| - name: Setup Kosli CLI | |
| if: success() || failure() | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Attest results to Kosli | |
| if: success() || failure() | |
| env: | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| run: | |
| kosli attest junit | |
| --name=runner.integration-test | |
| --results-dir=./test/client/reports/junit | |
| - name: Attest coverage to Kosli | |
| if: success() || failure() | |
| env: | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| REPORTS_DIR: ./test/client/reports | |
| run: | |
| kosli attest generic | |
| --attachments="${REPORTS_DIR}/coverage" | |
| --name=runner.integration-test-branch-coverage | |
| --user-data="${REPORTS_DIR}/coverage/summary.json" | |
| snyk-container-scan: | |
| runs-on: ubuntu-latest | |
| needs: [setup, build-image] | |
| env: | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| SARIF_FILENAME: snyk.container.scan.json | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Retrieve Docker image from cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.IMAGE_TAR_FILENAME }} | |
| key: ${{ needs.setup.outputs.image_name }} | |
| - name: Load Docker image | |
| run: | |
| docker image load --input ${{ env.IMAGE_TAR_FILENAME }} | |
| - name: Setup Snyk | |
| uses: snyk/actions/setup@master | |
| - name: Run Snyk container scan | |
| env: | |
| IMAGE_NAME: ${{ needs.setup.outputs.image_name }} | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| run: | |
| snyk container test ${IMAGE_NAME} | |
| --file=Dockerfile | |
| --policy-path=.snyk | |
| --sarif | |
| --sarif-file-output="${SARIF_FILENAME}" | |
| - name: Setup Kosli CLI | |
| if: ${{ success() || failure() }} | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Attest Snyk container scan results to Kosli Trail | |
| if: ${{ success() || failure() }} | |
| env: | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| KOSLI_ATTACHMENTS: .snyk | |
| run: | |
| kosli attest snyk | |
| --name=runner.snyk-container-scan | |
| --scan-results="${SARIF_FILENAME}" | |
| snyk-code-scan: | |
| runs-on: ubuntu-latest | |
| needs: [setup, build-image] | |
| env: | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| SARIF_FILENAME: snyk.code.scan.json | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| - name: Setup Snyk | |
| uses: snyk/actions/setup@master | |
| - name: Run Snyk code scan | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| run: | |
| snyk code test | |
| --sarif | |
| --sarif-file-output="${SARIF_FILENAME}" | |
| --policy-path=.snyk | |
| . | |
| - name: Setup Kosli CLI | |
| if: ${{ success() || failure() }} | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Attest Snyk code scan results to Kosli Trail | |
| if: ${{ success() || failure() }} | |
| env: | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| KOSLI_ATTACHMENTS: .snyk | |
| run: | |
| kosli attest snyk "${IMAGE_NAME}" | |
| --name=runner.snyk-code-scan | |
| --scan-results="${SARIF_FILENAME}" | |
| sdlc-control-gate: | |
| runs-on: ubuntu-latest | |
| needs: [setup, build-image, pull-request, lint, unit-tests, integration-tests, snyk-container-scan, snyk-code-scan] | |
| env: | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| steps: | |
| - name: Setup Kosli CLI | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Kosli SDLC gate to short-circuit the Trail | |
| env: | |
| IMAGE_NAME: ${{ needs.setup.outputs.image_name }} | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| run: | |
| kosli assert artifact ${IMAGE_NAME} | |
| approve-deployment-to-beta: | |
| runs-on: ubuntu-latest | |
| needs: [setup, build-image, sdlc-control-gate] | |
| env: | |
| KOSLI_DRY_RUN: ${{ needs.setup.outputs.kosli_dry_run }} | |
| environment: | |
| name: staging | |
| url: https://beta.cyber-dojo.org | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Kosli CLI | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Attest approval of deployment to Kosli | |
| env: | |
| IMAGE_NAME: ${{ needs.setup.outputs.image_name }} | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| KOSLI_ENVIRONMENT: aws-beta | |
| run: | |
| kosli report approval ${IMAGE_NAME} | |
| --approver="${{ github.actor }}" | |
| deploy-to-beta: | |
| needs: [setup, approve-deployment-to-beta] | |
| uses: ./.github/workflows/sub_deploy_to_beta.yml | |
| with: | |
| IMAGE_TAG: ${{ needs.setup.outputs.image_tag }} | |
| secrets: | |
| KOSLI_API_TOKEN: ${{ secrets.KOSLI_API_TOKEN }} | |
| approve-deployment-to-prod: | |
| needs: [setup, build-image, deploy-to-beta] | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: production | |
| url: https://cyber-dojo.org | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup Kosli CLI | |
| uses: kosli-dev/setup-cli-action@v2 | |
| with: | |
| version: ${{ vars.KOSLI_CLI_VERSION }} | |
| - name: Attest approval of deployment to Kosli | |
| env: | |
| IMAGE_NAME: ${{ needs.setup.outputs.image_name }} | |
| KOSLI_FINGERPRINT: ${{ needs.build-image.outputs.artifact_digest }} | |
| KOSLI_ENVIRONMENT: aws-prod | |
| run: | |
| kosli report approval ${IMAGE_NAME} | |
| --approver="${{ github.actor }}" | |
| deploy-to-prod: | |
| needs: [setup, approve-deployment-to-prod] | |
| uses: ./.github/workflows/sub_deploy_to_prod.yml | |
| with: | |
| IMAGE_TAG: ${{ needs.setup.outputs.image_tag }} | |
| secrets: | |
| KOSLI_API_TOKEN: ${{ secrets.KOSLI_API_TOKEN }} | |
| # The cyberdojo/versioner refresh-env.sh script | |
| # https://github.com/cyber-dojo/versioner/blob/master/sh/refresh-env.sh | |
| # relies on being able to: | |
| # - get the :latest image | |
| # - extract the SHA env-var embedded inside it | |
| # - use the 1st 7 chars of the SHA as a latest-equivalent tag | |
| push-latest: | |
| needs: [setup, deploy-to-prod] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| contents: write | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ needs.setup.outputs.aws_region }} | |
| role-duration-seconds: 900 | |
| role-session-name: ${{ github.event.repository.name }} | |
| role-to-assume: arn:aws:iam::${{ needs.setup.outputs.aws_account_id }}:role/${{ needs.setup.outputs.gh_actions_iam_role_name }} | |
| mask-aws-account-id: 'no' | |
| - name: Login to Amazon ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v2 | |
| - uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USER }} | |
| password: ${{ secrets.DOCKER_PASS }} | |
| - name: Tag image to :latest and push to Dockerhub Registry | |
| env: | |
| IMAGE_NAME: ${{ needs.setup.outputs.image_name }} | |
| IMAGE_TAG: ${{ needs.setup.outputs.image_tag }} | |
| run: | | |
| docker pull "${IMAGE_NAME}" | |
| docker tag "${IMAGE_NAME}" cyberdojo/${{ env.SERVICE_NAME }}:${IMAGE_TAG} | |
| docker tag "${IMAGE_NAME}" cyberdojo/${{ env.SERVICE_NAME }}:latest | |
| docker push cyberdojo/${{ env.SERVICE_NAME }}:${IMAGE_TAG} | |
| docker push cyberdojo/${{ env.SERVICE_NAME }}:latest |