build-and-test.yml

name: Build and Test

on:
  push:
    branches: [ main ]
    paths-ignore: [ '*.md' ]
  pull_request:
    branches: [ '**' ]

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - name: Install black and link shellcheck into expected location
        run: |
          pip install black --force-reinstall black==22.3.0
          sudo ln -s $(which shellcheck) /usr/local/bin/shellcheck
      - name: Set up JDK
        uses: actions/setup-java@v3
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: 'gradle'

      - name: Build all projects without running tests
        run: ./gradlew --build-cache build -x test

      - name: Upload spotbugs results
        uses: github/codeql-action/upload-sarif@main
        with:
          sarif_file: service/build/reports/spotbugs/main.sarif

  dispatch-trivy:
    needs: [ build ]
    runs-on: ubuntu-latest

    if: github.event_name == 'pull_request'

    steps:
      - name: Fire off Trivy action
        uses: broadinstitute/workflow-dispatch@v3
        with:
          workflow: Trivy
          token: ${{ secrets.BROADBOT_TOKEN }}
          ref: ${{ github.event.pull_request.head.ref }}

  source-clear:
    needs: [ build ]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK
        uses: actions/setup-java@v3
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: 'gradle'

      - name: SourceClear scan
        env:
          SRCCLR_API_TOKEN: ${{ secrets.SRCCLR_API_TOKEN }}
        run: ./gradlew --build-cache srcclr

  unit-tests-and-sonar:
    needs: [ build ]
    runs-on: ubuntu-latest

    services:
      postgres:
        image: postgres:13
        env:
          POSTGRES_PASSWORD: postgres
        ports: [ "5432:5432" ]

    steps:
      - uses: actions/checkout@v3
        # Needed by sonar to get the git history for the branch the PR will be merged into.
        with:
          fetch-depth: 0
      - name: Set up JDK
        uses: actions/setup-java@v3
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: 'gradle'

      - name: Make sure Postgres is ready and init
        env:
          PGPASSWORD: postgres
        run: |
          pg_isready -h localhost -t 10
          psql -h localhost -U postgres -f scripts/postgres-init.sql

      - name: Test with coverage
        run: ./gradlew --build-cache test jacocoTestReport

      # The Sonar scan is done here, so it can upload the coverage report generated by the tests.
      - name: Sonar scan
        run: ./gradlew --build-cache :common:sonar
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  admin-cli-test:
    needs: [ build ]
    runs-on: ubuntu-latest

    services:
      postgres:
        image: postgres:13
        env:
          POSTGRES_PASSWORD: postgres
        ports: [ "5432:5432" ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK
        uses: actions/setup-java@v3
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: 'gradle'

      - name: Make sure Postgres is ready and init
        env:
          PGPASSWORD: postgres
        run: |
          pg_isready -h localhost -t 10
          psql -h localhost -U postgres -f scripts/postgres-init.sql

      - name: Smoke test -- run the CLI
        run: UPGRADE_DB=true ./gradlew --build-cache :admin-cli:bootRun --args=list

  integration-tests:
    needs: [ build ]
    runs-on: ubuntu-latest

    services:
      postgres:
        image: postgres:13
        env:
          POSTGRES_PASSWORD: postgres
        ports: [ "5432:5432" ]

    steps:
      - uses: actions/checkout@v3
      - name: Set up JDK
        uses: actions/setup-java@v3
        with:
          java-version: '17'
          distribution: 'temurin'
          cache: 'gradle'

      - name: Construct docker image name and tag
        id: image-name
        run: |
          GITHUB_REPO=$(basename ${{ github.repository }})
          GIT_SHORT_HASH=$(git rev-parse --short HEAD)
          echo "name=${GITHUB_REPO}:${GIT_SHORT_HASH}" >> $GITHUB_OUTPUT

      - name: Build image locally with jib
        run: |
          ./gradlew --build-cache :service:jibDockerBuild \
            --image=${{ steps.image-name.outputs.name }} \
            -Djib.console=plain

      - name: Make sure Postgres is ready and init
        env:
          PGPASSWORD: postgres
        run: |
          pg_isready -h localhost -t 10
          psql -h localhost -U postgres -f scripts/postgres-init.sql

      - name: Render GitHub Secrets
        run: |
          echo "${{ secrets.DEV_FIRECLOUD_ACCOUNT_B64 }}" | base64 -d > "integration/src/main/resources/rendered/user-delegated-sa.json"
          echo "${{ secrets.PERF_TESTRUNNER_ACCOUNT_B64 }}" | base64 -d > "integration/src/main/resources/rendered/testrunner-perf.json"

      - name: Launch the background process for integration tests
        # 172.17.0.1 is the IP address of the gateway in the default docker bridge network. https://docs.docker.com/network/network-tutorial-standalone/
        run: docker run -e DATABASE_HOSTNAME=172.17.0.1 -e SPRING_PROFILES_INCLUDE=human-readable-logging -p 8080:8080 -d --name catalog ${{ steps.image-name.outputs.name }}

      - name: Wait for boot run to be ready
        run: |
          set +e
          timeout 20 bash -c 'until [[ $(curl -s localhost:8080/status | jq .ok) == "true" ]]; do sleep 1; done'
          resultStatus=$?
          set -e
          if [[ $resultStatus == 0 ]]; then
            echo "Server started successfully"
          else
            echo "Server did not start successfully. Dumping log file"
            docker logs catalog
            exit 1
          fi

      - name: Run the integration test suite
        run: ./gradlew --build-cache runTest --args="suites/local/FullIntegration.json build/reports"

  notify-slack:
    needs: [ unit-tests-and-sonar, source-clear, integration-tests, admin-cli-test ]
    runs-on: ubuntu-latest

    if: failure() && github.ref == 'refs/heads/main'

    steps:
      - name: Notify slack on failure
        uses: broadinstitute/action-slack@v3.15.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          channel: '#dsp-data-exploration'
          status: failure
          author_name: Build on dev
          fields: job,message
          text: 'Build failed :sadpanda:'
          username: 'Data Explorer GitHub Action'

  dispatch-tag:
    needs: [ unit-tests-and-sonar, source-clear, integration-tests, admin-cli-test ]
    runs-on: ubuntu-latest

    if: success() && github.ref == 'refs/heads/main'

    steps:
      - name: Fire off tag action
        uses: broadinstitute/workflow-dispatch@v3
        with:
          workflow: Tag
          token: ${{ secrets.BROADBOT_TOKEN }}