cli-test.yml

name: Run CLI tests
on:
  workflow_dispatch:
  schedule:
    - cron: '0 0 * * *'
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    paths:
      - .github/workflows/cli-test.yml
      - extensions/ql-vscode/src/codeql-cli/**
      - extensions/ql-vscode/src/language-support/**
      - extensions/ql-vscode/src/query-server/**
      - extensions/ql-vscode/supported_cli_versions.json
      - extensions/ql-vscode/src/variant-analysis/run-remote-query.ts

jobs:
  find-nightly:
    name: Find Nightly Release
    runs-on: ubuntu-latest
    outputs:
      url: ${{ steps.get-url.outputs.nightly-url }}
    steps:
      - name: Get Nightly Release URL
        id: get-url
        env:
          GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}'
        shell: bash
        # This workflow step gets an unstable testing version of the CodeQL CLI. It should not be used outside of these tests.
        run: |
          LATEST=`gh api repos/dsp-testing/codeql-cli-nightlies/releases --jq '.[].tag_name' --method GET --raw-field 'per_page=1'`
          echo "nightly-url=https://github.com/dsp-testing/codeql-cli-nightlies/releases/download/$LATEST" >> "$GITHUB_OUTPUT"

  set-matrix:
    name: Set Matrix for cli-test
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set the variables
        id: set-variables
        run: echo "cli-versions=$(cat ./extensions/ql-vscode/supported_cli_versions.json | jq -rc)" >> $GITHUB_OUTPUT
    outputs:
      cli-versions: ${{ steps.set-variables.outputs.cli-versions }}

  cli-test:
    name: CLI Test
    runs-on: ${{ matrix.os }}
    needs: [find-nightly, set-matrix]
    timeout-minutes: 30
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
        version: ${{ fromJson(needs.set-matrix.outputs.cli-versions) }}
      fail-fast: false
    env:
      CLI_VERSION: ${{ matrix.version }}
      NIGHTLY_URL: ${{ needs.find-nightly.outputs.url }}
      TEST_CODEQL_PATH: '${{ github.workspace }}/codeql'

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version-file: extensions/ql-vscode/.nvmrc
          cache: 'npm'
          cache-dependency-path: extensions/ql-vscode/package-lock.json

      - name: Install dependencies
        working-directory: extensions/ql-vscode
        run: |
          npm ci
        shell: bash

      - name: Build
        working-directory: extensions/ql-vscode
        run: |
          npm run build
        shell: bash

      - name: Decide on ref of CodeQL repo
        id: choose-ref
        shell: bash
        run: |
          if [[ "${{ matrix.version }}" == "nightly" ]]
          then
            REF="codeql-cli/latest"
          else
            REF="codeql-cli/${{ matrix.version }}"
          fi
          echo "ref=$REF" >> "$GITHUB_OUTPUT"

      - name: Checkout QL
        uses: actions/checkout@v4
        with:
          repository: github/codeql
          ref: ${{ steps.choose-ref.outputs.ref }}
          path: codeql

      - name: Run CLI tests (Linux)
        working-directory: extensions/ql-vscode
        if: matrix.os == 'ubuntu-latest'
        run: |
          unset DBUS_SESSION_BUS_ADDRESS
          /usr/bin/xvfb-run npm run test:cli-integration

      - name: Run CLI tests (Windows)
        working-directory: extensions/ql-vscode
        if: matrix.os == 'windows-latest'
        run: |
          npm run test:cli-integration

  report-failure:
    name: Report failure on the default branch
    runs-on: ubuntu-latest
    needs: [cli-test]
    if: failure() && github.ref == 'refs/heads/main'
    permissions:
      contents: read
      issues: write
    env:
      GH_TOKEN: ${{ github.token }}
    steps:
      - name: Create GitHub issue
        run: |
          # Set -eu so that we fail if the gh command fails.
          set -eu

          # Try to find an existing open issue if there is one
          ISSUE="$(gh issue list --repo "$GITHUB_REPOSITORY" --label "cli-test-failure" --state "open" --limit 1 --json number -q '.[0].number')"

          if [[ -n "$ISSUE" ]]; then
            echo "Found open issue number $ISSUE ($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/issues/$ISSUE)"
          else
            echo "Did not find an open tracking issue. Creating one."

            ISSUE_BODY="issue-body.md"
            printf "CLI tests have failed on the default branch.\n\n@github/code-scanning-secexp-reviewers" > "$ISSUE_BODY"

            ISSUE="$(gh issue create --repo "$GITHUB_REPOSITORY" --label "cli-test-failure" --title "CLI test failure" --body-file "$ISSUE_BODY")"
            # `gh issue create` returns the full issue URL, not just the number.
            echo "Created issue with URL $ISSUE"
          fi

          COMMENT_FILE="comment.md"
          RUN_URL=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID
          printf 'CLI test [%s](%s) failed on ref `%s`' "$GITHUB_RUN_ID" "$RUN_URL" "$GITHUB_REF" > "$COMMENT_FILE"

          # `gh issue create` returns an issue URL, and `gh issue list | cut -f 1` returns an issue number.
          # Both are accepted here.
          gh issue comment "$ISSUE" --repo "$GITHUB_REPOSITORY" --body-file "$COMMENT_FILE"