ci.yml

name: Continuous Integration

on:
  pull_request:
    branches: ["**"]
  push:
    branches: ["main"]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  checks:
    name: Check for code errors
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # Glue Pythonshell supports only Python 3.9.
        # Glue ETL supports only Python 3.10.
        python-version: ["3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4.2.1
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5.2.0
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Poetry Action
        uses: snok/install-poetry@v1.4.1
      - name: Set up Virtualenv Cache
        uses: actions/cache@v4.1.2
        with:
          path: .venv
          key: ${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('poetry.lock') }}
          restore-keys: ${{ runner.os }}-python${{ matrix.python-version }}-
      - name: Install Python Dependencies
        run: |
          poetry check --lock
          poetry install
      - name: Lint code using ruff
        run: |
          poetry run ruff format . --check
          poetry run ruff check .
      - name: Check type annotations using mypy
        run: MYPYPATH=src poetry run mypy .
      - name: Check if requirements.txt is up-to-date
        run: |
          poetry export --with=test --output docker/requirements.txt
          git diff --exit-code
      - name: Run tests that do not need the Glue container
        run: poetry run pytest test/test_*.py
      - name: Build python packages
        run: poetry build

  tests:
    name: Run tests in Glue container
    runs-on: ubuntu-latest
    permissions:
      contents: read
      actions: read
      checks: write
    steps:
      - uses: actions/checkout@v4.2.1
        with:
          fetch-depth: 0
      - name: Diffset
        id: diffset
        uses: softprops/diffset@v2.1.1
        with:
          base: main
          glue_files: |
            **/*.py
            docker/*
            pyproject.toml
      - name: Build and run tests in container
        if: steps.diffset.outputs.glue_files
        run: USER_ID=$(id -u) TARGET=coverage docker compose --file docker/docker-compose.yml run --rm glue-utils -c "pytest -p no:cacheprovider --junitxml=test-results/results.xml --cov=glue_utils --cov-report=xml"
      - name: Generate test report
        uses: dorny/test-reporter@v1.9.1
        if: steps.diffset.outputs.glue_files
        with:
          name: Test Results
          path: test-results/results.xml
          reporter: java-junit
          fail-on-error: false
      - name: SonarCloud Scan
        if: steps.diffset.outputs.glue_files
        uses: SonarSource/sonarcloud-github-action@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

  trivy:
    name: Check dependencies for vulnerabilities
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4.2.1
      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@0.28.0
        continue-on-error: true
        with:
          scan-type: "fs"
          scan-ref: "."
          output: "trivy.txt"
          hide-progress: true
      - name: Publish Trivy output to Summary
        run: |
          if [[ -s trivy.txt ]]; then
            {
              echo "### Trivy"
              echo "<details><summary>Click to expand</summary>"
              echo ""
              echo '```'
              cat trivy.txt
              echo '```'
              echo "</details>"
            } >> "$GITHUB_STEP_SUMMARY"
          fi

  dockerfile:
    name: Check Dockerfile
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4.2.1
      - name: Validate Dockerfile using hadolint
        uses: hadolint/hadolint-action@v3.1.0
        with:
          dockerfile: docker/Dockerfile

  checkmake:
    name: Check Makefile
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4.2.1
      - name: Check Makefile for errors
        uses: Uno-Takashi/checkmake-action@main

  checkov:
    name: Run Checkov
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write
    steps:
      - uses: actions/checkout@v4.2.1
      - name: Check code using Checkov
        uses: bridgecrewio/checkov-action@master
        with:
          directory: .
          quiet: true
          output_format: cli,sarif
          output_file_path: console,checkov.sarif
      - name: Upload SARIF results
        if: success() || failure()
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: checkov.sarif