pytest.yml

---
name: pytest

on:
  pull_request:
  merge_group:
    types:
      - checks_requested
  workflow_dispatch:

env:
  PUDL_OUTPUT: /home/runner/pudl-work/output/
  PUDL_INPUT: /home/runner/pudl-work/input/
  DAGSTER_HOME: /home/runner/pudl-work/dagster_home/

jobs:
  ci-docs:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
    defaults:
      run:
        shell: bash -l {0}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Install conda-lock environment with micromamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environments/conda-lock.yml
          environment-name: pudl-dev
          cache-environment: true

      - name: Log environment details
        run: |
          conda info
          conda list
          conda config --show-sources
          conda config --show
          printenv | sort

      - name: Make input, output and dagster dirs
        run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}

      - name: Lint and build PUDL documentation with Sphinx
        run: |
          pip install --no-deps --editable .
          make docs-build

      - name: Coverage debugging output
        run: |
          coverage debug config
          coverage debug sys
          coverage report --fail-under=0
          ls -a

      - name: Upload docs coverage artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-docs
          include-hidden-files: true
          path: .coverage

  ci-unit:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
    defaults:
      run:
        shell: bash -l {0}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Install conda-lock environment with micromamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environments/conda-lock.yml
          environment-name: pudl-dev
          cache-environment: true

      - name: Log environment details
        run: |
          conda info
          conda list
          conda config --show-sources
          conda config --show
          printenv | sort

      - name: Make input, output and dagster dirs
        run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}

      - name: Log SQLite3 version
        run: |
          which sqlite3
          sqlite3 --version

      - name: Run PUDL unit tests and collect test coverage
        run: |
          pip install --no-deps --editable .
          make pytest-unit

      - name: Coverage debugging output
        run: |
          coverage debug config
          coverage debug sys
          coverage report --fail-under=0
          ls -a

      - name: Upload unit tests coverage artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-unit
          include-hidden-files: true
          path: .coverage

  ci-integration:
    runs-on:
      group: large-runner-group
      labels: ubuntu-latest-4core
    if: github.event_name == 'workflow_dispatch' || github.event.merge_group
    permissions:
      contents: read
      id-token: write
    strategy:
      fail-fast: false
    defaults:
      run:
        shell: bash -l {0}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 2

      - name: Install Conda environment using mamba
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environments/conda-lock.yml
          environment-name: pudl-dev
          cache-environment: true

      - name: Log environment details
        run: |
          conda info
          conda list
          conda config --show-sources
          conda config --show
          printenv | sort

      - name: Log SQLite3 version
        run: |
          which sqlite3
          sqlite3 --version

      - name: Compile Zenodo datastore DOIs for cache invalidation
        run:
          grep -e '.*10\.\(5281\|5072\)/zenodo\..*' src/pudl/workspace/datastore.py
          | sed -e 's/",*$//g' | sed -e 's/^.*"//g' | sort > datastore-dois.txt

      - name: Restore Zenodo datastore from cache if possible
        uses: actions/cache@v4
        id: cache-zenodo-datastore
        with:
          path: ${{ env.PUDL_INPUT }}
          key: zenodo-datastore-${{ hashFiles('datastore-dois.txt') }}

      - name: Make input, output and dagster dirs
        run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }}

      - name: List workspace contents
        run: find /home/runner/pudl-work

      - name: Set default GCP credentials
        id: gcloud-auth
        continue-on-error: true
        uses: "google-github-actions/auth@v2"
        with:
          workload_identity_provider: "projects/345950277072/locations/global/workloadIdentityPools/gh-actions-pool/providers/gh-actions-provider"
          service_account: "tox-pytest-github-action@catalyst-cooperative-pudl.iam.gserviceaccount.com"

      - name: Run integration tests, trying to use GCS cache if possible
        run: |
          pip install --no-deps --editable .
          pudl_datastore --dataset epacems --partition year_quarter=2022q1
          make pytest-integration

      - name: Coverage debugging output
        run: |
          coverage debug config
          coverage debug sys
          coverage report --fail-under=0
          ls -a

      - name: Upload integration test coverage artifact
        uses: actions/upload-artifact@v4
        with:
          name: coverage-integration
          include-hidden-files: true
          path: .coverage

      - name: Log post-test Zenodo datastore contents
        run: find ${{ env.PUDL_INPUT }}

  ci-coverage:
    runs-on: ubuntu-latest
    needs:
      - ci-docs
      - ci-unit
      - ci-integration
    steps:
      - uses: actions/checkout@v4
      - name: Download coverage
        id: download-coverage
        uses: actions/download-artifact@v4
        with:
          path: coverage
      - name: List downloaded files
        run: |
          find coverage -type f
      - name: Install Micromamba
        uses: mamba-org/setup-micromamba@v1
        with:
          init-shell: bash
          environment-name: coverage
          create-args: >-
            python=3.12
            coverage>=7.6.1
      - name: Combine coverage data and output XML report
        run: |
          micromamba run -n coverage coverage combine coverage/*/.coverage
          micromamba run -n coverage coverage xml --fail-under=0
      - name: Upload XML coverage report to CodeCov
        uses: codecov/codecov-action@v4
        with:
          disable_search: true
          file: ./coverage.xml
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: true # optional (default = false)
          plugin: noop
          verbose: true
      - name: Display coverage report and ensure it meets required minimum
        # Required coverage is set in pyproject.toml section [tool.coverage.report]
        run: |
          micromamba run -n coverage coverage report