py-unittests.yml

name: Python cellxgene_census package unit tests

on:
  pull_request:
    paths-ignore:
      - "api/r/**"
  push:
    branches: [main]

# If a new commit is pushed, cancel the jobs from previous commits.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  CELLXGENE_CENSUS_USERAGENT: "CZI-test"

jobs:
  unit_tests_python_api:
    strategy:
      fail-fast: false  # Don't stop the workflow if one of the jobs fails
      matrix:
        os: [sc-dev-64g-runner, macos-latest]
        python-version: ["3.10", "3.11", "3.12"]
        exclude:
          - os: macos-latest
            python-version: "3.12"

    runs-on: ${{matrix.os}}

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: |
            api/**/pyproject.toml
            api/**/requirements*.txt
      - name: Install dependencies (including experimental)
        run: |
          python -m pip install -U pip setuptools wheel
          pip install --use-pep517 accumulation-tree # Geneformer dependency needs --use-pep517 for Cython
          GIT_CLONE_PROTECTION_ACTIVE=false pip install -r ./api/python/cellxgene_census/scripts/requirements-dev.txt
          pip install -e './api/python/cellxgene_census/[experimental]'
      - name: Report Dependency Versions
        run: pip list
      - name: Test with pytest (API, main tests)
        run: |
          PYTHONPATH=. coverage run --parallel-mode -m pytest -v -rP --durations=20 ./api/python/cellxgene_census/tests/
      - name: Test with pytest (API, experimental)
        run: |
          PYTHONPATH=. coverage run --parallel-mode -m pytest -v -rP --durations=20 --experimental ./api/python/cellxgene_census/tests/experimental
      - uses: actions/upload-artifact@v4
        with:
          name: coverage-api-${{ matrix.os }}-${{ matrix.python-version }}
          path: ./.coverage*
          include-hidden-files: true
          if-no-files-found: "error"
          retention-days: 3

  unit_tests_builder:
    strategy:
      matrix:
        os: [sc-dev-64g-runner]
        python-version: ["3.11"]

    runs-on: ${{matrix.os}}

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: |
            tools/**/pyproject.toml
            tools/**/requirements*.txt
      - name: Install dependencies
        run: |
          python -m pip install -U pip setuptools wheel
          pip install -e ./tools/cellxgene_census_builder/
          pip install -r ./tools/scripts/requirements-dev.txt
      - name: Test with pytest (builder)
        run: |
          # Run with and without Numba JIT. This gives a more accurate representation of code coverage
          PYTHONPATH=. coverage run --parallel-mode -m pytest -v -s -rP ./tools/cellxgene_census_builder/tests/
          PYTHONPATH=. NUMBA_DISABLE_JIT=1 coverage run --parallel-mode -m pytest -v -s -rP ./tools/cellxgene_census_builder/tests/
      - uses: actions/upload-artifact@v4
        with:
          name: coverage-builder-${{ matrix.os }}-${{ matrix.python-version }}
          path: ./.coverage*
          include-hidden-files: true
          if-no-files-found: "error"
          retention-days: 3

  submit-codecoverage:
    needs:
      - unit_tests_builder
      - unit_tests_python_api
    runs-on: [self-hosted, Linux, X64]
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/download-artifact@v4
        with:
          pattern: coverage-*
          merge-multiple: true
          path: .
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
          cache: "pip"
      - name: install requirements
        run: |
          pip install coverage
      - name: coverage report
        run: |
          coverage combine
          coverage xml
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          env_vars: OS,PYTHON
          files: ./coverage.xml
          flags: unittests
          name: codecov-umbrella