test.yml

name: Tests

on:
  workflow_dispatch:

  push:
    branches:
      - main
      - develop
    tags:
      - "v*.*.*"

  pull_request:
    branches:
      - main
      - develop

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  test:
    name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }})
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: ["3.9"]
        scikit-learn: ["1.0.*", "1.1.*", "1.2.*", "1.3.*", "1.4.*", "1.5.*"]
        os: [ubuntu-latest]
        sklearn-only: ["true"]
        include:
          - os: ubuntu-latest
            python-version: "3.8"  # no scikit-learn 0.23 release for Python 3.9
            scikit-learn: "0.23.1"
            sklearn-only: "true"
          # scikit-learn 0.24 relies on scipy defaults, so we need to fix the version
          # c.f. https://github.com/openml/openml-python/pull/1267
          - os: ubuntu-latest
            python-version: "3.9"
            scikit-learn: "0.24"
            scipy: "1.10.0"
            sklearn-only: "true"
          # Do a Windows and Ubuntu test for _all_ openml functionality
          # I am not sure why these are on 3.8 and older scikit-learn
          - os: windows-latest
            python-version: "3.8"
            scikit-learn: 0.24.*
            scipy: "1.10.0"
            sklearn-only: 'false'
          # Include a code cov version
          - os: ubuntu-latest
            code-cov: true
            python-version: "3.8"
            scikit-learn: 0.23.1
            sklearn-only: 'false'
      fail-fast:  false

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 2
    - name: Setup Python ${{ matrix.python-version }}
      if: matrix.os != 'windows-latest'  # windows-latest only uses preinstalled Python (3.9.13)
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install test dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -e .[test]
    - name: Install scikit-learn ${{ matrix.scikit-learn }}
      run: |
        pip install scikit-learn==${{ matrix.scikit-learn }}
    - name: Install numpy for Python 3.8
      # Python 3.8 & scikit-learn<0.24 requires numpy<=1.23.5
      if: ${{ matrix.python-version == '3.8' && matrix.scikit-learn == '0.23.1' }}
      run: |
        pip install numpy==1.23.5
    - name: "Install NumPy 1.x and SciPy <1.11 for scikit-learn < 1.4"
      if: ${{ contains(fromJSON('["1.0.*", "1.1.*", "1.2.*", "1.3.*"]'), matrix.scikit-learn) }}
      run: |
        # scipy has a change to the 'mode' behavior which breaks scikit-learn < 1.4
        # numpy 2.0 has several breaking changes
        pip install "numpy<2.0" "scipy<1.11"
    - name: Install scipy ${{ matrix.scipy }}
      if: ${{ matrix.scipy }}
      run: |
        pip install scipy==${{ matrix.scipy }}
    - name: Store repository status
      id: status-before
      run: |
        echo "::set-output name=BEFORE::$(git status --porcelain -b)"
    - name: Show installed dependencies
      run: python -m pip list
    - name: Run tests on Ubuntu
      if: matrix.os == 'ubuntu-latest'
      run: |
        if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long  --cov-report=xml'; fi
        # Most of the time, running only the scikit-learn tests is sufficient
        if [ ${{ matrix.sklearn-only }} = 'true' ]; then sklearn='-m sklearn'; fi
        echo pytest -n 4 --durations=20 --dist load -sv $codecov $sklearn -o log_cli=true
        pytest -n 4 --durations=20 --dist load -sv $codecov $sklearn -o log_cli=true
    - name: Run tests on Windows
      if: matrix.os == 'windows-latest'
      run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
        pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1
    - name: Check for files left behind by test
      if: matrix.os != 'windows-latest' && always()
      run: |
        before="${{ steps.status-before.outputs.BEFORE }}"
        after="$(git status --porcelain -b)"
        if [[ "$before" != "$after" ]]; then
            echo "git status from before: $before"
            echo "git status from after: $after"
            echo "Not all generated files have been deleted!"
            exit 1
        fi
    - name: Upload coverage
      if: matrix.code-cov && always()
      uses: codecov/codecov-action@v4
      with:
        files: coverage.xml
        token: ${{ secrets.CODECOV_TOKEN }}
        fail_ci_if_error: true
        verbose: true