Skip to content
ci

TEST-#9999: Fix engine filter. #16254

Sign in to view logs

TEST-#9999: Fix engine filter.

TEST-#9999: Fix engine filter. #16254

Workflow file for this run

.github/workflows/ci.yml at ccad21c
name: ci
on:
pull_request:
paths:
# NOTE: keep these paths in sync with the paths that trigger the
# fuzzydata Github Actions in .github/workflows/fuzzydata-test.yml
- .github/workflows/**
- .github/actions/**
- '!.github/workflows/push-to-main.yml'
- asv_bench/**
- modin/**
- requirements/**
- scripts/**
- environment-dev.yml
- requirements-dev.txt
- setup.cfg
- setup.py
- versioneer.py
push:
schedule:
- cron: "30 2 * * WED"
- cron: "30 2 * * THU"
concurrency:
# Cancel other jobs in the same branch. We don't care whether CI passes
# on old commits.
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
env:
MODIN_GITHUB_CI: true
jobs:
python-filter:
runs-on: ubuntu-latest
outputs:
python-version: ${{ steps.choose.outputs.python-version }}
steps:
- id: choose
run: |
if [[ "${{ github.event.schedule }}" = "30 2 * * WED" ]]
then
echo "python-version=3.10" >> "$GITHUB_OUTPUT"
elif [[ "${{ github.event.schedule }}" = "30 2 * * THU" ]]
then
echo "python-version=3.11" >> "$GITHUB_OUTPUT"
else
echo "python-version=3.9" >> "$GITHUB_OUTPUT"
fi
# lint-flake8:
# needs: [python-filter]
# name: lint (flake8)
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
# - uses: ./.github/actions/python-only
# with:
# python-version: ${{ needs.python-filter.outputs.python-version }}
# # NOTE: If you are changing the set of packages installed here, make sure that
# # the dev requirements match them.
# - run: pip install flake8 flake8-print flake8-no-implicit-concat
# # NOTE: keep the flake8 command here in sync with the pre-commit hook in
# # /contributing/pre-commit
# - run: flake8 modin/ asv_bench/benchmarks scripts/doc_checker.py
execution-filter:
# Choose which executions we want to run all tests for on a pull request.
# We always test 'native' and 'python' executions completely because they
# are fast, but we only test ray, dask, and unidist, if we think this pull
# request is affecting how we execute with those engines specifically.
runs-on: ubuntu-latest
outputs:
ray: ${{ steps.filter.outputs.ray }}
dask: ${{ steps.filter.outputs.dask }}
unidist: ${{ steps.filter.outputs.unidist }}
engines: ${{ steps.engines.outputs.engines }}
experimental: ${{ steps.experimental.outputs.experimental }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@v3
id: filter
with:
filters: |
shared: &shared
- 'modin/core/execution/dispatching/**'
ray:
- *shared
- 'modin/core/execution/ray/**'
dask:
- *shared
- 'modin/core/execution/dask/**'
unidist:
- *shared
- 'modin/core/execution/unidist/**'
experimental:
- 'modin/experimental/**'
- uses: actions/setup-python@v5
- id: engines
run: |
python -c "import sys, json; print('engines=' + json.dumps(['python', 'native'] + (sys.argv[1] == 'true' and ['ray'] or []) + (sys.argv[2] == 'true' and ['dask'] or []) ))" \
"${{ steps.filter.outputs.ray }}" "${{ steps.filter.outputs.dask }}" >> $GITHUB_OUTPUT
- id: show-variables
run: echo ${{ steps.filter.outputs.ray }} , ${{ steps.filter.outputs.dask }} , ${{ steps.filter.outputs.unidist }} , ${{ steps.filter.outputs.experimental }}, ${{ steps.filter.outputs.test-native-dataframe-mode }}
test-sanity:
# The "sanity" tests run on each pull request to test that a subset of the
# full tests work with the slower engines (ray, dask, and unidist-MPI).
needs: [execution-filter, python-filter]
# If we don't need any of these, we get a single job with an empty matrix
# (that is, os, execution, etc. are not set and so we treat them as "").
# so, if the matrix is going to be empty, we need to skip this job completely:
# https://stackoverflow.com/a/77118991
if: |
github.event_name == 'pull_request' &&
(
needs.execution-filter.outputs.ray != 'true' ||
needs.execution-filter.outputs.dask != 'true' ||
needs.execution-filter.outputs.unidist != 'true'
)
strategy:
matrix:
os:
- ubuntu
- windows
python-version: [ "${{ needs.python-filter.outputs.python-version }}" ]
running-all-ray-tests: [ "${{ needs.execution-filter.outputs.ray }}" ]
running-all-dask-tests: [ "${{needs.execution-filter.outputs.dask}}" ]
running-all-unidist-tests: [ "${{needs.execution-filter.outputs.unidist}}" ]
execution: [ray, dask, unidist]
exclude:
- running-all-ray-tests: 'true'
execution: ray
- running-all-dask-tests: 'true'
execution: dask
- running-all-unidist-tests: 'true'
execution: unidist
# execution:
# - name: ray
# shell-ex: "python -m pytest"
# # If we're going to run all ray tests because we've detected a
# # change to the ray engine, we don't need to run these sanity tests
# # on ray.
# if: needs.execution-filter.outputs.ray != 'true'
# - name: dask
# shell-ex: "python -m pytest"
# # If we're going to run all dask tests because we've detected a
# # change to the dask engine, we don't need to run these sanity tests
# # on dask.
# if: needs.execution-filter.outputs.dask != 'true'
# - name: unidist
# shell-ex: "mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest"
# # If we're going to run all unidist tests because we've detected a
# # change to the unidist engine, we don't need to run these sanity tests
# # on unidist.
# if: needs.execution-filter.outputs.unidist != 'true'
runs-on: ${{ matrix.os }}-latest
defaults:
run:
shell: bash -l {0}
env:
MODIN_ENGINE: ${{ matrix.execution.name }}
UNIDIST_BACKEND: "mpi"
PARALLEL: ${{ matrix.execution.name != 'unidist' && matrix.os != 'windows' && '-n 2' || '' }}
PYTEST_COMMAND: >-
${{
(
(matrix.execution.name == 'ray' || matrix.execution.name == 'dask') &&
'python -m pytest'
) ||
(
matrix.execution.name == 'unidist' &&
'mpiexec -n 1 -genv AWS_ACCESS_KEY_ID foobar_key -genv AWS_SECRET_ACCESS_KEY foobar_secret python -m pytest'
) ||
'UNKNOWN_PYTEST_COMMAND'
}}
name: test-${{ matrix.os }}-sanity (engine ${{ matrix.execution.name }}, python ${{matrix.python-version}})
services:
moto:
image: ${{ matrix.os != 'windows' && 'motoserver/moto:5.0.13' || '' }}
ports:
- 5000:5000
env:
AWS_ACCESS_KEY_ID: foobar_key
AWS_SECRET_ACCESS_KEY: foobar_secret
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/mamba-env
with:
environment-file: ${{ matrix.os == 'ubuntu' && matrix.execution.name == 'unidist' && 'requirements/env_unidist_linux.yml' || matrix.os == 'windows' && matrix.execution.name == 'unidist' && 'requirements/env_unidist_win.yml' || 'environment-dev.yml' }}
activate-environment: ${{ matrix.execution.name == 'unidist' && 'modin_on_unidist' || 'modin' }}
python-version: ${{matrix.python-version}}
- name: Install HDF5
run: sudo apt update && sudo apt install -y libhdf5-dev
if: matrix.os != 'windows'
- name: Limit ray memory
run: echo "MODIN_MEMORY=1000000000" >> $GITHUB_ENV
if: matrix.os != 'windows' && matrix.execution.name == 'ray'
- name: Tell Modin to use existing ray cluster
run: echo "MODIN_RAY_CLUSTER=True" >> $GITHUB_ENV
if: matrix.os == 'windows' && matrix.execution.name == 'ray'
- name: Start local ray cluster
# Try a few times to start ray to work around
# https://github.com/modin-project/modin/issues/4562
uses: nick-fields/retry@v3
with:
timeout_minutes: 5
max_attempts: 5
command: ray start --head --port=6379 --object-store-memory=1000000000
if: matrix.os == 'windows' && matrix.execution.name == 'ray'
- run: MODIN_BENCHMARK_MODE=True $PYTEST_COMMAND modin/tests/pandas/internals/test_benchmark_mode.py
- run: $PYTEST_COMMAND $PARALLEL modin/tests/test_partition_api.py
- run: $PYTEST_COMMAND modin/tests/pandas/extensions
- name: xgboost tests
run: |
# TODO(https://github.com/modin-project/modin/issues/5194): Uncap xgboost
# when we use collective instead of rabit.
mamba install "xgboost>=1.7.1,<2.0.0" scikit-learn -c conda-forge
$PYTEST_COMMAND $PARALLEL \
modin/tests/experimental/xgboost/test_default.py \
modin/tests/experimental/xgboost/test_xgboost.py \
modin/tests/experimental/xgboost/test_dmatrix.py
if: matrix.os != 'windows' && needs.execution-filter.outputs.experimental == 'true'
- run: $PYTEST_COMMAND $PARALLEL modin/tests/experimental/test_pipeline.py
if: matrix.os != 'windows' && matrix.execution.name != 'unidist' && needs.execution-filter.outputs.experimental == 'true'
- name: "test DF: binary, default, iter"
run: |
$PYTEST_COMMAND $PARALLEL \
modin/tests/pandas/dataframe/test_binary.py \
modin/tests/pandas/dataframe/test_default.py \
modin/tests/pandas/dataframe/test_iter.py
if: matrix.os != 'windows'
- name: "test DF: reduce, udf, window, pickle"
run: |
$PYTEST_COMMAND $PARALLEL \
modin/tests/pandas/dataframe/test_reduce.py \
modin/tests/pandas/dataframe/test_udf.py \
modin/tests/pandas/dataframe/test_window.py \
modin/tests/pandas/dataframe/test_pickle.py
if: matrix.os != 'windows'
- run: $PYTEST_COMMAND modin/tests/pandas/test_series.py
if: matrix.execution.name == 'ray'
- run: $PYTEST_COMMAND -m "not exclude_in_sanity" modin/tests/pandas/test_series.py
if: matrix.execution.name != 'ray'
- run: $PYTEST_COMMAND modin/tests/pandas/dataframe/test_map_metadata.py
if: matrix.execution.name == 'ray'
- run: $PYTEST_COMMAND -m "not exclude_in_sanity" modin/tests/pandas/dataframe/test_map_metadata.py
if: matrix.execution.name != 'ray'
- name: "test rolling, expanding, reshape, general, concat"
run: |
$PYTEST_COMMAND $PARALLEL \
modin/tests/pandas/test_rolling.py \
modin/tests/pandas/test_expanding.py \
modin/tests/pandas/test_reshape.py \
modin/tests/pandas/test_general.py \
modin/tests/pandas/test_concat.py
if: matrix.os != 'windows'
- run: $PYTEST_COMMAND $PARALLEL modin/tests/numpy
- run: $PYTEST_COMMAND -m "not exclude_in_sanity" modin/tests/pandas/test_io.py --verbose
if: matrix.execution.name != 'unidist'
- uses: nick-fields/retry@v3
# to avoid issues with non-stable `to_csv` tests for unidist on MPI backend.
# for details see: https://github.com/modin-project/modin/pull/6776
with:
timeout_minutes: 15
max_attempts: 3
command: conda run --no-capture-output -n modin_on_unidist $PYTEST_COMMAND -m "not exclude_in_sanity" modin/tests/pandas/test_io.py --verbose
if: matrix.execution.name == 'unidist'
- run: $PYTEST_COMMAND modin/tests/experimental/test_io_exp.py
- run: $PYTEST_COMMAND $PARALLEL modin/tests/interchange/dataframe_protocol/test_general.py
- run: $PYTEST_COMMAND $PARALLEL modin/tests/interchange/dataframe_protocol/pandas/test_protocol.py
- name: Stop local ray cluster
run: ray stop
if: matrix.os == 'windows' && matrix.execution.name == 'ray'
- uses: ./.github/actions/upload-coverage