From 58a02c25998f46b9de587cc76a51d6351d96e415 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Fri, 29 Mar 2024 15:12:23 -0400 Subject: [PATCH] MAINT: Restore 2 jobs on Windows (#12520) --- .github/workflows/tests.yml | 9 ++- azure-pipelines.yml | 6 +- mne/export/_edf.py | 19 ++++++- mne/export/tests/test_export.py | 2 +- mne/morph.py | 19 ++++--- mne/utils/dataframe.py | 13 ++++- tools/azure_dependencies.sh | 38 +++---------- tools/github_actions_dependencies.sh | 44 +-------------- tools/install_pre_requirements.sh | 82 ++++++++++++++++++++++++++++ 9 files changed, 138 insertions(+), 94 deletions(-) create mode 100755 tools/install_pre_requirements.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d491a97029c..68979e20033 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - uses: pre-commit/action@v3.0.1 bandit: @@ -54,17 +54,16 @@ jobs: matrix: include: - os: ubuntu-latest - python: '3.11' + python: '3.12' kind: pip-pre - os: ubuntu-latest python: '3.12' kind: conda - # 3.12 needs https://github.com/conda-forge/dipy-feedstock/pull/50 - os: macos-14 # arm64 - python: '3.11' + python: '3.12' kind: mamba - os: macos-latest # intel - python: '3.11' + python: '3.12' kind: mamba - os: windows-latest python: '3.10' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b6f2fd679a1..09e2f2648c7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -238,7 +238,7 @@ stages: variables: MNE_LOGGING_LEVEL: 'warning' MNE_FORCE_SERIAL: 'true' - OPENBLAS_NUM_THREADS: '1' # deal with OpenBLAS conflicts safely on Windows + OPENBLAS_NUM_THREADS: '2' OMP_DYNAMIC: 'false' PYTHONUNBUFFERED: 1 PYTHONIOENCODING: 'utf-8' @@ -251,9 +251,9 @@ stages: 3.9 pip: TEST_MODE: 'pip' PYTHON_VERSION: '3.9' - 3.11 pip pre: + 3.12 pip pre: TEST_MODE: 'pip-pre' - PYTHON_VERSION: '3.11' + PYTHON_VERSION: '3.12' steps: - task: UsePythonVersion@0 inputs: diff --git a/mne/export/_edf.py b/mne/export/_edf.py index 7f7111a36d8..3f7e55b3d77 100644 --- a/mne/export/_edf.py +++ b/mne/export/_edf.py @@ -4,6 +4,7 @@ # Copyright the MNE-Python contributors. import datetime as dt +from typing import Callable import numpy as np @@ -11,7 +12,21 @@ _check_edfio_installed() from edfio import Edf, EdfAnnotation, EdfSignal, Patient, Recording # noqa: E402 -from edfio._utils import round_float_to_8_characters # noqa: E402 + + +# copied from edfio (Apache license) +def _round_float_to_8_characters( + value: float, + round_func: Callable[[float], int], +) -> float: + if isinstance(value, int) or value.is_integer(): + return value + length = 8 + integer_part_length = str(value).find(".") + if integer_part_length == length: + return round_func(value) + factor = 10 ** (length - 1 - integer_part_length) + return round_func(value * factor) / factor def _export_raw(fname, raw, physical_range, add_ch_type): @@ -52,7 +67,7 @@ def _export_raw(fname, raw, physical_range, add_ch_type): ) data = np.pad(data, (0, int(pad_width))) else: - data_record_duration = round_float_to_8_characters( + data_record_duration = _round_float_to_8_characters( np.floor(sfreq) / sfreq, round ) out_sfreq = np.floor(sfreq) / data_record_duration diff --git a/mne/export/tests/test_export.py b/mne/export/tests/test_export.py index 9ce72eb79de..9c8a60f50bb 100644 --- a/mne/export/tests/test_export.py +++ b/mne/export/tests/test_export.py @@ -145,7 +145,7 @@ def _create_raw_for_edf_tests(stim_channel_index=None): edfio_mark = pytest.mark.skipif( - not _check_edfio_installed(strict=False), reason="edfio not installed" + not _check_edfio_installed(strict=False), reason="unsafe use of private module" ) diff --git a/mne/morph.py b/mne/morph.py index 812ba23e095..5b8bfba41a7 100644 --- a/mne/morph.py +++ b/mne/morph.py @@ -190,7 +190,7 @@ def compute_source_morph( .. footbibliography:: """ src_data, kind, src_subject = _get_src_data(src) - subject_from = _check_subject_src(subject_from, src_subject) + subject_from = _check_subject_src(subject_from, src_subject, warn_none=True) del src _validate_type(src_to, (SourceSpaces, None), "src_to") _validate_type(subject_to, (str, None), "subject_to") @@ -823,19 +823,22 @@ def _resample_from_to(img, affine, to_vox_map): ############################################################################### # I/O -def _check_subject_src(subject, src, name="subject_from", src_name="src"): +def _check_subject_src( + subject, src, name="subject_from", src_name="src", *, warn_none=False +): if isinstance(src, str): subject_check = src elif src is None: # assume it's correct although dangerous but unlikely subject_check = subject else: subject_check = src._subject - if subject_check is None: - warn( - "The source space does not contain the subject name, we " - "recommend regenerating the source space (and forward / " - "inverse if applicable) for better code reliability" - ) + warn_none = True + if subject_check is None and warn_none: + warn( + "The source space does not contain the subject name, we " + "recommend regenerating the source space (and forward / " + "inverse if applicable) for better code reliability" + ) if subject is None: subject = subject_check elif subject_check is not None and subject != subject_check: diff --git a/mne/utils/dataframe.py b/mne/utils/dataframe.py index e4012c4d45d..95618c614fa 100644 --- a/mne/utils/dataframe.py +++ b/mne/utils/dataframe.py @@ -10,6 +10,7 @@ from ..defaults import _handle_default from ._logging import logger, verbose +from .check import check_version @verbose @@ -50,9 +51,17 @@ def _convert_times(times, time_format, meas_date=None, first_time=0): def _inplace(df, method, **kwargs): - """Handle transition: inplace=True (pandas <1.5) → copy=False (>=1.5).""" + # Handle transition: inplace=True (pandas <1.5) → copy=False (>=1.5) + # and 3.0 warning: + # E DeprecationWarning: The copy keyword is deprecated and will be removed in a + # future version. Copy-on-Write is active in pandas since 3.0 which utilizes a + # lazy copy mechanism that defers copies until necessary. Use .copy() to make + # an eager copy if necessary. _meth = getattr(df, method) # used for set_index() and rename() - if "copy" in signature(_meth).parameters: + + if check_version("pandas", "3.0"): + return _meth(**kwargs) + elif "copy" in signature(_meth).parameters: return _meth(**kwargs, copy=False) else: _meth(**kwargs, inplace=True) diff --git a/tools/azure_dependencies.sh b/tools/azure_dependencies.sh index c680fb100d6..cf4dd4726b9 100755 --- a/tools/azure_dependencies.sh +++ b/tools/azure_dependencies.sh @@ -1,38 +1,14 @@ -#!/bin/bash -ef +#!/bin/bash +set -eo pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) STD_ARGS="--progress-bar off --upgrade" -python -m pip install $STD_ARGS pip setuptools wheel packaging setuptools_scm +python -m pip install $STD_ARGS pip setuptools wheel if [ "${TEST_MODE}" == "pip" ]; then - python -m pip install --only-binary="numba,llvmlite,numpy,scipy,vtk" -e .[test,full] + python -m pip install $STD_ARGS --only-binary="numba,llvmlite,numpy,scipy,vtk" -e .[test,full] elif [ "${TEST_MODE}" == "pip-pre" ]; then - STD_ARGS="$STD_ARGS --pre" - # python -m pip install $STD_ARGS --only-binary ":all:" --extra-index-url "https://www.riverbankcomputing.com/pypi/simple" "PyQt6!=6.6.1,!=6.6.2" PyQt6-sip PyQt6-Qt6 "PyQt6-Qt6!=6.6.1,!=6.6.2" - python -m pip install $STD_ARGS --only-binary ":all:" "PyQt6!=6.6.1,!=6.6.2" PyQt6-sip PyQt6-Qt6 "PyQt6-Qt6!=6.6.1,!=6.6.2" - echo "Numpy etc." - # No pyarrow yet https://github.com/apache/arrow/issues/40216 - # No h5py (and thus dipy) yet until they improve/refactor thier wheel building infrastructure for Windows - python -m pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" "numpy>=2.1.0.dev0" "scipy>=1.14.0.dev0" "scikit-learn>=1.5.dev0" matplotlib pillow statsmodels - echo "OpenMEEG" - pip install $STD_ARGS --only-binary ":all:" --extra-index-url "https://test.pypi.org/simple" "openmeeg>=2.6.0.dev4" - echo "vtk" - python -m pip install $STD_ARGS --only-binary ":all:" --extra-index-url "https://wheels.vtk.org" vtk - echo "nilearn" - python -m pip install $STD_ARGS git+https://github.com/nilearn/nilearn - echo "pyvista/pyvistaqt" - python -m pip install --progress-bar off git+https://github.com/pyvista/pyvista - python -m pip install --progress-bar off git+https://github.com/pyvista/pyvistaqt - echo "misc" - python -m pip install $STD_ARGS imageio-ffmpeg xlrd mffpy pillow traitlets pybv eeglabio - echo "nibabel with workaround" - python -m pip install --progress-bar off git+https://github.com/nipy/nibabel.git - echo "joblib" - python -m pip install --progress-bar off git+https://github.com/joblib/joblib@master - echo "EDFlib-Python" - python -m pip install $STD_ARGS git+https://github.com/the-siesta-group/edfio - # echo "pysnirf2" # Needs h5py - # python -m pip install $STD_ARGS git+https://github.com/BUNPC/pysnirf2 - ./tools/check_qt_import.sh PyQt6 - python -m pip install $STD_ARGS -e .[test] + ${SCRIPT_DIR}/install_pre_requirements.sh + python -m pip install $STD_ARGS --pre -e .[test] else echo "Unknown run type ${TEST_MODE}" exit 1 diff --git a/tools/github_actions_dependencies.sh b/tools/github_actions_dependencies.sh index c41e4c8e2d8..36a0e16d4fb 100755 --- a/tools/github_actions_dependencies.sh +++ b/tools/github_actions_dependencies.sh @@ -2,6 +2,7 @@ set -o pipefail +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) STD_ARGS="--progress-bar off --upgrade" INSTALL_ARGS="-e" INSTALL_KIND="test_extra,hdf5" @@ -19,51 +20,10 @@ elif [ ! -z "$CONDA_DEPENDENCIES" ]; then STD_ARGS="--progress-bar off" INSTALL_KIND="test" else - echo "Install pip-pre dependencies" test "${MNE_CI_KIND}" == "pip-pre" STD_ARGS="$STD_ARGS --pre" - python -m pip install $STD_ARGS pip - echo "Numpy" - pip uninstall -yq numpy - echo "PyQt6" - # Now broken in latest release and in the pre release: - # pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 --extra-index-url https://www.riverbankcomputing.com/pypi/simple "PyQt6!=6.6.1,!=6.6.2" "PyQt6-Qt6!=6.6.1,!=6.6.2" - pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 "PyQt6!=6.6.1,!=6.6.2" "PyQt6-Qt6!=6.6.1,!=6.6.2" - echo "NumPy/SciPy/pandas etc." - # No pyarrow yet https://github.com/apache/arrow/issues/40216 - # No dipy yet https://github.com/dipy/dipy/issues/2979 - pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 --index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" "numpy>=2.1.0.dev0" h5py - pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 --extra-index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" "numpy>=2.1.0.dev0" "scipy>=1.14.0.dev0" "scikit-learn>=1.5.dev0" matplotlib pillow statsmodels pandas - # No python-picard (needs numexpr) until they update to NumPy 2.0 compat + ${SCRIPT_DIR}/install_pre_requirements.sh INSTALL_KIND="test_extra" - echo "OpenMEEG" - pip install $STD_ARGS --only-binary ":all:" --extra-index-url "https://test.pypi.org/simple" "openmeeg>=2.6.0.dev4" - # No Numba because it forces an old NumPy version - echo "nilearn" - pip install $STD_ARGS git+https://github.com/nilearn/nilearn - echo "VTK" - pip install $STD_ARGS --only-binary ":all:" --extra-index-url "https://wheels.vtk.org" vtk - python -c "import vtk" - echo "PyVista" - pip install $STD_ARGS git+https://github.com/drammock/pyvista@numpy-2-compat - echo "pyvistaqt" - pip install $STD_ARGS git+https://github.com/pyvista/pyvistaqt - echo "imageio-ffmpeg, xlrd, mffpy" - pip install $STD_ARGS imageio-ffmpeg xlrd mffpy patsy traitlets pybv eeglabio - echo "mne-qt-browser" - pip install $STD_ARGS git+https://github.com/mne-tools/mne-qt-browser - echo "nibabel with workaround" - pip install $STD_ARGS git+https://github.com/nipy/nibabel.git - echo "joblib" - pip install $STD_ARGS git+https://github.com/joblib/joblib@master - echo "edfio" - pip install $STD_ARGS git+https://github.com/the-siesta-group/edfio - echo "h5io" - pip install $STD_ARGS git+https://github.com/h5io/h5io - echo "pysnirf2" - pip install $STD_ARGS git+https://github.com/BUNPC/pysnirf2 - # Make sure we're on a NumPy 2.0 variant - python -c "import numpy as np; assert np.__version__[0] == '2', np.__version__" fi echo "" diff --git a/tools/install_pre_requirements.sh b/tools/install_pre_requirements.sh new file mode 100755 index 00000000000..1f54da654fe --- /dev/null +++ b/tools/install_pre_requirements.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +PLATFORM=$(python -c 'import platform; print(platform.system())') + +echo "Installing pip-pre dependencies on ${PLATFORM}" +STD_ARGS="--progress-bar off --upgrade --pre" + +# Dependencies of scientific-python-nightly-wheels are installed here so that +# we can use strict --index-url (instead of --extra-index-url) below +python -m pip install $STD_ARGS pip setuptools packaging \ + threadpoolctl cycler fonttools kiwisolver pyparsing pillow python-dateutil \ + patsy pytz tzdata nibabel tqdm trx-python joblib +echo "PyQt6" +# Now broken in latest release and in the pre release: +# pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 --extra-index-url https://www.riverbankcomputing.com/pypi/simple "PyQt6!=6.6.1,!=6.6.2" "PyQt6-Qt6!=6.6.1,!=6.6.2" +python -m pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 "PyQt6!=6.6.1,!=6.6.2" "PyQt6-Qt6!=6.6.1,!=6.6.2" +echo "NumPy/SciPy/pandas etc." +python -m pip uninstall -yq numpy +# No pyarrow yet https://github.com/apache/arrow/issues/40216 +# No h5py (and thus dipy) yet until they improve/refactor thier wheel building infrastructure for Windows +OTHERS="" +if [[ "${PLATFORM}" == "Linux" ]]; then + OTHERS="h5py dipy" +fi +python -m pip install $STD_ARGS --only-binary ":all:" --default-timeout=60 \ + --index-url "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" \ + "numpy>=2.1.0.dev0" "scipy>=1.14.0.dev0" "scikit-learn>=1.5.dev0" \ + matplotlib statsmodels pandas \ + $OTHERS + +# No python-picard (needs numexpr) until they update to NumPy 2.0 compat +# No Numba because it forces an old NumPy version + +echo "OpenMEEG" +python -m pip install $STD_ARGS --only-binary ":all:" --extra-index-url "https://test.pypi.org/simple" "openmeeg>=2.6.0.dev4" + +echo "nilearn" +python -m pip install $STD_ARGS git+https://github.com/nilearn/nilearn + +echo "VTK" +python -m pip install $STD_ARGS --only-binary ":all:" --extra-index-url "https://wheels.vtk.org" vtk +python -c "import vtk" + +echo "PyVista" +python -m pip install $STD_ARGS git+https://github.com/pyvista/pyvista + +echo "pyvistaqt" +pip install $STD_ARGS git+https://github.com/pyvista/pyvistaqt + +echo "imageio-ffmpeg, xlrd, mffpy" +pip install $STD_ARGS imageio-ffmpeg xlrd mffpy traitlets pybv eeglabio + +echo "mne-qt-browser" +pip install $STD_ARGS git+https://github.com/mne-tools/mne-qt-browser + +echo "nibabel" +pip install $STD_ARGS git+https://github.com/nipy/nibabel + +echo "joblib" +pip install $STD_ARGS git+https://github.com/joblib/joblib + +echo "edfio" +pip install $STD_ARGS git+https://github.com/the-siesta-group/edfio + +if [[ "${PLATFORM}" == "Linux" ]]; then + echo "h5io" + pip install $STD_ARGS git+https://github.com/h5io/h5io + + echo "pysnirf2" + pip install $STD_ARGS git+https://github.com/BUNPC/pysnirf2 +fi + +# Make sure we're on a NumPy 2.0 variant +echo "Checking NumPy version" +python -c "import numpy as np; assert np.__version__[0] == '2', np.__version__" + +# And that Qt works +echo "Checking Qt" +${SCRIPT_DIR}/check_qt_import.sh PyQt6