From 3075cb373d59d7c8fc836aeaea23a0327c98de0e Mon Sep 17 00:00:00 2001 From: Marcel Stimberg Date: Wed, 26 Jun 2024 17:20:58 +0200 Subject: [PATCH 1/7] Switch to pytest-cov --- .coveragerc | 3 +++ .github/workflows/testsuite.yml | 9 +++------ dev/continuous-integration/run_test_suite.py | 14 ++++++++++++-- numpy2.pyproject.toml | 2 +- pyproject.toml | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.coveragerc b/.coveragerc index 116e03964..362d72dcc 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,9 @@ # .coveragerc to control coverage.py # following the example at http://nedbatchelder.com/code/coverage/config.html [run] +concurrency = multiprocessing +parallel = True +sigterm = True relative_files = True branch = True source_pkgs = brian2 diff --git a/.github/workflows/testsuite.yml b/.github/workflows/testsuite.yml index 085a227fa..4b9a20eb8 100644 --- a/.github/workflows/testsuite.yml +++ b/.github/workflows/testsuite.yml @@ -73,7 +73,7 @@ jobs: - name: Install Brian2 and dependencies run: | conda install --quiet --yes pip gsl - python -m pip install .[test] coverage + python -m pip install .[test] - name: Determine Cython cache dir id: cython-cache @@ -92,9 +92,8 @@ jobs: - name: Run Tests run: | cd $GITHUB_WORKSPACE/.. # move out of the workspace to avoid direct import - coverage run --rcfile=$GITHUB_WORKSPACE/.coveragerc $GITHUB_WORKSPACE/$SCRIPT_NAME - coverage lcov --rcfile=$GITHUB_WORKSPACE/.coveragerc - cp coverage.lcov $GITHUB_WORKSPACE/ + python $GITHUB_WORKSPACE/$SCRIPT_NAME + cp coverage.xml $GITHUB_WORKSPACE/ env: SCRIPT_NAME: dev/continuous-integration/run_test_suite.py SPHINX_DIR: ${{ github.workspace }}/docs_sphinx @@ -106,8 +105,6 @@ jobs: uses: coverallsapp/github-action@v2.3.0 with: parallel: true - format: lcov - file: coverage.lcov flag-name: run ${{ join(matrix.*, ' - ') }} coveralls: diff --git a/dev/continuous-integration/run_test_suite.py b/dev/continuous-integration/run_test_suite.py index 0361b493d..6b4fa13a6 100644 --- a/dev/continuous-integration/run_test_suite.py +++ b/dev/continuous-integration/run_test_suite.py @@ -20,7 +20,6 @@ operating_system = os.environ.get('AGENT_OS', 'unknown').lower() cross_compiled = os.environ.get('CROSS_COMPILED', 'FALSE').lower() in ['yes', 'true'] do_not_reset_preferences = os.environ.get('DO_NOT_RESET_PREFERENCES', 'false').lower() in ['yes', 'true'] - report_coverage = os.environ.get('REPORT_COVERAGE', 'no').lower() in ['yes', 'true'] dtype_32_bit = os.environ.get('FLOAT_DTYPE_32', 'no').lower() in ['yes', 'true'] sphinx_dir = os.environ.get('SPHINX_DIR') src_dir = os.environ.get('SRCDIR') @@ -54,7 +53,18 @@ if deprecation_error: args = ['-W', 'error::DeprecationWarning', '--tb=short'] else: - args = [] + # Use coverage when running on GitHub + if "GITHUB_WORKSPACE" in os.environ: + args = [ + "--cov", + "--cov-append", + "--cov-report", + "xml", + "--cov-report", + "term", + "--cov-config", + os.path.join(os.environ["GITHUB_WORKSPACE"], ".coveragerc"), + ] if standalone: result = brian2.test([], diff --git a/numpy2.pyproject.toml b/numpy2.pyproject.toml index d3c0fb458..534306db4 100644 --- a/numpy2.pyproject.toml +++ b/numpy2.pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] [project.optional-dependencies] -test = ['pytest', 'pytest-xdist>=1.22.3'] +test = ['pytest', 'pytest-xdist>=1.22.3', 'pytest-cov>=2.0'] docs = ['sphinx>=7', 'ipython>=5', 'sphinx-tabs'] [project.urls] diff --git a/pyproject.toml b/pyproject.toml index 1873012ef..d95dfcad6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] [project.optional-dependencies] -test = ['pytest', 'pytest-xdist>=1.22.3'] +test = ['pytest', 'pytest-xdist>=1.22.3', 'pytest-cov>=2.0'] docs = ['sphinx>=7', 'ipython>=5', 'sphinx-tabs'] [project.urls] From 239f12fc27b6ccdbdaf060ac9bd829bd6fb5d309 Mon Sep 17 00:00:00 2001 From: Marcel Stimberg Date: Wed, 26 Jun 2024 17:20:58 +0200 Subject: [PATCH 2/7] Use spawn for multiprocessing in test suite --- dev/continuous-integration/run_test_suite.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/continuous-integration/run_test_suite.py b/dev/continuous-integration/run_test_suite.py index 6b4fa13a6..72f753b22 100644 --- a/dev/continuous-integration/run_test_suite.py +++ b/dev/continuous-integration/run_test_suite.py @@ -5,6 +5,8 @@ # Importing multiprocessing here seems to fix hangs in the test suite on OS X # see https://github.com/scipy/scipy/issues/11835 import multiprocessing +# Prevent potential issues on multi-threaded execution +multiprocessing.set_start_method('spawn') import os import sys From 9073cf11d3229c1fb7cacd6125068ccffec7056a Mon Sep 17 00:00:00 2001 From: Marcel Stimberg Date: Wed, 26 Jun 2024 17:20:58 +0200 Subject: [PATCH 3/7] Don't use multiprocessing context manager in tests --- brian2/tests/test_cpp_standalone.py | 6 +++++- brian2/tests/test_logger.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/brian2/tests/test_cpp_standalone.py b/brian2/tests/test_cpp_standalone.py index e75a636a9..7a23c3c26 100644 --- a/brian2/tests/test_cpp_standalone.py +++ b/brian2/tests/test_cpp_standalone.py @@ -958,8 +958,12 @@ def test_change_parameters_multiprocessing(): import multiprocessing - with multiprocessing.Pool() as p: + p = multiprocessing.Pool() + try: results = p.map(sim.run_sim, range(5)) + finally: + p.close() + p.join() for idx, result in zip(range(5), results): v, w, x = result diff --git a/brian2/tests/test_logger.py b/brian2/tests/test_logger.py index 77806795b..d1b8bff30 100644 --- a/brian2/tests/test_logger.py +++ b/brian2/tests/test_logger.py @@ -59,9 +59,13 @@ def run_in_process_with_logger(x): @pytest.mark.codegen_independent def test_file_logging_multiprocessing(): logger.info("info message before multiprocessing") + p = multiprocessing.Pool() - with multiprocessing.Pool() as p: + try: p.map(run_in_process, range(3)) + finally: + p.close() + p.join() BrianLogger.file_handler.flush() assert os.path.isfile(BrianLogger.tmp_log) @@ -75,8 +79,12 @@ def test_file_logging_multiprocessing(): def test_file_logging_multiprocessing_with_loggers(): logger.info("info message before multiprocessing") - with multiprocessing.Pool() as p: + p = multiprocessing.Pool() + try: log_files = p.map(run_in_process_with_logger, range(3)) + finally: + p.close() + p.join() BrianLogger.file_handler.flush() assert os.path.isfile(BrianLogger.tmp_log) From 9a6e5eb3af09f4076f0565f5c70e67049188c807 Mon Sep 17 00:00:00 2001 From: Marcel Stimberg Date: Wed, 26 Jun 2024 17:20:58 +0200 Subject: [PATCH 4/7] Stop hanging tests after 5min --- brian2/tests/pytest.ini | 3 +++ numpy2.pyproject.toml | 2 +- pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/brian2/tests/pytest.ini b/brian2/tests/pytest.ini index c8a9b88e0..67831b14d 100644 --- a/brian2/tests/pytest.ini +++ b/brian2/tests/pytest.ini @@ -20,3 +20,6 @@ filterwarnings = ignore:BaseException:DeprecationWarning ignore:invalid value:RuntimeWarning ignore:divide by zero:RuntimeWarning + +# Fail tests after 5 minutes +timeout = 300 \ No newline at end of file diff --git a/numpy2.pyproject.toml b/numpy2.pyproject.toml index 534306db4..c9579fceb 100644 --- a/numpy2.pyproject.toml +++ b/numpy2.pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] [project.optional-dependencies] -test = ['pytest', 'pytest-xdist>=1.22.3', 'pytest-cov>=2.0'] +test = ['pytest', 'pytest-xdist>=1.22.3', 'pytest-cov>=2.0', 'pytest-timeout'] docs = ['sphinx>=7', 'ipython>=5', 'sphinx-tabs'] [project.urls] diff --git a/pyproject.toml b/pyproject.toml index d95dfcad6..aab005e37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ classifiers = [ ] [project.optional-dependencies] -test = ['pytest', 'pytest-xdist>=1.22.3', 'pytest-cov>=2.0'] +test = ['pytest', 'pytest-xdist>=1.22.3', 'pytest-cov>=2.0', 'pytest-timeout'] docs = ['sphinx>=7', 'ipython>=5', 'sphinx-tabs'] [project.urls] From 76c88bba50e44186b4d59f802c643f809dea6345 Mon Sep 17 00:00:00 2001 From: Marcel Stimberg Date: Wed, 26 Jun 2024 17:20:58 +0200 Subject: [PATCH 5/7] force multiprocessing start method --- dev/continuous-integration/run_test_suite.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/continuous-integration/run_test_suite.py b/dev/continuous-integration/run_test_suite.py index 72f753b22..7d4b6f43d 100644 --- a/dev/continuous-integration/run_test_suite.py +++ b/dev/continuous-integration/run_test_suite.py @@ -6,7 +6,7 @@ # see https://github.com/scipy/scipy/issues/11835 import multiprocessing # Prevent potential issues on multi-threaded execution -multiprocessing.set_start_method('spawn') +multiprocessing.set_start_method('spawn', force=True) import os import sys From 58171f86f1e1b1b07b4f5badebb32f49b6262533 Mon Sep 17 00:00:00 2001 From: Marcel Stimberg Date: Wed, 26 Jun 2024 17:20:58 +0200 Subject: [PATCH 6/7] Do not log general python warnings to file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mostly because it messes with the logging tests in our test suite… --- brian2/utils/logger.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/brian2/utils/logger.py b/brian2/utils/logger.py index da65bbf24..c4a5b0772 100644 --- a/brian2/utils/logger.py +++ b/brian2/utils/logger.py @@ -697,8 +697,6 @@ def initialize(): # interface... warn_logger = logging.getLogger("py.warnings") warn_logger.addHandler(BrianLogger.console_handler) - if BrianLogger.file_handler is not None: - warn_logger.addHandler(BrianLogger.file_handler) # Put some standard info into the log file logger.log( From 1a73a1f2cc72c12389ba0fa9f6629d4ca4ee255b Mon Sep 17 00:00:00 2001 From: Marcel Stimberg Date: Thu, 27 Jun 2024 09:55:04 +0200 Subject: [PATCH 7/7] Increase timout and filter overflow warnings --- brian2/tests/pytest.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/brian2/tests/pytest.ini b/brian2/tests/pytest.ini index 67831b14d..b4f93e659 100644 --- a/brian2/tests/pytest.ini +++ b/brian2/tests/pytest.ini @@ -20,6 +20,7 @@ filterwarnings = ignore:BaseException:DeprecationWarning ignore:invalid value:RuntimeWarning ignore:divide by zero:RuntimeWarning + ignore:overflow:RuntimeWarning -# Fail tests after 5 minutes -timeout = 300 \ No newline at end of file +# Fail tests after 10 minutes +timeout = 600 \ No newline at end of file