From d3560d9e5b036243b416fe13f45a63b11f527951 Mon Sep 17 00:00:00 2001 From: Chris B Date: Sun, 10 May 2020 11:01:31 +0100 Subject: [PATCH 1/8] Quick sketch of switching to nbval to discover issues/plan how it would look. --- README.rst | 107 ++++++++++++++++--------------- nbsmoke/__init__.py | 144 ++++++++++++------------------------------ setup.py | 23 ++----- tests/test_nbsmoke.py | 16 +---- tests/test_run.py | 56 ++++++++-------- 5 files changed, 132 insertions(+), 214 deletions(-) diff --git a/README.rst b/README.rst index 67ca035..304d720 100644 --- a/README.rst +++ b/README.rst @@ -14,12 +14,7 @@ nbsmoke ======= -Basic notebook smoke tests: Do they run ok? Do they contain lint? - ----- - -This `Pytest`_ plugin was generated with `Cookiecutter`_ along with `@hackebrot`_'s `Cookiecutter-pytest-plugin`_ template. - +Static checking of notebooks (e.g. do they contain lint?) Installation @@ -29,23 +24,14 @@ You can install nbsmoke via `pip`_ from `PyPI`_:: $ pip install nbsmoke -Or you can install nbsmoke via `conda`_ from `anaconda.org`_:: +Or via `conda`_ from `anaconda.org`_:: - $ conda install -c pyviz/label/dev -c conda-forge nbsmoke + $ conda install nbsmoke Usage ----- -Check all notebooks run without errors:: - - $ pytest --nbsmoke-run - -Check all notebooks run without errors, and store html to look at -afterwards:: - - $ pytest --nbsmoke-run --store-html=/scratch - Lint check notebooks:: $ pytest --nbsmoke-lint @@ -56,27 +42,18 @@ Lint failures as warnings only:: Instead of all files in a directory, you can specify a list e.g.:: - $ pytest --nbsmoke-run notebooks/Untitled*.ipynb + $ pytest --nbsmoke-lint notebooks/Untitled*.ipynb If you want to restrict pytest to running only your notebook tests, use `-k`, e.g.:: - $ pytest --nbsmoke-run -k ".ipynb" + $ pytest --nbsmoke-lint -k ".ipynb" +TODO: add ``--nbsmoke-verify`` docs! + Additional options are available by standard pytest 'ini' configuration in setup.cfg, pytest.ini, or tox.ini:: [pytest] - # when running, seconds allowed per cell (see nbconvert timeout) - nbsmoke_cell_timeout = 600 - - # notebooks to skip running; one case insensitive re to match per line - nbsmoke_skip_run = ^.*skipme\.ipynb$ - ^.*skipmetoo.*$ - - # case insensitive re to match for file to be considered notebook; - # defaults to ``^.*\.ipynb`` - it_is_nb_file = ^.*\.something$ - # flakes you don't want to hear about (regex) nbsmoke_flakes_to_ignore = .*hvplot.* imported but unused.* @@ -91,30 +68,19 @@ configuration in setup.cfg, pytest.ini, or tox.ini:: nbsmoke supports ``# noqa`` comments to mark that something should be ignored during lint checking. -The ``nbsmoke_skip_run`` list in a project's config can be ignored by -passing ``--ignore-nbsmoke-skip-run`` (useful if sometimes you want to -run all notebooks for a project where many are typically skipped). - What's the point? ----------------- -Although more sophisticated testing of notebooks is possible (e.g. see -nbval), just checking that notebooks run from start to finish without -error in a fresh kernel (or on a neutral CI service) can be useful -during development. Practical experience of working on several -projects with notebooks confirms this, but that's all the evidence I -have. +Checking notebooks for lint can find things like undefined names +faster than by running them. -Checking notebooks for lint might seem trivial/pointless, but it -frequently uncovers unused names (typically unused imports). It's also -quite common to find python 2 vs 3 problems, and sometimes undefined -names - in a way that's faster than running the notebook (over -multiple versions of python). +TODO: be able to switch linter? Or explicitly call the "lint" +pyflakes. -Unused imports/names themselves might seem trivial, but they can -hinder understanding of a notebook by readers, or add dependencies -that are not required. +Things that aren't errors, such as unused imports/names, might seem +trivial, but they can hinder understanding of a notebook by readers, +or add dependencies that are not required. Hopefully you don't have mysterious (unused) imports in your notebook, but if you do, you can add ``# noqa: explanation`` to stop flake @@ -126,6 +92,46 @@ simple promise: it will never complain about style, and it will try very, very hard to never emit false positives." +Deprecated Usage +---------------- + +nbsmoke used to support checking that notebooks run without error, and +could save the generated html. However, we now recommend using nbval +instead. ``--nbsmoke-run`` is still available, but just calls nbval; +eventually all options related to ``--nbsmoke-run`` will be removed from +nbsmoke. + +Old...Check all notebooks run without errors:: + + $ pytest --nbsmoke-run + +New...Use nbval instead:: + + $ pytest --nbval-lax + +Old stuff... + +If you want to restrict pytest to running only your notebook tests, use `-k`, e.g.:: + + $ pytest --nbsmoke-run -k ".ipynb" + +Additional options are available by standard pytest 'ini' +configuration in setup.cfg, pytest.ini, or tox.ini:: + + [pytest] + # when running, seconds allowed per cell (see nbconvert timeout) + nbsmoke_cell_timeout = 600 + + # notebooks to skip running; one case insensitive re to match per line + nbsmoke_skip_run = ^.*skipme\.ipynb$ + ^.*skipmetoo.*$ + + +The ``nbsmoke_skip_run`` list in a project's config can be ignored by +passing ``--ignore-nbsmoke-skip-run`` (useful if sometimes you want to +run all notebooks for a project where many are typically skipped). + + Contributing ------------ @@ -153,11 +159,8 @@ Issues If you encounter any problems, please `file an issue`_ (ideally including a copy of any problematic notebook). -.. _`Cookiecutter`: https://github.com/audreyr/cookiecutter -.. _`@hackebrot`: https://github.com/hackebrot .. _`BSD-3`: http://opensource.org/licenses/BSD-3-Clause -.. _`cookiecutter-pytest-plugin`: https://github.com/pytest-dev/cookiecutter-pytest-plugin -.. _`file an issue`: https://github.com/pyviz/nbsmoke/issues +.. _`file an issue`: https://github.com/pyviz-dev/nbsmoke/issues .. _`pytest`: https://github.com/pytest-dev/pytest .. _`tox`: https://tox.readthedocs.io/en/latest/ .. _`pip`: https://pypi.python.org/pypi/pip/ diff --git a/nbsmoke/__init__.py b/nbsmoke/__init__.py index 1906b03..db644d4 100644 --- a/nbsmoke/__init__.py +++ b/nbsmoke/__init__.py @@ -1,18 +1,10 @@ # -*- coding: utf-8 -*- -# Note: created with cookiecutter by someone with no experience of how -# to make a pytest plugin. Please question anything related to the -# pytest integration! - import re import os import io -import contextlib import pytest -import nbformat -import nbconvert -from nbconvert.preprocessors import ExecutePreprocessor try: from version import Version @@ -30,10 +22,6 @@ def pytest_addoption(parser): group = parser.getgroup('nbsmoke') - group.addoption( - '--nbsmoke-run', - action="store_true", - help="Run notebooks using nbconvert to check for exceptions.") group.addoption( '--nbsmoke-lint', @@ -55,111 +43,61 @@ def pytest_addoption(parser): action="store_true", help="Verify notebooks") + parser.addini('nbsmoke_flakes_to_ignore', "flake messages to ignore during nbsmoke's flake checking") + parser.addini('nbsmoke_flakes_cell_magics_blacklist', "cell magics you don't want to see - i.e. treat as lint.") + parser.addini('nbsmoke_flakes_line_magics_blacklist', "line magics you don't want to see - i.e. treat as lint") + + + ################## + ### DEPRECATED ### group.addoption( - '--store-html', - action="store", - dest='store_html', - default='', - help="When running, store rendered-to-html notebooks in the supplied path.") + '--nbsmoke-run', + action="store_true", + help="**DEPRECATED: Use --nbval-lax instead** Run notebooks using nbconvert to check for exceptions.") + # TODO! parser.addini('nbsmoke_cell_timeout', "nbsmoke's nbconvert cell timeout") - #### # TODO: hacks to work around pyviz team desire to not use pytest's markers - parser.addini('nbsmoke_skip_run', 're to skip (multi-line; one pattern per line)') + parser.addini('nbsmoke_skip_run', '**DEPRECATED: something something!** re to skip (multi-line; one pattern per line)') group.addoption( '--ignore-nbsmoke-skip-run', action="store_true", - help="Ignore any skip list in the ini file (allows to run all nbs if desired)") + help="**DEPRECATED: something something!** Ignore any skip list in the ini file (allows to run all nbs if desired)") #### - - # TODO: remove/rename/see pytest python_files - parser.addini('it_is_nb_file', 're to determine whether file is notebook') - - parser.addini('nbsmoke_flakes_to_ignore', "flake messages to ignore during nbsmoke's flake checking") - - parser.addini('nbsmoke_flakes_cell_magics_blacklist', "cell magics you don't want to see - i.e. treat as lint.") - parser.addini('nbsmoke_flakes_line_magics_blacklist', "line magics you don't want to see - i.e. treat as lint") - - -@contextlib.contextmanager -def cwd(d): - orig = os.getcwd() - os.chdir(d) - try: - yield - finally: - os.chdir(orig) - - - -################################################### - - -class RunNb(pytest.Item): - - def repr_failure(self, excinfo): - return excinfo.exconly(True) - - def runtest(self): - self._skip() - with io.open(self.name,encoding='utf8') as nb: - notebook = nbformat.read(nb, as_version=4) - - # TODO: which kernel? run in pytest's or use new one (make it option) - _timeout = self.parent.parent.config.getini('nbsmoke_cell_timeout') - kwargs = dict(timeout=int(_timeout) if _timeout!='' else 300, - allow_errors=False, - # or sys.version_info[1] ? - kernel_name='python') - - ep = ExecutePreprocessor(**kwargs) - with cwd(os.path.dirname(self.name)): # jupyter notebook always does this, right? - ep.preprocess(notebook,{}) - - # TODO: clean up this option handling - if self.parent.parent.config.option.store_html != '': - he = nbconvert.HTMLExporter() - # could maybe use this for chance of testing the html? but not the aim of this project - #he.template_file = 'basic' - html, resources = he.from_notebook_node(notebook) - with io.open(os.path.join(self.parent.parent.config.option.store_html,os.path.basename(self.name)+'.html'),'w',encoding='utf8') as f: - f.write(html) - - def _skip(self): - _skip_patterns = self.parent.parent.config.getini('nbsmoke_skip_run') - if not self.parent.parent.config.option.ignore_nbsmoke_skip_run: - for pattern in _skip_patterns.splitlines(): - if re.match(pattern,self.nodeid.split("::")[0],re.IGNORECASE): - pytest.skip() - + ################## + class IPyNbFile(pytest.File): - def __init__(self, fspath, parent=None, config=None, session=None, dowhat=RunNb): - self._dowhat = dowhat + def __init__(self, type_, fspath, parent=None, config=None, session=None): + self._type = type_ super(IPyNbFile,self).__init__(fspath, parent=parent, config=None, session=None) def collect(self): - yield self._dowhat(str(self.fspath), self) - + yield self._type(str(self.fspath), self) + def pytest_collect_file(path, parent): + if not path.fnmatch("*.ipynb"): + return + opt = parent.config.option - # TODO: Make this pattern standard/configurable. - # match .ipynb except .nbval.ipynb - it_is_nb_file = parent.config.getini('it_is_nb_file') - if it_is_nb_file == '': - #"^((?!\.nbval).)*\.ipynb$" - it_is_nb_file = r"^.*\.ipynb" - if re.match(it_is_nb_file,path.strpath,re.IGNORECASE): - if opt.nbsmoke_run or opt.nbsmoke_lint or opt.nbsmoke_verify: - # TODO express via the options system if you ever figure it out - # Hmm, should be able to do all - clean up! - assert (opt.nbsmoke_run ^ opt.nbsmoke_lint) ^ opt.nbsmoke_verify - if opt.nbsmoke_run: - dowhat = RunNb - elif opt.nbsmoke_lint: - dowhat = LintNb - elif opt.nbsmoke_verify: - dowhat = VerifyNb - return IPyNbFile(path, parent, dowhat=dowhat) + + # you have to pick one - can't currently run and lint and verify + # (though you should be able to) + + if opt.nbsmoke_run: + import nbval.plugin + # TODO: emit a deprecation warning + _skip_patterns = parent.config.getini('nbsmoke_skip_run') + # if skip_patterns: emit another deprecation warning! + if opt.ignore_nbsmoke_skip_run: + # TODO: emit another deprecation warning!! + _skip_patterns = [] + nbval.plugin.IPyNbFile._skip_patterns = _skip_patterns + return nbval.plugin.IPyNbFile(path, parent) + + elif opt.nbsmoke_lint: + return IPyNbFile(LintNb, path, parent) + elif opt.nbsmoke_verify: + return IPyNbFile(VerifyNb, path, parent) diff --git a/setup.py b/setup.py index dedb612..d2798dd 100644 --- a/setup.py +++ b/setup.py @@ -7,24 +7,24 @@ import version - def read(fname): file_path = os.path.join(os.path.dirname(__file__), fname) return codecs.open(file_path, encoding='utf-8').read() extras_require = { + 'run-checks': ['nbval'], 'holoviews-magics': ['holoviews'], 'verify': ['requests', 'beautifulsoup4'], } setup_args = dict( name='nbsmoke', - description='Basic notebook checks. Do they run? Do they contain lint?', + description='Static checking of Jupyter notebooks.', version = version.get_setup_version('nbsmoke'), - url='https://github.com/pyviz/nbsmoke', + url='https://github.com/pyviz-dev/nbsmoke', long_description=read('README.rst'), - author='pyviz contributors', - author_email = "dev@pyviz.org", + author='pyviz-dev contributors', + author_email = "developers@holoviz.org", license='BSD-3', packages=find_packages(), include_package_data = True, @@ -39,25 +39,16 @@ def read(fname): 'Operating System :: OS Independent', 'License :: OSI Approved :: BSD License', ], - - python_requires = ">=2.7", - + python_requires = ">=2.7", install_requires=[ 'pytest >=3.1.1', ########## lint stuff 'pyflakes', - ########## notebook stuff (reading, executing) + ########## notebook stuff # * Required imports: nbconvert, nbformat. - # * Need to be able to execute ipython notebooks. # * Optional: process ipython magics (required import: IPython) - 'jupyter_client', 'nbformat', 'nbconvert', - # TODO: I think the decision was to go with python/setup.py for this stuff, - # right? (but if so, how do I specify it's the runtime python version - # I'm talking aobut, not the buildtime python version?) - # Also - not sure exactly what is required now - ] + (['ipykernel'] if (sys.version_info[0]>=3 and sys.version_info[1]>4) else ['ipykernel <5']), extras_require = extras_require, entry_points={ 'pytest11': [ diff --git a/tests/test_nbsmoke.py b/tests/test_nbsmoke.py index f466317..1eb5ad5 100644 --- a/tests/test_nbsmoke.py +++ b/tests/test_nbsmoke.py @@ -8,11 +8,12 @@ def test_help_message(testdir): ) result.stdout.fnmatch_lines([ 'nbsmoke:', - '*--nbsmoke-run*', '*--nbsmoke-lint*', + '*--nbsmoke-lint-debug*', '*--nbsmoke-lint-onlywarn*', '*--nbsmoke-verify*', - '*--store-html*' + '*--nbsmoke-run*', + '*--ignore-nbsmoke-skip-run*', ]) @@ -40,14 +41,3 @@ def test_nbsmoke_cell_timeout(nbsmoke_cell_timeout): ]) assert result.ret == 0 - - -def test_it_is_nbfile(testdir): - testdir.makeini(r""" - [pytest] - it_is_nb_file = ^.*\.something$ - """) - testdir.makefile('.ipynb', testing123=nb_basic%{'the_source':"1/0"}) - result = testdir.runpytest('--nbsmoke-run','-v') - assert result.ret == 5 - result.stdout.fnmatch_lines(['*collected 0 items*']) diff --git a/tests/test_run.py b/tests/test_run.py index 7d18cc7..726d682 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- +# TODO: run is deprecated - will be able to remove when --nbsmoke-run +# is removed. + import os import io import sys @@ -22,8 +25,12 @@ def test_run_good(testdir): result = testdir.runpytest(*run_args) assert result.ret == 0 result.stdout.re_match_lines_random( - [".*collected 1 item$", - ".*testing123.ipynb.*PASSED.*"]) + [".*collected 4 items$", + ".*testing123::ipynb::Cell 0 PASSED.*", + ".*testing123::ipynb::Cell 1 PASSED.*", + ".*testing123::ipynb::Cell 2 PASSED.*", + ".*testing123::ipynb::Cell 3 PASSED.*", + ]) def test_run_bad(testdir): testdir.makefile('.ipynb', testing123=nb_basic%{'the_source':"1/0"}) @@ -31,29 +38,6 @@ def test_run_bad(testdir): assert result.ret == 1 result.stdout.re_match_lines_random([".*ZeroDivisionError.*"]) -def test_run_good_html(testdir): - outhtml = os.path.join(testdir.tmpdir.strpath,'testing123.ipynb.html') - assert not os.path.exists(outhtml) - - testdir.makefile('.ipynb', testing123=nb_basic%{'the_source':"42"}) - - result = testdir.runpytest(*(run_args+['--store-html=%s'%testdir.tmpdir.strpath])) - assert result.ret == 0 - - # test that html has happened - targets = [ - "
42
", - # note: this is really what happens in a python2 notebook - "
'中国'
" if sys.version_info[0]==3 else r"
u'\u4e2d\u56fd'
"] - answer = [None,None] - - with io.open(outhtml,encoding='utf8') as f: - for line in f: - for i,target in enumerate(targets): - if target in line: - answer[i] = 42 - assert answer == [42,42] - def test_skip_run(testdir): testdir.makeini(r""" @@ -68,11 +52,23 @@ def test_skip_run(testdir): result = testdir.runpytest(*run_args) assert result.ret == 0 result.stdout.re_match_lines_random( - [".*collected 4 items$", - ".*alsoskipme.ipynb.*SKIPPED", - ".*skipme.ipynb.*SKIPPED", - ".*skipmenot.ipynb.*PASSED", - ".*skipmetoo.ipynb.*SKIPPED"]) + [".*collected 16 items$", + ".*alsoskipme::ipynb.*SKIPPED", + ".*alsoskipme::ipynb::Cell 1 SKIPPED", + ".*alsoskipme::ipynb::Cell 2 SKIPPED", + ".*alsoskipme::ipynb::Cell 3 SKIPPED", + ".*skipme::ipynb::Cell 0 SKIPPED", + ".*skipme::ipynb::Cell 1 SKIPPED", + ".*skipme::ipynb::Cell 2 SKIPPED", + ".*skipme::ipynb::Cell 3 SKIPPED", + ".*skipmenot::ipynb::Cell 0 PASSED", + ".*skipmenot::ipynb::Cell 1 PASSED", + ".*skipmenot::ipynb::Cell 2 PASSED", + ".*skipmenot::ipynb::Cell 3 PASSED", + ".*skipmetoo::ipynb::Cell 0 SKIPPED", + ".*skipmetoo::ipynb::Cell 1 SKIPPED", + ".*skipmetoo::ipynb::Cell 2 SKIPPED", + ".*skipmetoo::ipynb::Cell 3 SKIPPED"]) def test_cwd_like_jupyter_notebook(testdir): p = testdir.tmpdir.mkdir("sub").join("hello.txt") From 3cf47aacdeba17beb4bc4257a06968906883ebf4 Mon Sep 17 00:00:00 2001 From: Chris B Date: Sun, 10 May 2020 11:09:09 +0100 Subject: [PATCH 2/8] lint --- nbsmoke/__init__.py | 2 -- tests/test_nbsmoke.py | 2 -- tests/test_run.py | 2 -- 3 files changed, 6 deletions(-) diff --git a/nbsmoke/__init__.py b/nbsmoke/__init__.py index db644d4..73aa05d 100644 --- a/nbsmoke/__init__.py +++ b/nbsmoke/__init__.py @@ -1,8 +1,6 @@ # -*- coding: utf-8 -*- -import re import os -import io import pytest diff --git a/tests/test_nbsmoke.py b/tests/test_nbsmoke.py index 1eb5ad5..4c3eadc 100644 --- a/tests/test_nbsmoke.py +++ b/tests/test_nbsmoke.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from . import nb_basic - def test_help_message(testdir): result = testdir.runpytest( '--help', diff --git a/tests/test_run.py b/tests/test_run.py index 726d682..ff74287 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -4,8 +4,6 @@ # is removed. import os -import io -import sys import shutil from . import nb_basic, run_args From 73a8095dca312ee07a491c6bcb7ecc168c37e487 Mon Sep 17 00:00:00 2001 From: Chris B Date: Sun, 10 May 2020 11:14:13 +0100 Subject: [PATCH 3/8] Fix setup syntax. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d2798dd..edff8d1 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ def read(fname): # * Required imports: nbconvert, nbformat. # * Optional: process ipython magics (required import: IPython) 'nbformat', - 'nbconvert', + 'nbconvert'], extras_require = extras_require, entry_points={ 'pytest11': [ From 9b84caf1071b4a6f08b8e8da0ef9e1af82859ce5 Mon Sep 17 00:00:00 2001 From: Chris B Date: Sun, 10 May 2020 11:27:32 +0100 Subject: [PATCH 4/8] lint --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index edff8d1..3384de5 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -import sys + import os import codecs from setuptools import setup, find_packages From 796f3cdbb6703bd4958e4ed31c8ce095e22e48f4 Mon Sep 17 00:00:00 2001 From: Chris B Date: Mon, 11 May 2020 11:46:24 +0100 Subject: [PATCH 5/8] Update test_skip_run. --- tests/test_run.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index ff74287..7c36486 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -36,13 +36,21 @@ def test_run_bad(testdir): assert result.ret == 1 result.stdout.re_match_lines_random([".*ZeroDivisionError.*"]) - def test_skip_run(testdir): - testdir.makeini(r""" - [pytest] - nbsmoke_skip_run = ^.*skipme\.ipynb$ - ^.*skipmetoo.*$ + testdir.makeconftest(""" + import pytest + import re + + skip_patterns = [".*skipme\.ipynb.*", + ".*skipmetoo.*"] + + def pytest_runtest_setup(item): + for pattern in skip_patterns: + if re.match(pattern, item.nodeid): + pytest.skip("Skipped by conftest") + break """) + testdir.makefile('.ipynb', skipme=nb_basic%{'the_source':"1/0"}) testdir.makefile('.ipynb', alsoskipme=nb_basic%{'the_source':"1/0"}) testdir.makefile('.ipynb', skipmetoo=nb_basic%{'the_source':"1/0"}) From 5eba5849a564ece691400f5a385c78a404a961be Mon Sep 17 00:00:00 2001 From: Chris B Date: Mon, 11 May 2020 11:46:27 +0100 Subject: [PATCH 6/8] Clarify deprecated options. --- README.rst | 66 +++++++++++++++++++++++++++++++++------------ nbsmoke/__init__.py | 41 +++++++++++++++++----------- 2 files changed, 75 insertions(+), 32 deletions(-) diff --git a/README.rst b/README.rst index 304d720..444ea3b 100644 --- a/README.rst +++ b/README.rst @@ -49,7 +49,7 @@ If you want to restrict pytest to running only your notebook tests, use `-k`, e. $ pytest --nbsmoke-lint -k ".ipynb" TODO: add ``--nbsmoke-verify`` docs! - + Additional options are available by standard pytest 'ini' configuration in setup.cfg, pytest.ini, or tox.ini:: @@ -92,44 +92,76 @@ simple promise: it will never complain about style, and it will try very, very hard to never emit false positives." -Deprecated Usage ----------------- +Unsupported usage +----------------- nbsmoke used to support checking that notebooks run without error, and could save the generated html. However, we now recommend using nbval -instead. ``--nbsmoke-run`` is still available, but just calls nbval; -eventually all options related to ``--nbsmoke-run`` will be removed from -nbsmoke. +instead. ``--nbsmoke-run`` is still available, but it just calls +nbval; eventually all options related to ``--nbsmoke-run`` will be +removed from nbsmoke. -Old...Check all notebooks run without errors:: +Before (deprecated)---check all notebooks run without errors:: $ pytest --nbsmoke-run -New...Use nbval instead:: +After---use nbval instead:: $ pytest --nbval-lax -Old stuff... - -If you want to restrict pytest to running only your notebook tests, use `-k`, e.g.:: - $ pytest --nbsmoke-run -k ".ipynb" +Note about kernel used to run notebook +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Additional options are available by standard pytest 'ini' -configuration in setup.cfg, pytest.ini, or tox.ini:: +By default, nbsmoke used the current environment's kernel, whereas +nbval uses the kernel stored in the notebook by default. To obtain +nbsmoke's behavior, pass ``--current-env``. See also: +https://github.com/computationalmodelling/nbval/issues/140 + + +Cell timeout +~~~~~~~~~~~~ + +Before (deprecated)---pytest configuration options in setup.cfg, +pytest.ini, or tox.ini:: [pytest] # when running, seconds allowed per cell (see nbconvert timeout) nbsmoke_cell_timeout = 600 + +After---nbval has the ``--nbval-cell-timeout`` option. Specify at the +command line, or add to pytest's options (in one of the above files):: + + [pytest] + addopts = --nbval-cell-timeout=600 + + +Skipping notebooks +~~~~~~~~~~~~~~~~~~ + +Before (no longer supported):: + # notebooks to skip running; one case insensitive re to match per line nbsmoke_skip_run = ^.*skipme\.ipynb$ ^.*skipmetoo.*$ -The ``nbsmoke_skip_run`` list in a project's config can be ignored by -passing ``--ignore-nbsmoke-skip-run`` (useful if sometimes you want to -run all notebooks for a project where many are typically skipped). +After---use pytest's own test selection and skipping +functionality. You can ignore certain files using ``--ignore`` or +``--ignore-glob`` at the command line, or add to pytest's options (in +one of the above files):: + + [pytest] + addopts = --ignore=path/to/skipme.ipynb + --ignore=path/of/skipmetoo.ipynb + + +Alternatively, for more complex scenarios or to explicitly get "skip" +in your test results, see pytest's ``-k`` option or use a +``conftest.py`` file. nbsmoke has an example of using ``conftest.py`` +in its own test suite (``test_skip_run`` in +https://github.com/pyviz-dev/nbsmoke/blob/master/tests/test_run.py). Contributing diff --git a/nbsmoke/__init__.py b/nbsmoke/__init__.py index 73aa05d..fa4de12 100644 --- a/nbsmoke/__init__.py +++ b/nbsmoke/__init__.py @@ -48,20 +48,20 @@ def pytest_addoption(parser): ################## ### DEPRECATED ### + # remove in 0.6 group.addoption( '--nbsmoke-run', action="store_true", - help="**DEPRECATED: Use --nbval-lax instead** Run notebooks using nbconvert to check for exceptions.") + help="**DEPRECATED: Use nbval instead** Run notebooks using nbconvert to check for exceptions.") - # TODO! - parser.addini('nbsmoke_cell_timeout', "nbsmoke's nbconvert cell timeout") + parser.addini('nbsmoke_cell_timeout', "**DEPRECATED: Use nbval instead** nbsmoke's nbconvert cell timeout") # TODO: hacks to work around pyviz team desire to not use pytest's markers - parser.addini('nbsmoke_skip_run', '**DEPRECATED: something something!** re to skip (multi-line; one pattern per line)') + parser.addini('nbsmoke_skip_run', '**DEPRECATED: Use a pytest option such as --ignore, --ignore-glob, -k, or conftest.py** re to skip (multi-line; one pattern per line)') group.addoption( '--ignore-nbsmoke-skip-run', action="store_true", - help="**DEPRECATED: something something!** Ignore any skip list in the ini file (allows to run all nbs if desired)") + help="**DEPRECATED: Use a pytest option such as --ignore, --ignore-glob, -k, or conftest.py** Ignore any skip list in the ini file (allows to run all nbs if desired)") #### ################## @@ -81,18 +81,29 @@ def pytest_collect_file(path, parent): opt = parent.config.option - # you have to pick one - can't currently run and lint and verify - # (though you should be able to) + # TODO: you have to pick one - can't currently run and lint and + # verify (though you should be able to) if opt.nbsmoke_run: - import nbval.plugin - # TODO: emit a deprecation warning - _skip_patterns = parent.config.getini('nbsmoke_skip_run') - # if skip_patterns: emit another deprecation warning! - if opt.ignore_nbsmoke_skip_run: - # TODO: emit another deprecation warning!! - _skip_patterns = [] - nbval.plugin.IPyNbFile._skip_patterns = _skip_patterns + import warnings + warnings.warn("--nbsmoke-run is deprecated: please use nbval (--nbval-lax) instead.", DeprecationWarning) + + import sys + import nbval.plugin + + if '--current-env' not in sys.argv: + opt.current_env = True + + if '--nbval-cell-timeout' not in sys.argv: + timeout = parent.config.getini('nbsmoke_cell_timeout') + if timeout != '': + opt.nbval_cell_timeout = timeout + + skip_patterns = parent.config.getini('nbsmoke_skip_run') + if skip_patterns.strip() != '': + if not '--ignore-nbsmoke-skip-run' in sys.argv: + raise ValueError("nbsmoke_skip_run regex no longer supported; use pytest one of pytest's own options instead: -k, --ignore, --ignore-glob, conftest.py." + return nbval.plugin.IPyNbFile(path, parent) elif opt.nbsmoke_lint: From ea827a3c0950bc5c569e3dba273b6fb07004d44d Mon Sep 17 00:00:00 2001 From: Chris B Date: Mon, 11 May 2020 11:50:28 +0100 Subject: [PATCH 7/8] Typo --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 444ea3b..00868eb 100644 --- a/README.rst +++ b/README.rst @@ -92,8 +92,8 @@ simple promise: it will never complain about style, and it will try very, very hard to never emit false positives." -Unsupported usage ------------------ +Deprecated usage +---------------- nbsmoke used to support checking that notebooks run without error, and could save the generated html. However, we now recommend using nbval From f674050666b744320619a63f128973442509ace7 Mon Sep 17 00:00:00 2001 From: Chris B Date: Mon, 11 May 2020 11:52:04 +0100 Subject: [PATCH 8/8] code typo, ha ha I should run the tests or lint check or all those things this project is about ;) --- nbsmoke/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nbsmoke/__init__.py b/nbsmoke/__init__.py index fa4de12..4284d4f 100644 --- a/nbsmoke/__init__.py +++ b/nbsmoke/__init__.py @@ -102,7 +102,7 @@ def pytest_collect_file(path, parent): skip_patterns = parent.config.getini('nbsmoke_skip_run') if skip_patterns.strip() != '': if not '--ignore-nbsmoke-skip-run' in sys.argv: - raise ValueError("nbsmoke_skip_run regex no longer supported; use pytest one of pytest's own options instead: -k, --ignore, --ignore-glob, conftest.py." + raise ValueError("nbsmoke_skip_run regex no longer supported; use pytest one of pytest's own options instead: -k, --ignore, --ignore-glob, conftest.py.") return nbval.plugin.IPyNbFile(path, parent)