From dbc24e115938f6caa9448724008818fbd21fb396 Mon Sep 17 00:00:00 2001 From: Zac Hatfield-Dodds Date: Thu, 1 Feb 2024 23:44:58 -0800 Subject: [PATCH] Update deps + release new version --- deps/check.txt | 54 +++++++---------------- deps/docs.txt | 63 +++++++++++++-------------- deps/test.txt | 45 ++++++++++--------- docs-src/changelog.md | 5 +++ setup.py | 2 +- src/hypofuzz/__init__.py | 2 +- src/hypofuzz/dashboard.py | 15 +++++-- src/hypofuzz/hy.py | 13 +++--- src/hypofuzz/interface.py | 1 + tests/test_fuzz_process.py | 89 +++++++++++++++++++------------------- 10 files changed, 139 insertions(+), 150 deletions(-) diff --git a/deps/check.txt b/deps/check.txt index 384d93b..fc82781 100644 --- a/deps/check.txt +++ b/deps/check.txt @@ -4,68 +4,44 @@ # # pip-compile --annotation-style=line --output-file=deps/check.txt deps/check.in # -annotated-types==0.6.0 # via pydantic -attrs==23.1.0 # via flake8-bugbear +attrs==23.2.0 # via flake8-bugbear autoflake==2.2.1 # via shed -bandit==1.7.6 # via flake8-bandit -black==23.11.0 # via shed -cerberus==1.3.5 # via plette -certifi==2023.11.17 # via requests -charset-normalizer==3.3.2 # via requests +bandit==1.7.7 # via flake8-bandit +black==24.1.1 # via shed click==8.1.7 # via black com2ann==0.3.0 # via shed -distlib==0.3.7 # via requirementslib -docopt==0.6.2 # via pipreqs -flake8==6.1.0 # via -r deps/check.in, flake8-bandit, flake8-bugbear, flake8-builtins, flake8-comprehensions, flake8-docstrings, flake8-print, pep8-naming +flake8==7.0.0 # via -r deps/check.in, flake8-bandit, flake8-bugbear, flake8-builtins, flake8-comprehensions, flake8-docstrings, flake8-print, pep8-naming flake8-bandit==4.1.1 # via -r deps/check.in -flake8-bugbear==23.12.2 # via -r deps/check.in +flake8-bugbear==24.1.17 # via -r deps/check.in flake8-builtins==2.2.0 # via -r deps/check.in flake8-comprehensions==3.14.0 # via -r deps/check.in flake8-docstrings==1.7.0 # via -r deps/check.in flake8-print==5.0.0 # via -r deps/check.in -gitdb==4.0.11 # via gitpython -gitpython==3.1.40 # via bandit -idna==3.6 # via requests -isort==5.13.0 # via shed +isort==5.13.2 # via shed libcst==1.1.0 # via shed markdown-it-py==3.0.0 # via rich mccabe==0.7.0 # via flake8 mdurl==0.1.2 # via markdown-it-py -mypy==1.7.1 # via -r deps/check.in +mypy==1.8.0 # via -r deps/check.in mypy-extensions==1.0.0 # via black, mypy, typing-inspect packaging==23.2 # via black -pathspec==0.11.2 # via black +pathspec==0.12.1 # via black pbr==6.0.0 # via stevedore -pep517==0.13.1 # via requirementslib pep8-naming==0.13.3 # via -r deps/check.in -pip-api==0.0.30 # via isort -pipreqs==0.4.13 # via isort -platformdirs==4.1.0 # via black, requirementslib -plette[validation]==0.4.4 # via requirementslib +platformdirs==4.2.0 # via black pycodestyle==2.11.1 # via flake8, flake8-print -pydantic==2.5.2 # via requirementslib -pydantic-core==2.14.5 # via pydantic pydocstyle==6.3.0 # via flake8-docstrings -pyflakes==3.1.0 # via autoflake, flake8 +pyflakes==3.2.0 # via autoflake, flake8 pygments==2.17.2 # via rich pyupgrade==3.15.0 # via shed pyyaml==6.0.1 # via bandit, libcst -requests==2.31.0 # via requirementslib, yarg -requirementslib==3.0.0 # via isort rich==13.7.0 # via bandit -shed==2023.6.1 # via -r deps/check.in -smmap==5.0.1 # via gitdb +shed==2024.1.1 # via -r deps/check.in snowballstemmer==2.2.0 # via pydocstyle stevedore==5.1.0 # via bandit tokenize-rt==5.2.0 # via pyupgrade -tomli==2.0.1 # via autoflake, black, mypy, pep517 -tomlkit==0.12.3 # via plette, requirementslib -types-requests==2.31.0.10 # via -r deps/check.in -typing-extensions==4.8.0 # via black, libcst, mypy, pydantic, pydantic-core, typing-inspect +tomli==2.0.1 # via autoflake, black, mypy +types-requests==2.31.0.20240125 # via -r deps/check.in +typing-extensions==4.9.0 # via black, libcst, mypy, typing-inspect typing-inspect==0.9.0 # via libcst -urllib3==2.1.0 # via requests, types-requests -yarg==0.1.9 # via pipreqs - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools +urllib3==2.2.0 # via types-requests diff --git a/deps/docs.txt b/deps/docs.txt index cfc641d..2d9d62c 100644 --- a/deps/docs.txt +++ b/deps/docs.txt @@ -4,53 +4,52 @@ # # pip-compile --annotation-style=line --output-file=deps/docs.txt deps/docs.in setup.py # -alabaster==0.7.13 # via sphinx -ansi2html==1.8.0 # via dash -attrs==23.1.0 # via hypothesis -babel==2.13.1 # via sphinx -black==23.11.0 # via hypofuzz (setup.py), hypothesis +alabaster==0.7.16 # via sphinx +attrs==23.2.0 # via hypothesis +babel==2.14.0 # via sphinx +black==24.1.1 # via hypofuzz (setup.py), hypothesis blinker==1.7.0 # via flask -certifi==2023.11.17 # via requests +certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via black, flask, hypothesis -coverage==7.3.2 # via hypofuzz (setup.py) -dash==2.14.2 # via hypofuzz (setup.py) +coverage==7.4.1 # via hypofuzz (setup.py) +dash==2.15.0 # via hypofuzz (setup.py) dash-core-components==2.0.0 # via dash dash-html-components==2.0.0 # via dash dash-table==5.0.0 # via dash docutils==0.20.1 # via myst-parser, pybtex-docutils, sphinx, sphinx-rtd-theme, sphinxcontrib-bibtex exceptiongroup==1.2.0 # via hypothesis, pytest -flask==3.0.0 # via dash -hypothesis[cli]==6.91.1 # via hypofuzz (setup.py) +flask==3.0.1 # via dash +hypothesis[cli]==6.97.4 # via hypofuzz (setup.py) idna==3.6 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.0 # via dash +importlib-metadata==7.0.1 # via dash iniconfig==2.0.0 # via pytest itsdangerous==2.1.2 # via flask -jinja2==3.1.2 # via flask, myst-parser, sphinx +jinja2==3.1.3 # via flask, myst-parser, sphinx latexcodec==2.0.1 # via pybtex libcst==1.1.0 # via hypofuzz (setup.py) markdown-it-py==3.0.0 # via mdit-py-plugins, myst-parser, rich -markupsafe==2.1.3 # via jinja2, werkzeug +markupsafe==2.1.4 # via jinja2, werkzeug mdit-py-plugins==0.4.0 # via myst-parser mdurl==0.1.2 # via markdown-it-py mypy-extensions==1.0.0 # via black, typing-inspect myst-parser==2.0.0 # via -r deps/docs.in -nest-asyncio==1.5.8 # via dash -numpy==1.26.2 # via pandas +nest-asyncio==1.6.0 # via dash +numpy==1.26.3 # via pandas packaging==23.2 # via black, plotly, pytest, sphinx -pandas==2.1.4 # via hypofuzz (setup.py) -pathspec==0.11.2 # via black -platformdirs==4.1.0 # via black +pandas==2.2.0 # via hypofuzz (setup.py) +pathspec==0.12.1 # via black +platformdirs==4.2.0 # via black plotly==5.18.0 # via dash -pluggy==1.3.0 # via pytest -psutil==5.9.6 # via hypofuzz (setup.py) +pluggy==1.4.0 # via pytest +psutil==5.9.8 # via hypofuzz (setup.py) pybtex==0.24.0 # via pybtex-docutils, sphinxcontrib-bibtex pybtex-docutils==1.0.3 # via sphinxcontrib-bibtex pygments==2.17.2 # via rich, sphinx -pytest==7.4.3 # via hypofuzz (setup.py) +pytest==8.0.0 # via hypofuzz (setup.py) python-dateutil==2.8.2 # via pandas -pytz==2023.3.post1 # via pandas +pytz==2024.1 # via pandas pyyaml==6.0.1 # via libcst, myst-parser, pybtex requests==2.31.0 # via dash, hypofuzz (setup.py), sphinx retrying==1.3.4 # via dash @@ -58,23 +57,23 @@ rich==13.7.0 # via hypothesis six==1.16.0 # via latexcodec, pybtex, python-dateutil, retrying snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via hypothesis -sphinx==7.2.6 # via -r deps/docs.in, myst-parser, sphinx-rtd-theme, sphinxcontrib-applehelp, sphinxcontrib-bibtex, sphinxcontrib-devhelp, sphinxcontrib-htmlhelp, sphinxcontrib-jquery, sphinxcontrib-programoutput, sphinxcontrib-qthelp, sphinxcontrib-serializinghtml +sphinx==7.2.6 # via -r deps/docs.in, myst-parser, sphinx-rtd-theme, sphinxcontrib-bibtex, sphinxcontrib-jquery, sphinxcontrib-programoutput sphinx-rtd-theme==2.0.0 # via -r deps/docs.in -sphinxcontrib-applehelp==1.0.7 # via sphinx -sphinxcontrib-bibtex==2.6.1 # via -r deps/docs.in -sphinxcontrib-devhelp==1.0.5 # via sphinx -sphinxcontrib-htmlhelp==2.0.4 # via sphinx +sphinxcontrib-applehelp==1.0.8 # via sphinx +sphinxcontrib-bibtex==2.6.2 # via -r deps/docs.in +sphinxcontrib-devhelp==1.0.6 # via sphinx +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jquery==4.1 # via sphinx-rtd-theme sphinxcontrib-jsmath==1.0.1 # via sphinx sphinxcontrib-programoutput==0.17 # via -r deps/docs.in -sphinxcontrib-qthelp==1.0.6 # via sphinx -sphinxcontrib-serializinghtml==1.1.9 # via sphinx +sphinxcontrib-qthelp==1.0.7 # via sphinx +sphinxcontrib-serializinghtml==1.1.10 # via sphinx tenacity==8.2.3 # via plotly tomli==2.0.1 # via black, pytest -typing-extensions==4.8.0 # via black, dash, libcst, typing-inspect +typing-extensions==4.9.0 # via black, dash, libcst, typing-inspect typing-inspect==0.9.0 # via libcst -tzdata==2023.3 # via pandas -urllib3==2.1.0 # via requests +tzdata==2023.4 # via pandas +urllib3==2.2.0 # via requests werkzeug==3.0.1 # via dash, flask zipp==3.17.0 # via importlib-metadata diff --git a/deps/test.txt b/deps/test.txt index 4738d20..5636d1b 100644 --- a/deps/test.txt +++ b/deps/test.txt @@ -4,47 +4,46 @@ # # pip-compile --annotation-style=line --output-file=deps/test.txt deps/test.in setup.py # -ansi2html==1.8.0 # via dash -attrs==23.1.0 # via hypothesis -black==23.11.0 # via hypofuzz (setup.py), hypothesis +attrs==23.2.0 # via hypothesis +black==24.1.1 # via hypofuzz (setup.py), hypothesis blinker==1.7.0 # via flask -certifi==2023.11.17 # via requests +certifi==2024.2.2 # via requests charset-normalizer==3.3.2 # via requests click==8.1.7 # via black, flask, hypothesis -coverage[toml]==7.3.2 # via hypofuzz (setup.py), pytest-cov -dash==2.14.2 # via hypofuzz (setup.py) +coverage[toml]==7.4.1 # via hypofuzz (setup.py), pytest-cov +dash==2.15.0 # via hypofuzz (setup.py) dash-core-components==2.0.0 # via dash dash-html-components==2.0.0 # via dash dash-table==5.0.0 # via dash exceptiongroup==1.2.0 # via hypothesis, pytest execnet==2.0.2 # via pytest-xdist -flask==3.0.0 # via dash -hypothesis[cli]==6.91.1 # via hypofuzz (setup.py) +flask==3.0.1 # via dash +hypothesis[cli]==6.97.4 # via hypofuzz (setup.py) idna==3.6 # via requests -importlib-metadata==7.0.0 # via dash +importlib-metadata==7.0.1 # via dash iniconfig==2.0.0 # via pytest itsdangerous==2.1.2 # via flask -jinja2==3.1.2 # via flask +jinja2==3.1.3 # via flask libcst==1.1.0 # via hypofuzz (setup.py) markdown-it-py==3.0.0 # via rich -markupsafe==2.1.3 # via jinja2, werkzeug +markupsafe==2.1.4 # via jinja2, werkzeug mdurl==0.1.2 # via markdown-it-py mypy-extensions==1.0.0 # via black, typing-inspect -nest-asyncio==1.5.8 # via dash -numpy==1.26.2 # via pandas +nest-asyncio==1.6.0 # via dash +numpy==1.26.3 # via pandas packaging==23.2 # via black, plotly, pytest -pandas==2.1.4 # via hypofuzz (setup.py) -pathspec==0.11.2 # via black -platformdirs==4.1.0 # via black +pandas==2.2.0 # via hypofuzz (setup.py) +pathspec==0.12.1 # via black +platformdirs==4.2.0 # via black plotly==5.18.0 # via dash -pluggy==1.3.0 # via pytest -psutil==5.9.6 # via hypofuzz (setup.py) +pluggy==1.4.0 # via pytest +psutil==5.9.8 # via hypofuzz (setup.py) pygments==2.17.2 # via rich -pytest==7.4.3 # via -r deps/test.in, hypofuzz (setup.py), pytest-cov, pytest-xdist +pytest==8.0.0 # via -r deps/test.in, hypofuzz (setup.py), pytest-cov, pytest-xdist pytest-cov==4.1.0 # via -r deps/test.in pytest-xdist==3.5.0 # via -r deps/test.in python-dateutil==2.8.2 # via pandas -pytz==2023.3.post1 # via pandas +pytz==2024.1 # via pandas pyyaml==6.0.1 # via libcst requests==2.31.0 # via dash, hypofuzz (setup.py) retrying==1.3.4 # via dash @@ -53,10 +52,10 @@ six==1.16.0 # via python-dateutil, retrying sortedcontainers==2.4.0 # via hypothesis tenacity==8.2.3 # via plotly tomli==2.0.1 # via black, coverage, pytest -typing-extensions==4.8.0 # via black, dash, libcst, typing-inspect +typing-extensions==4.9.0 # via black, dash, libcst, typing-inspect typing-inspect==0.9.0 # via libcst -tzdata==2023.3 # via pandas -urllib3==2.1.0 # via requests +tzdata==2023.4 # via pandas +urllib3==2.2.0 # via requests werkzeug==3.0.1 # via dash, flask zipp==3.17.0 # via importlib-metadata diff --git a/docs-src/changelog.md b/docs-src/changelog.md index 8d83b87..6a26e77 100644 --- a/docs-src/changelog.md +++ b/docs-src/changelog.md @@ -2,6 +2,11 @@ HypoFuzz uses [calendar-based versioning](https://calver.org/), with a `YY-MM-patch` format. +## 24.02.1 +Now requires [Hypothesis 6.93.2](https://hypothesis.readthedocs.io/en/latest/changes.html#v6-93-2) +or later, fixing compatibility with some unstable internals that HypoFuzz hooks into (yes, again). +Also deduplicates the displayed covering examples in the dashboard, when their reprs are identical. + ## 23.12.1 Now requires [Hypothesis 6.91](https://hypothesis.readthedocs.io/en/latest/changes.html#v6-91-0) or later, fixing compatibility with some unstable internals that HypoFuzz hooks into. diff --git a/setup.py b/setup.py index bfd55c7..67b7cd7 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ def local_file(name: str) -> str: "black >= 23.3.0", "coverage >= 5.2.1", "dash >= 2.0.0", - "hypothesis[cli] >= 6.91.0", + "hypothesis[cli] >= 6.93.2", "libcst >= 1.0.0", "pandas >= 1.0.0", "psutil >= 3.0.0", diff --git a/src/hypofuzz/__init__.py b/src/hypofuzz/__init__.py index 40acadd..cfde731 100644 --- a/src/hypofuzz/__init__.py +++ b/src/hypofuzz/__init__.py @@ -1,4 +1,4 @@ """Adaptive fuzzing for property-based tests using Hypothesis.""" -__version__ = "23.12.1" +__version__ = "24.2.1" __all__: list = [] diff --git a/src/hypofuzz/dashboard.py b/src/hypofuzz/dashboard.py index d54b405..ad73d2d 100644 --- a/src/hypofuzz/dashboard.py +++ b/src/hypofuzz/dashboard.py @@ -1,4 +1,5 @@ """Live web dashboard for a fuzzing run.""" + import atexit import datetime import os @@ -142,6 +143,15 @@ def display_page(pathname: str) -> html.Div: href=f"https://app.pytrace.com/?open={url}", ) add.append(html.Pre(children=[link])) + + _seen_cov_examples = set() + covering_examples = [] + for row in last_update.get("seed_pool", []): + example = try_format(row[1]), row[2] + if example not in _seen_cov_examples: + _seen_cov_examples.add(example) + covering_examples.append(example) + return html.Div( children=[ dcc.Link("Back to main dashboard", href="/"), @@ -169,10 +179,7 @@ def display_page(pathname: str) -> html.Div: "not covered by any previous, more-minimal, example." ] ), - ] - + [ - html.Pre([html.Code([try_format(row[1]), row[2], "\n"])]) - for row in last_update.get("seed_pool", []) + *(html.Pre([html.Code([*ex, "\n"])]) for ex in covering_examples), ] ) diff --git a/src/hypofuzz/hy.py b/src/hypofuzz/hy.py index ad70501..bac6a36 100644 --- a/src/hypofuzz/hy.py +++ b/src/hypofuzz/hy.py @@ -274,10 +274,9 @@ def _run_test_on( # Generate all arguments to the test function. args = self.__stuff.args kwargs = dict(self.__stuff.kwargs) - a, kw, argslices = context.prep_args_kwargs_from_strategies( - (), self.__stuff.given_kwargs + kw, argslices = context.prep_args_kwargs_from_strategies( + self.__stuff.given_kwargs ) - assert not a, "strategies all moved to kwargs by now" kwargs.update(kw) printer = RepresentationPrinter(context=context) @@ -373,9 +372,11 @@ def _json_description(self) -> Report: "loaded_from_db": len(self.pool._loaded_from_database), "status_counts": self.status_counts, "seed_pool": self.pool.json_report, - "note": "replaying saved examples" - if self._replay_buffer - else ("shrinking known examples" if self.pool._in_distill_phase else ""), + "note": ( + "replaying saved examples" + if self._replay_buffer + else ("shrinking known examples" if self.pool._in_distill_phase else "") + ), } if self.pool.interesting_examples: report["note"] = ( diff --git a/src/hypofuzz/interface.py b/src/hypofuzz/interface.py index 2e69efb..5392d65 100644 --- a/src/hypofuzz/interface.py +++ b/src/hypofuzz/interface.py @@ -1,4 +1,5 @@ """CLI and Python API for the fuzzer.""" + import io import sys from contextlib import redirect_stdout, suppress diff --git a/tests/test_fuzz_process.py b/tests/test_fuzz_process.py index 2f10a95..aa40541 100644 --- a/tests/test_fuzz_process.py +++ b/tests/test_fuzz_process.py @@ -1,44 +1,45 @@ -"""Tests for the hypofuzz library.""" -from hypothesis import given, strategies as st -from hypothesis.internal.conjecture.data import Status - -from hypofuzz.hy import FuzzProcess - - -@given(st.integers()) -def pbt(x): - pass - - -def test_fuzz_one_process(): - # This is a terrible test but better than nothing - fp = FuzzProcess.from_hypothesis_test(pbt) - for _ in range(100): - fp.run_one() - - # We expect that this test will always pass; check that. - assert fp.status_counts[Status.INTERESTING.name] == 0 - - -class CustomError(Exception): - pass - - -@given(st.integers(0, 10), st.integers(0, 10)) -def failing_pbt(x, y): - if x: - raise CustomError(f"x={x}") - - -def test_fuzz_one_process_explain_mode(): - # This is a terrible test but better than nothing - fp = FuzzProcess.from_hypothesis_test(failing_pbt) - for _ in range(10): - fp.run_one() - - # Check that we got the expected failure, including message + explain-mode output. - assert fp.status_counts[Status.INTERESTING.name] >= 1 - (call_repr, _, _, tb_repr), *rest = fp._json_description["failures"] - assert not rest # expected only one failure - assert tb_repr.endswith("test_fuzz_process.CustomError: x=1\n") - assert call_repr == "failing_pbt(\n x=1,\n y=0,\n)" +"""Tests for the hypofuzz library.""" + +from hypothesis import given, strategies as st +from hypothesis.internal.conjecture.data import Status + +from hypofuzz.hy import FuzzProcess + + +@given(st.integers()) +def pbt(x): + pass + + +def test_fuzz_one_process(): + # This is a terrible test but better than nothing + fp = FuzzProcess.from_hypothesis_test(pbt) + for _ in range(100): + fp.run_one() + + # We expect that this test will always pass; check that. + assert fp.status_counts[Status.INTERESTING.name] == 0 + + +class CustomError(Exception): + pass + + +@given(st.integers(0, 10), st.integers(0, 10)) +def failing_pbt(x, y): + if x: + raise CustomError(f"x={x}") + + +def test_fuzz_one_process_explain_mode(): + # This is a terrible test but better than nothing + fp = FuzzProcess.from_hypothesis_test(failing_pbt) + for _ in range(10): + fp.run_one() + + # Check that we got the expected failure, including message + explain-mode output. + assert fp.status_counts[Status.INTERESTING.name] >= 1 + (call_repr, _, _, tb_repr), *rest = fp._json_description["failures"] + assert not rest # expected only one failure + assert tb_repr.endswith("test_fuzz_process.CustomError: x=1\n") + assert call_repr == "failing_pbt(\n x=1,\n y=0,\n)"