diff --git a/pytest-fuzz.ini b/pytest-fuzz.ini new file mode 100644 index 0000000000..67f643feb3 --- /dev/null +++ b/pytest-fuzz.ini @@ -0,0 +1,19 @@ +[pytest] +console_output_style = count +minversion = 7.0 +python_files = *.py +testpaths = tests/ +markers = + slow + fuzzable +addopts = + -p pytest_plugins.filler.fuzzer + -p pytest_plugins.filler.pre_alloc + -p pytest_plugins.filler.filler + -p pytest_plugins.forks.forks + -p pytest_plugins.spec_version_checker.spec_version_checker + -p pytest_plugins.help.help + -m "not eip_version_check" + --tb short + --dist loadscope + --ignore tests/cancun/eip4844_blobs/point_evaluation_vectors/ diff --git a/pytest.ini b/pytest.ini index f478e20bf5..9e3b24732f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,6 +5,7 @@ python_files = *.py testpaths = tests/ markers = slow + fuzzable addopts = -p pytest_plugins.filler.pre_alloc -p pytest_plugins.filler.filler diff --git a/setup.cfg b/setup.cfg index 169265bb46..e88357d254 100644 --- a/setup.cfg +++ b/setup.cfg @@ -78,6 +78,7 @@ pytest_plugins = [options.entry_points] console_scripts = fill = cli.pytest_commands:fill + fuzz = cli.pytest_commands:fuzz tf = cli.pytest_commands:tf checkfixtures = cli.check_fixtures:check_fixtures consume = cli.pytest_commands:consume diff --git a/src/cli/pytest_commands.py b/src/cli/pytest_commands.py index bb688a9df1..229f8a93f5 100644 --- a/src/cli/pytest_commands.py +++ b/src/cli/pytest_commands.py @@ -142,6 +142,23 @@ def fill( sys.exit(result) +@click.command(context_settings=dict(ignore_unknown_options=True)) +@common_click_options +def fuzz( + pytest_args: List[str], + help_flag: bool, + pytest_help_flag: bool, +) -> None: + """ + Entry point for the fuzz command. + """ + args = handle_help_flags(pytest_args, help_flag, pytest_help_flag) + args += ["-c", "pytest-fuzz.ini"] + args = handle_stdout_flags(args) + result = pytest.main(args) + sys.exit(result) + + def get_hive_flags_from_env(): """ Read simulator flags from environment variables and convert them, as best as diff --git a/src/ethereum_test_fuzzing/__init__.py b/src/ethereum_test_fuzzing/__init__.py new file mode 100644 index 0000000000..330a9e776d --- /dev/null +++ b/src/ethereum_test_fuzzing/__init__.py @@ -0,0 +1,8 @@ +""" +Module containing tools for fuzzing of cross-client Ethereum execution layer +tests. +""" + +from .helpers import type_fuzzer_generator + +__all__ = ["type_fuzzer_generator"] diff --git a/src/ethereum_test_fuzzing/fuzzers.py b/src/ethereum_test_fuzzing/fuzzers.py new file mode 100644 index 0000000000..583f72d0ff --- /dev/null +++ b/src/ethereum_test_fuzzing/fuzzers.py @@ -0,0 +1,3 @@ +""" +Type fuzzers. +""" diff --git a/src/ethereum_test_fuzzing/helpers.py b/src/ethereum_test_fuzzing/helpers.py new file mode 100644 index 0000000000..4e8e9f0479 --- /dev/null +++ b/src/ethereum_test_fuzzing/helpers.py @@ -0,0 +1,20 @@ +""" +Main module for ethereum test fuzzing tools. +""" + +import random +from typing import Annotated, Any, Callable, get_args, get_origin + + +def type_fuzzer_generator(annotation: Any) -> Callable | None: + """ + Returns a callable that generates a single fuzzed value for the given annotation. + """ + if get_origin(annotation) is Annotated: + # Parameter type should be annotated with a fuzz generator + args = get_args(annotation)[1:] + # TODO: Create a fuzz generator type and match each argument until found + return args[0] + if annotation == int: + return lambda: random.randint(0, 2**256) + return None diff --git a/src/pytest_plugins/filler/fuzzer.py b/src/pytest_plugins/filler/fuzzer.py new file mode 100644 index 0000000000..8039b6d8b6 --- /dev/null +++ b/src/pytest_plugins/filler/fuzzer.py @@ -0,0 +1,93 @@ +""" +Pytest plug-in that allows fuzzed parametrization. +""" + +import inspect +from copy import copy +from typing import Any, List + +import pytest + +from ethereum_test_fuzzing import type_fuzzer_generator + + +def pytest_addoption(parser: pytest.Parser): + """ + Adds command-line options to pytest. + """ + fuzzer_group = parser.getgroup("fuzzer", "Arguments defining fuzzer behavior") + fuzzer_group.addoption( + "--iterations", + action="store", + dest="iterations", + type=int, + default=1, + help=("Number of iterations per test to generate"), + ) + + +@pytest.hookimpl(trylast=True) +def pytest_report_header(config: pytest.Config): + """Add lines to pytest's console output header""" + if config.option.collectonly: + return + iterations = config.getoption("iterations") + return [f"{iterations}"] + + +def pytest_generate_tests(metafunc: pytest.Metafunc): + """ + Pytest hook used to dynamically generate fuzzed test cases. + """ + markers = list(metafunc.definition.iter_markers("fuzzable")) + assert len(markers) <= 1, "Test should contain only one 'fuzzable' marker" + + if len(markers) == 0: + pytest.skip("non-fuzzable") + + fuzz_arguments = markers[0].args + + pytest_params: List[Any] = [] + + # Remove existing parametrize markers that conflict with fuzzing + i = 0 + while i < len(metafunc.definition.own_markers): + own_marker = metafunc.definition.own_markers[i] + if own_marker.name == "parametrize": + if type(own_marker.args[0]) is str: + parametrize_args = own_marker.args[0].split(",") + else: + parametrize_args = copy(own_marker.args[0]) + if any(fuzz_argument in parametrize_args for fuzz_argument in fuzz_arguments): + for fuzz_argument in fuzz_arguments: + if fuzz_argument in parametrize_args: + parametrize_args.remove(fuzz_argument) + assert ( + len(parametrize_args) == 0 + ), "`pytest.mark.parametrize` mixes fuzzable and non-fuzzable arguments" + del metafunc.definition.own_markers[i] + i += 1 + + iterations = metafunc.config.getoption("iterations") + + fuzz_generators = [] + parameters = inspect.signature(metafunc.function).parameters + for fuzz_argument in fuzz_arguments: + assert ( + fuzz_argument in parameters + ), f"fuzz argument not found in function signature {fuzz_argument}" + annotation = parameters[fuzz_argument].annotation + fuzzer = type_fuzzer_generator(annotation) + if fuzzer is None: + raise ValueError( + f"no fuzzer available for type {annotation} for parameter {fuzz_argument}" + ) + fuzz_generators.append(fuzzer) + + for _ in range(iterations): + iteration_args = [] + for fuzz_generator in fuzz_generators: + iteration_args.append(fuzz_generator()) + pytest_params.append(pytest.param(*iteration_args)) + + metafunc.parametrize(f"{','.join(fuzz_arguments)}", pytest_params, scope="function") diff --git a/tests/frontier/opcodes/test_fuzzing.py b/tests/frontier/opcodes/test_fuzzing.py new file mode 100644 index 0000000000..388d095ac0 --- /dev/null +++ b/tests/frontier/opcodes/test_fuzzing.py @@ -0,0 +1,55 @@ +""" +Test fuzzer example. +""" + +import random +from typing import Annotated + +import pytest + +from ethereum_test_forks import Fork, Frontier, Homestead +from ethereum_test_tools import Account, Alloc, Environment +from ethereum_test_tools import Opcodes as Op +from ethereum_test_tools import StateTestFiller, Transaction + + +@pytest.mark.parametrize( + "key,value", + [(1, 1)], +) +@pytest.mark.fuzzable("key", "value") +def test_sstore( + state_test: StateTestFiller, + fork: Fork, + pre: Alloc, + key: Annotated[int, lambda: random.randint(0, 2**64)], + value: int, +): + """ + Simple SSTORE test that can be fuzzed to generate any number of tests. + """ + env = Environment() + sender = pre.fund_eoa() + post = {} + + account_code = Op.SSTORE(key, value) + Op.STOP + + account = pre.deploy_contract(account_code) + + tx = Transaction( + to=account, + gas_limit=500000, + data="", + sender=sender, + protected=False if fork in [Frontier, Homestead] else True, + ) + + post = { + account: Account( + storage={ + key: value, + } + ) + } + + state_test(env=env, pre=pre, post=post, tx=tx) diff --git a/whitelist.txt b/whitelist.txt index bf2ec2d516..d249c668c0 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -146,6 +146,9 @@ formatter fromhex frozenbidict func +fuzz +fuzzer +fuzzable fp fp2 g1