feat(all): Fuzzing

pytest-fuzz.ini

@@ -0,0 +1,19 @@
+[pytest]
+console_output_style = count
+minversion = 7.0
+python_files = *.py
+testpaths = tests/
+markers =
+    slow
+    fuzzable
+addopts = 
+    -p pytest_plugins.filler.fuzzer
+    -p pytest_plugins.filler.pre_alloc
+    -p pytest_plugins.filler.filler
+    -p pytest_plugins.forks.forks
+    -p pytest_plugins.spec_version_checker.spec_version_checker
+    -p pytest_plugins.help.help
+    -m "not eip_version_check"
+    --tb short
+    --dist loadscope
+    --ignore tests/cancun/eip4844_blobs/point_evaluation_vectors/

pytest.ini

@@ -5,6 +5,7 @@ python_files = *.py
 testpaths = tests/
 markers =
     slow
+    fuzzable
 addopts = 
     -p pytest_plugins.filler.pre_alloc
     -p pytest_plugins.filler.filler

setup.cfg

@@ -78,6 +78,7 @@ pytest_plugins =
 [options.entry_points]
 console_scripts =
     fill = cli.pytest_commands:fill
+    fuzz = cli.pytest_commands:fuzz
     tf = cli.pytest_commands:tf
     checkfixtures = cli.check_fixtures:check_fixtures
     consume = cli.pytest_commands:consume

src/cli/pytest_commands.py

@@ -142,6 +142,23 @@ def fill(
     sys.exit(result)
 
 
+@click.command(context_settings=dict(ignore_unknown_options=True))
+@common_click_options
+def fuzz(
+    pytest_args: List[str],
+    help_flag: bool,
+    pytest_help_flag: bool,
+) -> None:
+    """
+    Entry point for the fuzz command.
+    """
+    args = handle_help_flags(pytest_args, help_flag, pytest_help_flag)
+    args += ["-c", "pytest-fuzz.ini"]
+    args = handle_stdout_flags(args)
+    result = pytest.main(args)
+    sys.exit(result)
+
+
 def get_hive_flags_from_env():
     """
     Read simulator flags from environment variables and convert them, as best as

src/ethereum_test_fuzzing/__init__.py

@@ -0,0 +1,8 @@
+"""
+Module containing tools for fuzzing of cross-client Ethereum execution layer
+tests.
+"""
+
+from .helpers import type_fuzzer_generator
+
+__all__ = ["type_fuzzer_generator"]

src/ethereum_test_fuzzing/fuzzers.py

@@ -0,0 +1,3 @@
+"""
+Type fuzzers.
+"""

src/ethereum_test_fuzzing/helpers.py

@@ -0,0 +1,20 @@
+"""
+Main module for ethereum test fuzzing tools.
+"""
+
+import random
+from typing import Annotated, Any, Callable, get_args, get_origin
+
+
+def type_fuzzer_generator(annotation: Any) -> Callable | None:
+    """
+    Returns a callable that generates a single fuzzed value for the given annotation.
+    """
+    if get_origin(annotation) is Annotated:
+        # Parameter type should be annotated with a fuzz generator
+        args = get_args(annotation)[1:]
+        # TODO: Create a fuzz generator type and match each argument until found
+        return args[0]
+    if annotation == int:
+        return lambda: random.randint(0, 2**256)
+    return None

src/pytest_plugins/filler/fuzzer.py

@@ -0,0 +1,93 @@
+"""
+Pytest plug-in that allows fuzzed parametrization.
+"""
+
+import inspect
+from copy import copy
+from typing import Any, List
+
+import pytest
+
+from ethereum_test_fuzzing import type_fuzzer_generator
+
+
+def pytest_addoption(parser: pytest.Parser):
+    """
+    Adds command-line options to pytest.
+    """
+    fuzzer_group = parser.getgroup("fuzzer", "Arguments defining fuzzer behavior")
+    fuzzer_group.addoption(
+        "--iterations",
+        action="store",
+        dest="iterations",
+        type=int,
+        default=1,
+        help=("Number of iterations per test to generate"),
+    )
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_report_header(config: pytest.Config):
+    """Add lines to pytest's console output header"""
+    if config.option.collectonly:
+        return
+    iterations = config.getoption("iterations")
+    return [f"{iterations}"]
+
+
+def pytest_generate_tests(metafunc: pytest.Metafunc):
+    """
+    Pytest hook used to dynamically generate fuzzed test cases.
+    """
+    markers = list(metafunc.definition.iter_markers("fuzzable"))
+    assert len(markers) <= 1, "Test should contain only one 'fuzzable' marker"
+
+    if len(markers) == 0:
+        pytest.skip("non-fuzzable")
+
+    fuzz_arguments = markers[0].args
+
+    pytest_params: List[Any] = []
+
+    # Remove existing parametrize markers that conflict with fuzzing
+    i = 0
+    while i < len(metafunc.definition.own_markers):
+        own_marker = metafunc.definition.own_markers[i]
+        if own_marker.name == "parametrize":
+            if type(own_marker.args[0]) is str:
+                parametrize_args = own_marker.args[0].split(",")
+            else:
+                parametrize_args = copy(own_marker.args[0])
+            if any(fuzz_argument in parametrize_args for fuzz_argument in fuzz_arguments):
+                for fuzz_argument in fuzz_arguments:
+                    if fuzz_argument in parametrize_args:
+                        parametrize_args.remove(fuzz_argument)
+                assert (
+                    len(parametrize_args) == 0
+                ), "`pytest.mark.parametrize` mixes fuzzable and non-fuzzable arguments"
+                del metafunc.definition.own_markers[i]
+        i += 1
+
+    iterations = metafunc.config.getoption("iterations")
+
+    fuzz_generators = []
+    parameters = inspect.signature(metafunc.function).parameters
+    for fuzz_argument in fuzz_arguments:
+        assert (
+            fuzz_argument in parameters
+        ), f"fuzz argument not found in function signature {fuzz_argument}"
+        annotation = parameters[fuzz_argument].annotation
+        fuzzer = type_fuzzer_generator(annotation)
+        if fuzzer is None:
+            raise ValueError(
+                f"no fuzzer available for type {annotation} for parameter {fuzz_argument}"
+            )
+        fuzz_generators.append(fuzzer)
+
+    for _ in range(iterations):
+        iteration_args = []
+        for fuzz_generator in fuzz_generators:
+            iteration_args.append(fuzz_generator())
+        pytest_params.append(pytest.param(*iteration_args))
+
+    metafunc.parametrize(f"{','.join(fuzz_arguments)}", pytest_params, scope="function")

tests/frontier/opcodes/test_fuzzing.py

@@ -0,0 +1,55 @@
+"""
+Test fuzzer example.
+"""
+
+import random
+from typing import Annotated
+
+import pytest
+
+from ethereum_test_forks import Fork, Frontier, Homestead
+from ethereum_test_tools import Account, Alloc, Environment
+from ethereum_test_tools import Opcodes as Op
+from ethereum_test_tools import StateTestFiller, Transaction
+
+
+@pytest.mark.parametrize(
+    "key,value",
+    [(1, 1)],
+)
+@pytest.mark.fuzzable("key", "value")
+def test_sstore(
+    state_test: StateTestFiller,
+    fork: Fork,
+    pre: Alloc,
+    key: Annotated[int, lambda: random.randint(0, 2**64)],
+    value: int,
+):
+    """
+    Simple SSTORE test that can be fuzzed to generate any number of tests.
+    """
+    env = Environment()
+    sender = pre.fund_eoa()
+    post = {}
+
+    account_code = Op.SSTORE(key, value) + Op.STOP
+
+    account = pre.deploy_contract(account_code)
+
+    tx = Transaction(
+        to=account,
+        gas_limit=500000,
+        data="",
+        sender=sender,
+        protected=False if fork in [Frontier, Homestead] else True,
+    )
+
+    post = {
+        account: Account(
+            storage={
+                key: value,
+            }
+        )
+    }
+
+    state_test(env=env, pre=pre, post=post, tx=tx)

whitelist.txt

@@ -146,6 +146,9 @@ formatter
 fromhex
 frozenbidict
 func
+fuzz
+fuzzer
+fuzzable
 fp
 fp2
 g1