From 41ea99841ffbe82e1277e49eb1644f04368cfca5 Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 May 2024 10:33:23 +0200 Subject: [PATCH 01/12] implement and test SCFG->AST conversion This implements the creation of Python source code from a potentially restructured SCFG. The main entry-point is: ``` from numba_rvsdg import SCFG2AST ``` And the round-trip test for the entry-points shows how to use the API: ``` class TestEntryPoints(TestCase): def test_rondtrip(self): def function() -> int: x = 0 for i in range(2): x += i return x, i scfg = AST2SCFG(function) scfg.restructure() ast_ = SCFG2AST(function, scfg) exec_locals = {} exec(ast.unparse(ast_), {}, exec_locals) transformed = exec_locals["transformed_function"] assert function() == transformed() ``` Special attention was paid to the testing of the transform. For all the previously defined testing functions to test the AST -> SCFG direction, the tests were augmented to also test the SCFG -> AST direction. To test a transformed function, we assert behavioral equivalence by running the original and the transformed through a set of given arguments and ensure they always produce the same result. Additionally we ensure that all lines of the test function are covered using a custom `sys.monitoring` setup. (As a result the package now needs at least 3.12 for testing). This ensures that that the set of arguments covers the original function and also that the transformation back to Python doesn't create any dead code. Special thanks to @stuartarchibald for the stater patch for the custom `sys.monitoring` based tracer. Overall the iteration over the SCFG is still somewhat unprincipled, however, the tests and overall coverage do seem to suggest the approach works. Now that we can transform Python programs and have solid tests too, more thought ought to be invested into storing the SCFG in a non-recursive data-structure and developing a more elegant API to traverse the graph and it's regions depending on the use-case. Typing and annotations are a nbit haphazard, there are multiple issues in the existing classes so some parts of this code just choose to use `Any` and `# type: ignore` pragmas. --- makefile | 4 +- numba_rvsdg/__init__.py | 5 +- .../core/datastructures/ast_transforms.py | 272 +++++++++- numba_rvsdg/tests/test_ast_transforms.py | 510 ++++++++++++------ 4 files changed, 594 insertions(+), 197 deletions(-) diff --git a/makefile b/makefile index 94530c3..b36d0ed 100644 --- a/makefile +++ b/makefile @@ -4,7 +4,9 @@ all: build: python -m pip install -vv -e . test: - coverage run -m pytest --pyargs numba_rvsdg + # Activate using the sys.monitoring implementation of coverage. + # Needs at least coverage veraion 7.4.0 to work. + COVERAGE_CORE=sysmon coverage run -m pytest --pyargs numba_rvsdg coverage report lint: pre-commit run --verbose --all-files diff --git a/numba_rvsdg/__init__.py b/numba_rvsdg/__init__.py index 3c59890..d29d637 100644 --- a/numba_rvsdg/__init__.py +++ b/numba_rvsdg/__init__.py @@ -1 +1,4 @@ -from numba_rvsdg.core.datastructures.ast_transforms import AST2SCFG # noqa +from numba_rvsdg.core.datastructures.ast_transforms import ( # noqa + AST2SCFG, + SCFG2AST, +) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index 30a93ad..17a652b 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -3,9 +3,40 @@ from typing import Callable, Any, MutableMapping import textwrap from dataclasses import dataclass +from collections import defaultdict from numba_rvsdg.core.datastructures.scfg import SCFG -from numba_rvsdg.core.datastructures.basic_block import PythonASTBlock +from numba_rvsdg.core.datastructures.basic_block import ( + PythonASTBlock, + RegionBlock, + SyntheticHead, + SyntheticTail, + SyntheticFill, + SyntheticReturn, + SyntheticAssignment, + SyntheticExitingLatch, + SyntheticExitBranch, +) + + +def unparse_code( + code: str | list[ast.FunctionDef] | Callable[..., Any] +) -> list[type[ast.AST]]: + # Convert source code into AST. + if isinstance(code, str): + tree = ast.parse(code).body + elif callable(code): + tree = ast.parse(textwrap.dedent(inspect.getsource(code))).body + elif ( + isinstance(code, list) + and len(code) > 0 + and all([isinstance(i, ast.AST) for i in code]) + ): + tree = code # type: ignore + else: + msg = "Type: '{type(self.code}}' is not implemented." + raise NotImplementedError(msg) + return tree # type: ignore class WritableASTBlock: @@ -217,7 +248,7 @@ def __init__( ) -> None: self.prune = prune self.code = code - self.tree = AST2SCFGTransformer.unparse_code(code) + self.tree = unparse_code(code) self.block_index: int = 1 # 0 is reserved for genesis block self.blocks = ASTCFG() # Initialize first (genesis) block, assume it's named zero. @@ -225,26 +256,6 @@ def __init__( self.add_block(0) self.loop_stack: list[LoopIndices] = [] - @staticmethod - def unparse_code( - code: str | list[ast.FunctionDef] | Callable[..., Any] - ) -> list[type[ast.AST]]: - # Convert source code into AST. - if isinstance(code, str): - tree = ast.parse(code).body - elif callable(code): - tree = ast.parse(textwrap.dedent(inspect.getsource(code))).body - elif ( - isinstance(code, list) - and len(code) > 0 - and all([isinstance(i, ast.AST) for i in code]) - ): - tree = code # type: ignore - else: - msg = "Type: '{type(self.code}}' is not implemented." - raise NotImplementedError(msg) - return tree # type: ignore - def transform_to_ASTCFG(self) -> ASTCFG: """Generate ASTCFG from Python function.""" self.transform() @@ -329,7 +340,7 @@ def handle_function_def(self, node: ast.FunctionDef) -> None: # end up being an unreachable block if all other paths through the # program already call return. if not isinstance(node.body[-1], ast.Return): - node.body.append(ast.Return(None)) + node.body.append(ast.Return()) self.codegen(node.body) def handle_if(self, node: ast.If) -> None: @@ -655,11 +666,220 @@ def render(self) -> None: self.blocks.to_SCFG().render() +class SCFG2ASTTransformer: + + def transform( + self, original: ast.FunctionDef, scfg: SCFG + ) -> ast.FunctionDef: + body: list[ast.AST] = [] + self.region_stack = [scfg.region] + self.scfg = scfg + for name, block in scfg.concealed_region_view.items(): + if type(block) is RegionBlock and block.kind == "branch": + continue + body.extend(self.codegen(block)) + fdef = ast.FunctionDef( + name="transformed_function", + args=original.args, + body=body, + lineno=0, + decorator_list=original.decorator_list, + returns=original.returns, + ) + return fdef + + def lookup(self, item: Any) -> Any: + subregion_scfg = self.region_stack[-1].subregion + parent_region_block = self.region_stack[-1].parent_region + if item in subregion_scfg: # type: ignore + return subregion_scfg[item] # type: ignore + else: + return self.rlookup(parent_region_block, item) # type: ignore + + def rlookup(self, region_block: RegionBlock, item: Any) -> Any: + if item in region_block.subregion: # type: ignore + return region_block.subregion[item] # type: ignore + elif region_block.parent_region is not None: + return self.rlookup(region_block.parent_region, item) + else: + raise KeyError(f"Item {item} not found in subregion or parent") + + def codegen(self, block: Any) -> list[ast.AST]: + if type(block) is PythonASTBlock: + if len(block.jump_targets) == 2: + if type(block.tree[-1]) in (ast.Name, ast.Compare): + test = block.tree[-1] + else: + test = block.tree[-1].value # type: ignore + body = self.codegen(self.lookup(block.jump_targets[0])) + orelse = self.codegen(self.lookup(block.jump_targets[1])) + if_node = ast.If(test, body, orelse) + return block.tree[:-1] + [if_node] + elif block.fallthrough and type(block.tree[-1]) is ast.Return: + # The value of the ast.Return could be either None or an + # ast.AST type. In the case of None, this refers to a plain + # 'return', which is implicitly 'return None'. So, if it is + # None, we assign the __return_value__ a ast.Constant(None) and + # whatever the ast.AST node is otherwise. + val = block.tree[-1].value + return block.tree[:-1] + [ + ast.Assign( + [ast.Name("__return_value__")], + (ast.Constant(None) if val is None else val), + lineno=0, + ) + ] + elif block.fallthrough or block.is_exiting: + return block.tree + else: + raise NotImplementedError + elif type(block) is RegionBlock: + # We maintain a stack of the current region, in order to allow for + # random node lookup by name. + self.region_stack.append(block) + + # This is a custom view that uses the concealed_region_view and + # additionally filters all branch regions. Essentially, branch + # regions will be visited by calling codegen recursively from + # blocks with multiple jump targets and all other regions must be + # visited linearly. + def codegen_view() -> list[Any]: + return [ + self.codegen(b) + for b in block.subregion.concealed_region_view.values() # type: ignore # noqa + if not (type(b) is RegionBlock and b.kind == "branch") + ] + + # Head, tail and branch regions themselves to use the custom view + # above. + if block.kind == "head": + rval = codegen_view() + elif block.kind == "tail": + rval = codegen_view() + elif block.kind == "branch": + rval = codegen_view() + elif block.kind == "loop": + # A loop region gives rise to a Python while True loop. We + # recursively visit the body. + rval = [ + ast.While( + test=ast.Constant(value=True), + body=codegen_view(), + orelse=[], + ) + ] + else: + raise NotImplementedError + self.region_stack.pop() + return rval + elif type(block) is SyntheticAssignment: + # Synthetic assignments just create Python assignments, one for + # each variable.. + return [ + ast.Assign([ast.Name(t)], ast.Constant(v), lineno=0) + for t, v in block.variable_assignment.items() + ] + elif type(block) is SyntheticTail: + # Synthetic tails do nothing. + pass + elif type(block) is SyntheticFill: + # Synthetic fills must have a pass statement to main syntactical + # correctness of the final program. + return [ast.Pass()] + elif type(block) is SyntheticReturn: + # Synthetic return blocks must re-assigne the return value to a + # special reserved variable. + return [ast.Return(ast.Name("__return_value__"))] + elif type(block) is SyntheticExitingLatch: + # The synthetic exiting latch much create a query on the variable + # it holds and then insert a Python if that will either break or + # continue. This effectively generates the backedge for the looping + # region. + assert len(block.jump_targets) == 1 + assert len(block.backedges) == 1 + compare_value = [ + i + for i, v in block.branch_value_table.items() + if v == block.backedges[0] + ][0] + if_beak_node_test = ast.Compare( + left=ast.Name(block.variable), + ops=[ast.Eq()], + comparators=[ast.Constant(compare_value)], + ) + if_break_node = ast.If( + test=if_beak_node_test, + body=[ast.Continue()], + orelse=[ast.Break()], + ) + return [if_break_node] + elif type(block) in (SyntheticExitBranch, SyntheticHead): + # Both the Synthetic exit branch and the synthetic head contain a + # branching statement with potentially multiple outgoing branches. + # This means we must recursively generate an if-cascade in Python, + # such that all jump targets may be visisted. Looking at the + # resulting AST, it does appear as though the compilation of the + # AST to source code will use `elif` statements. + + # Create a reverse lookup from the branch_value_table + # branch_name --> list of variables that lead there + reverse = defaultdict(list) + for ( + variable_value, + jump_target, + ) in block.branch_value_table.items(): + reverse[jump_target].append(variable_value) + # recursive generation of if-cascade + + def if_cascade(jump_targets: list[str]) -> list[ast.AST]: + if len(jump_targets) == 1: + # base case, final else + return self.codegen(self.lookup(jump_targets.pop())) + else: + # otherwise generate if statement for current jump_target + current = jump_targets.pop() + # compare to all variable values that point to this + # jump_target + if_test = ast.Compare( + left=ast.Name(block.variable), + ops=[ast.In()], + comparators=[ + ast.Tuple( + elts=[ + ast.Constant(i) for i in reverse[current] + ], + ctx=ast.Load(), + ) + ], + ) + # Create the the if-statement itself, using the test. Do + # code-gen for the block that the is being pointed to and + # recurse for the rest of the jump_targets. + if_node = ast.If( + test=if_test, + body=self.codegen(self.lookup(current)), + orelse=if_cascade(jump_targets), + ) + return [if_node] + + # Send in a copy of the jump_targets as this list will be mutated. + return if_cascade(list(block.jump_targets[::-1])) + else: + raise NotImplementedError + return [] + + def AST2SCFG(code: str | list[ast.FunctionDef] | Callable[..., Any]) -> SCFG: """Transform Python function into an SCFG.""" return AST2SCFGTransformer(code).transform_to_SCFG() -def SCFG2AST(scfg: SCFG) -> ast.FunctionDef: # type: ignore - """Transform SCFG with PythonASTBlocks into an AST FunctionDef.""" - # TODO +def SCFG2AST( + code: str | list[ast.FunctionDef] | Callable[..., Any], scfg: SCFG +) -> ast.FunctionDef: + """Transform SCFG with PythonASTBlocks into an AST FunctionDef defined in + code.""" + original_ast = unparse_code(code)[0] + return SCFG2ASTTransformer().transform( + original=original_ast, scfg=scfg # type: ignore + ) diff --git a/numba_rvsdg/tests/test_ast_transforms.py b/numba_rvsdg/tests/test_ast_transforms.py index 2377eb0..3337926 100644 --- a/numba_rvsdg/tests/test_ast_transforms.py +++ b/numba_rvsdg/tests/test_ast_transforms.py @@ -3,8 +3,26 @@ import textwrap from typing import Callable, Any from unittest import main, TestCase +from sys import monitoring as sm -from numba_rvsdg.core.datastructures.ast_transforms import AST2SCFGTransformer +from numba_rvsdg.core.datastructures.ast_transforms import ( + unparse_code, + AST2SCFGTransformer, + SCFG2ASTTransformer, + AST2SCFG, + SCFG2AST, +) + +sm.use_tool_id(sm.PROFILER_ID, "custom_tracer") + + +class LineTraceCallback: + + def __init__(self): + self.lines = set() + + def __call__(self, code, line): + self.lines.add(line) class TestAST2SCFGTransformer(TestCase): @@ -15,14 +33,91 @@ def compare( expected: dict[str, dict[str, Any]], unreachable: set[int] = set(), empty: set[int] = set(), + arguments: list[any] = [], ): - transformer = AST2SCFGTransformer(function) - astcfg = transformer.transform_to_ASTCFG() + # Execute function with first argument, if given. Ensure function is + # sane and make sure it's picked up by coverage. + if type(function) is Callable: + if arguments: + function(*arguments[0]) + else: + function() + # First, test against the expected CFG... + ast2scfg_transformer = AST2SCFGTransformer(function) + astcfg = ast2scfg_transformer.transform_to_ASTCFG() self.assertEqual(expected, astcfg.to_dict()) self.assertEqual(unreachable, {i.name for i in astcfg.unreachable}) self.assertEqual(empty, {i.name for i in astcfg.empty}) + # Then restructure, synthesize python and run original and transformed + # on the same arguments and assert they are the same. + scfg = astcfg.to_SCFG() + scfg.restructure() + scfg2ast = SCFG2ASTTransformer() + original_ast = unparse_code(function)[0] + transformed_ast = scfg2ast.transform(original=original_ast, scfg=scfg) + + # use exec to obtin the function and the transformed_function + original_exec_locals = {} + exec(ast.unparse(original_ast), {}, original_exec_locals) + temporary_function = original_exec_locals["function"] + temporary_exec_locals = {} + exec(ast.unparse(transformed_ast), {}, temporary_exec_locals) + temporary_transformed_function = temporary_exec_locals[ + "transformed_function" + ] + + # Setup the profiler for both funstions and initialize the callbacks + sm.set_local_events( + sm.PROFILER_ID, temporary_function.__code__, sm.events.LINE + ) + sm.set_local_events( + sm.PROFILER_ID, + temporary_transformed_function.__code__, + sm.events.LINE, + ) + original_callback = LineTraceCallback() + transformed_callback = LineTraceCallback() + + # Register the callbacks one at a time and collect results. + sm.register_callback(sm.PROFILER_ID, sm.events.LINE, original_callback) + if arguments: + original_results = [temporary_function(*a) for a in arguments] + else: + original_results = [temporary_function()] + + # Only one callback can be registered at a time. + sm.register_callback( + sm.PROFILER_ID, sm.events.LINE, transformed_callback + ) + if arguments: + transformed_results = [ + temporary_transformed_function(*a) for a in arguments + ] + else: + transformed_results = [temporary_transformed_function()] + + # Check call results + assert original_results == transformed_results + + # Check line trace of original + original_source = ast.unparse(original_ast).splitlines() + assert [ + i + 1 + for i, l in enumerate(original_source) + if not l.startswith("def") and "else:" not in l + ] == sorted(original_callback.lines) + + # Check line trace of transformed + transformed_source = ast.unparse(transformed_ast).splitlines() + assert [ + i + 1 + for i, l in enumerate(transformed_source) + if not l.startswith("def") and "else:" not in l + ] == sorted(transformed_callback.lines) + def setUp(self): + # Enable pytest verbose output. self.maxDiff = None def test_solo_return(self): @@ -116,13 +211,13 @@ def function() -> int: def test_if_return(self): def function(x: int) -> int: - if x < 10: + if x < 1: return 1 return 2 expected = { "0": { - "instructions": ["x < 10"], + "instructions": ["x < 1"], "jump_targets": ["1", "3"], "name": "0", }, @@ -137,18 +232,18 @@ def function(x: int) -> int: "name": "3", }, } - self.compare(function, expected, empty={"2"}) + self.compare(function, expected, empty={"2"}, arguments=[(0,), (1,)]) def test_if_else_return(self): def function(x: int) -> int: - if x < 10: + if x < 1: return 1 else: return 2 expected = { "0": { - "instructions": ["x < 10"], + "instructions": ["x < 1"], "jump_targets": ["1", "2"], "name": "0", }, @@ -163,174 +258,199 @@ def function(x: int) -> int: "name": "2", }, } - self.compare(function, expected, unreachable={"3"}) + self.compare( + function, expected, unreachable={"3"}, arguments=[(0,), (1,)] + ) def test_if_else_assign(self): def function(x: int) -> int: - if x < 10: - z = 1 + if x < 1: + y = 1 else: - z = 2 - return z + y = 2 + return y expected = { "0": { - "instructions": ["x < 10"], + "instructions": ["x < 1"], "jump_targets": ["1", "2"], "name": "0", }, "1": { - "instructions": ["z = 1"], + "instructions": ["y = 1"], "jump_targets": ["3"], "name": "1", }, "2": { - "instructions": ["z = 2"], + "instructions": ["y = 2"], "jump_targets": ["3"], "name": "2", }, "3": { - "instructions": ["return z"], + "instructions": ["return y"], "jump_targets": [], "name": "3", }, } - self.compare(function, expected) + self.compare(function, expected, arguments=[(0,), (1,)]) def test_nested_if(self): def function(x: int, y: int) -> int: - if x < 10: - if y < 5: - y = 1 + if x < 1: + if y < 1: + z = 1 else: - y = 2 + z = 2 else: - if y < 15: - y = 3 + if y < 2: + z = 3 else: - y = 4 - return y + z = 4 + return z expected = { "0": { - "instructions": ["x < 10"], + "instructions": ["x < 1"], "jump_targets": ["1", "2"], "name": "0", }, "1": { - "instructions": ["y < 5"], + "instructions": ["y < 1"], "jump_targets": ["4", "5"], "name": "1", }, "2": { - "instructions": ["y < 15"], + "instructions": ["y < 2"], "jump_targets": ["7", "8"], "name": "2", }, "3": { - "instructions": ["return y"], + "instructions": ["return z"], "jump_targets": [], "name": "3", }, "4": { - "instructions": ["y = 1"], + "instructions": ["z = 1"], "jump_targets": ["3"], "name": "4", }, "5": { - "instructions": ["y = 2"], + "instructions": ["z = 2"], "jump_targets": ["3"], "name": "5", }, "7": { - "instructions": ["y = 3"], + "instructions": ["z = 3"], "jump_targets": ["3"], "name": "7", }, "8": { - "instructions": ["y = 4"], + "instructions": ["z = 4"], "jump_targets": ["3"], "name": "8", }, } - self.compare(function, expected, empty={"6", "9"}) + self.compare( + function, + expected, + empty={"6", "9"}, + arguments=[(0, 0), (0, 1), (1, 1), (1, 2)], + ) def test_nested_if_with_empty_else_and_return(self): def function(x: int, y: int) -> None: - y << 2 - if x < 10: - y -= 1 - if y < 5: - y = 1 + z = 0 + if x < 1: + y << 2 + if y < 1: + z = 1 else: - if y < 15: - y = 2 + if y < 2: + z = 2 else: - return + return y, 4 y += 1 - return y + return y, z expected = { "0": { - "instructions": ["y << 2", "x < 10"], + "instructions": ["z = 0", "x < 1"], "jump_targets": ["1", "2"], "name": "0", }, "1": { - "instructions": ["y -= 1", "y < 5"], + "instructions": ["y << 2", "y < 1"], "jump_targets": ["4", "3"], "name": "1", }, "2": { - "instructions": ["y < 15"], + "instructions": ["y < 2"], "jump_targets": ["7", "8"], "name": "2", }, "3": { - "instructions": ["return y"], + "instructions": ["return (y, z)"], "jump_targets": [], "name": "3", }, "4": { - "instructions": ["y = 1"], + "instructions": ["z = 1"], "jump_targets": ["3"], "name": "4", }, "7": { - "instructions": ["y = 2"], + "instructions": ["z = 2"], "jump_targets": ["9"], "name": "7", }, - "8": {"instructions": ["return"], "jump_targets": [], "name": "8"}, + "8": { + "instructions": ["return (y, 4)"], + "jump_targets": [], + "name": "8", + }, "9": { "instructions": ["y += 1"], "jump_targets": ["3"], "name": "9", }, } - self.compare(function, expected, empty={"5", "6"}) + self.compare( + function, + expected, + empty={"5", "6"}, + arguments=[ + (0, 0), + (0, 1), + (1, 1), + (1, 2), + ], + ) def test_elif(self): - def function(x: int, a: int, b: int) -> int: - if x < 10: - return - elif x < 15: - y = b - a - elif x < 20: - y = a**2 + def function(x: int) -> int: + if x < 1: + return 10 + elif x < 2: + y = 20 + elif x < 3: + y = 30 else: - y = a - b + y = 40 return y expected = { "0": { - "instructions": ["x < 10"], + "instructions": ["x < 1"], "jump_targets": ["1", "2"], "name": "0", }, - "1": {"instructions": ["return"], "jump_targets": [], "name": "1"}, + "1": { + "instructions": ["return 10"], + "jump_targets": [], + "name": "1", + }, "2": { - "instructions": ["x < 15"], + "instructions": ["x < 2"], "jump_targets": ["4", "5"], "name": "2", }, @@ -340,27 +460,37 @@ def function(x: int, a: int, b: int) -> int: "name": "3", }, "4": { - "instructions": ["y = b - a"], + "instructions": ["y = 20"], "jump_targets": ["3"], "name": "4", }, "5": { - "instructions": ["x < 20"], + "instructions": ["x < 3"], "jump_targets": ["7", "8"], "name": "5", }, "7": { - "instructions": ["y = a ** 2"], + "instructions": ["y = 30"], "jump_targets": ["3"], "name": "7", }, "8": { - "instructions": ["y = a - b"], + "instructions": ["y = 40"], "jump_targets": ["3"], "name": "8", }, } - self.compare(function, expected, empty={"9", "6"}) + self.compare( + function, + expected, + empty={"9", "6"}, + arguments=[ + (0,), + (1,), + (2,), + (3,), + ], + ) def test_simple_while(self): def function() -> int: @@ -483,9 +613,9 @@ def function() -> int: self.compare(function, expected, empty={"4", "7"}) def test_while_in_if(self): - def function(a: bool) -> int: + def function(y: int) -> int: x = 0 - if a is True: + if y == 0: while x < 10: x += 2 else: @@ -495,7 +625,7 @@ def function(a: bool) -> int: expected = { "0": { - "instructions": ["x = 0", "a is True"], + "instructions": ["x = 0", "y == 0"], "jump_targets": ["4", "8"], "name": "0", }, @@ -526,55 +656,63 @@ def function(a: bool) -> int: }, } self.compare( - function, expected, empty={"1", "2", "6", "7", "10", "11"} + function, + expected, + empty={"1", "2", "6", "7", "10", "11"}, + arguments=[(0,), (1,)], ) def test_while_break_continue(self): - def function() -> int: - x = 0 - while x < 10: - x += 1 - if x % 2 == 0: + def function(x: int) -> int: + y = 0 + while y < 10: + y += 1 + if x == 0: continue - elif x == 9: + elif x == 1: break else: - x += 1 - return x + y += 10 + return y expected = { "0": { - "instructions": ["x = 0"], + "instructions": ["y = 0"], "jump_targets": ["1"], "name": "0", }, "1": { - "instructions": ["x < 10"], + "instructions": ["y < 10"], "jump_targets": ["2", "3"], "name": "1", }, "2": { - "instructions": ["x += 1", "x % 2 == 0"], + "instructions": ["y += 1", "x == 0"], "jump_targets": ["1", "6"], "name": "2", }, "3": { - "instructions": ["return x"], + "instructions": ["return y"], "jump_targets": [], "name": "3", }, "6": { - "instructions": ["x == 9"], + "instructions": ["x == 1"], "jump_targets": ["3", "9"], "name": "6", }, "9": { - "instructions": ["x += 1"], + "instructions": ["y += 10"], "jump_targets": ["1"], "name": "9", }, } - self.compare(function, expected, empty={"4", "5", "7", "8", "10"}) + self.compare( + function, + expected, + empty={"4", "5", "7", "8", "10"}, + arguments=[(0,), (1,), (2,)], + ) def test_while_else(self): def function() -> int: @@ -616,15 +754,15 @@ def function() -> int: def test_simple_for(self): def function() -> int: - c = 0 + x = 0 for i in range(10): - c += i - return c + x += i + return x, i expected = { "0": { "instructions": [ - "c = 0", + "x = 0", "__iterator_1__ = iter(range(10))", "i = None", ], @@ -641,7 +779,7 @@ def function() -> int: "name": "1", }, "2": { - "instructions": ["c += i"], + "instructions": ["x += i"], "jump_targets": ["1"], "name": "2", }, @@ -651,7 +789,7 @@ def function() -> int: "name": "3", }, "4": { - "instructions": ["return c"], + "instructions": ["return (x, i)"], "jump_targets": [], "name": "4", }, @@ -660,17 +798,17 @@ def function() -> int: def test_nested_for(self): def function() -> int: - c = 0 + x = 0 for i in range(3): - c += i + x += i for j in range(3): - c += j - return c + x += j + return x, i, j expected = { "0": { "instructions": [ - "c = 0", + "x = 0", "__iterator_1__ = iter(range(3))", "i = None", ], @@ -688,7 +826,7 @@ def function() -> int: }, "2": { "instructions": [ - "c += i", + "x += i", "__iterator_5__ = iter(range(3))", "j = None", ], @@ -701,7 +839,7 @@ def function() -> int: "name": "3", }, "4": { - "instructions": ["return c"], + "instructions": ["return (x, i, j)"], "jump_targets": [], "name": "4", }, @@ -715,7 +853,7 @@ def function() -> int: "name": "5", }, "6": { - "instructions": ["c += j"], + "instructions": ["x += j"], "jump_targets": ["5"], "name": "6", }, @@ -728,12 +866,12 @@ def function() -> int: self.compare(function, expected, empty={"8"}) def test_for_with_return_break_and_continue(self): - def function(a: int, b: int) -> int: + def function(x: int, y: int) -> int: for i in range(2): - if i == a: + if i == x: i = 3 return i - elif i == b: + elif i == y: i = 4 break else: @@ -759,7 +897,7 @@ def function(a: int, b: int) -> int: "name": "1", }, "2": { - "instructions": ["i == a"], + "instructions": ["i == x"], "jump_targets": ["5", "6"], "name": "2", }, @@ -779,7 +917,7 @@ def function(a: int, b: int) -> int: "name": "5", }, "6": { - "instructions": ["i == b"], + "instructions": ["i == y"], "jump_targets": ["8", "1"], "name": "6", }, @@ -789,24 +927,30 @@ def function(a: int, b: int) -> int: "name": "8", }, } - self.compare(function, expected, unreachable={"7", "10"}, empty={"9"}) + self.compare( + function, + expected, + unreachable={"7", "10"}, + empty={"9"}, + arguments=[(0, 0), (2, 0), (2, 2)], + ) def test_for_with_if_in_else(self): - def function(a: int): - c = 0 + def function(y: int): + x = 0 for i in range(10): - c += i + x += i else: - if a: - r = c + if y == 0: + x += 10 else: - r = -1 * c - return r + x += 20 + return (x, y, i) expected = { "0": { "instructions": [ - "c = 0", + "x = 0", "__iterator_1__ = iter(range(10))", "i = None", ], @@ -823,56 +967,56 @@ def function(a: int): "name": "1", }, "2": { - "instructions": ["c += i"], + "instructions": ["x += i"], "jump_targets": ["1"], "name": "2", }, "3": { - "instructions": ["i = __iter_last_1__", "a"], + "instructions": ["i = __iter_last_1__", "y == 0"], "jump_targets": ["5", "6"], "name": "3", }, "4": { - "instructions": ["return r"], + "instructions": ["return (x, y, i)"], "jump_targets": [], "name": "4", }, "5": { - "instructions": ["r = c"], + "instructions": ["x += 10"], "jump_targets": ["4"], "name": "5", }, "6": { - "instructions": ["r = -1 * c"], + "instructions": ["x += 20"], "jump_targets": ["4"], "name": "6", }, } - self.compare(function, expected, empty={"7"}) + self.compare(function, expected, empty={"7"}, arguments=[(0,), (1,)]) def test_for_with_nested_for_else(self): - def function(a: bool) -> int: - c = 1 + def function(y: int) -> int: + x = 1 for i in range(1): for j in range(1): - if a: - c *= 3 + if y == 0: + x *= 3 break # This break decides, if True skip continue. else: - c *= 5 + x *= 5 continue # Causes break below to be skipped. - c *= 7 + x *= 7 break # Causes the else below to be skipped else: - c *= 9 # Not breaking in inner loop leads here - return c + x *= 9 # Not breaking in inner loop leads here + return x, y, i - self.assertEqual(function(True), 3 * 7) - self.assertEqual(function(False), 5 * 9) + self.assertEqual(function(1)[0], 5 * 9) + self.assertEqual(function(0)[0], 3 * 7) expected = { "0": { "instructions": [ - "c = 1", + "x = 1", "__iterator_1__ = iter(range(1))", "i = None", ], @@ -897,12 +1041,12 @@ def function(a: bool) -> int: "name": "2", }, "3": { - "instructions": ["i = __iter_last_1__", "c *= 9"], + "instructions": ["i = __iter_last_1__", "x *= 9"], "jump_targets": ["4"], "name": "3", }, "4": { - "instructions": ["return c"], + "instructions": ["return (x, y, i)"], "jump_targets": [], "name": "4", }, @@ -916,56 +1060,58 @@ def function(a: bool) -> int: "name": "5", }, "6": { - "instructions": ["a"], + "instructions": ["y == 0"], "jump_targets": ["9", "5"], "name": "6", }, "7": { - "instructions": ["j = __iter_last_5__", "c *= 5"], + "instructions": ["j = __iter_last_5__", "x *= 5"], "jump_targets": ["1"], "name": "7", }, "8": { - "instructions": ["c *= 7"], + "instructions": ["x *= 7"], "jump_targets": ["4"], "name": "8", }, "9": { - "instructions": ["c *= 3"], + "instructions": ["x *= 3"], "jump_targets": ["8"], "name": "9", }, } - self.compare(function, expected, empty={"11", "10"}) + self.compare( + function, expected, empty={"11", "10"}, arguments=[(0,), (1,)] + ) def test_for_with_nested_else_return_break_and_continue(self): - def function(a: int, b: int, c: int, d: int, e: int, f: int) -> int: + def function(x: int) -> int: for i in range(2): - if i == a: - i = 3 + if x == 1: + i += 1 return i - elif i == b: - i = 4 + elif x == 2: + i += 2 break - elif i == c: - i = 5 + elif x == 3: + i += 3 continue else: while i < 10: i += 1 - if i == d: - i = 3 + if x == 4: + i += 4 return i - elif i == e: - i = 4 + elif x == 5: + i += 5 break - elif i == f: - i = 5 + elif x == 6: + i += 6 continue else: - i += 1 - return i + i += 7 + return x, i expected = { "0": { @@ -986,7 +1132,7 @@ def function(a: int, b: int, c: int, d: int, e: int, f: int) -> int: "name": "1", }, "11": { - "instructions": ["i = 5"], + "instructions": ["i += 3"], "jump_targets": ["1"], "name": "11", }, @@ -996,42 +1142,42 @@ def function(a: int, b: int, c: int, d: int, e: int, f: int) -> int: "name": "14", }, "15": { - "instructions": ["i += 1", "i == d"], + "instructions": ["i += 1", "x == 4"], "jump_targets": ["18", "19"], "name": "15", }, "18": { - "instructions": ["i = 3", "return i"], + "instructions": ["i += 4", "return i"], "jump_targets": [], "name": "18", }, "19": { - "instructions": ["i == e"], + "instructions": ["x == 5"], "jump_targets": ["21", "22"], "name": "19", }, "2": { - "instructions": ["i == a"], + "instructions": ["x == 1"], "jump_targets": ["5", "6"], "name": "2", }, "21": { - "instructions": ["i = 4"], + "instructions": ["i += 5"], "jump_targets": ["1"], "name": "21", }, "22": { - "instructions": ["i == f"], + "instructions": ["x == 6"], "jump_targets": ["24", "25"], "name": "22", }, "24": { - "instructions": ["i = 5"], + "instructions": ["i += 6"], "jump_targets": ["14"], "name": "24", }, "25": { - "instructions": ["i += 1"], + "instructions": ["i += 7"], "jump_targets": ["14"], "name": "25", }, @@ -1041,33 +1187,59 @@ def function(a: int, b: int, c: int, d: int, e: int, f: int) -> int: "name": "3", }, "4": { - "instructions": ["return i"], + "instructions": ["return (x, i)"], "jump_targets": [], "name": "4", }, "5": { - "instructions": ["i = 3", "return i"], + "instructions": ["i += 1", "return i"], "jump_targets": [], "name": "5", }, "6": { - "instructions": ["i == b"], + "instructions": ["x == 2"], "jump_targets": ["8", "9"], "name": "6", }, "8": { - "instructions": ["i = 4"], + "instructions": ["i += 2"], "jump_targets": ["4"], "name": "8", }, "9": { - "instructions": ["i == c"], + "instructions": ["x == 3"], "jump_targets": ["11", "14"], "name": "9", }, } empty = {"7", "10", "12", "13", "16", "17", "20", "23", "26"} - self.compare(function, expected, empty=empty) + arguments = [(1,), (2,), (3,), (4,), (5,), (6,), (7,)] + self.compare( + function, + expected, + empty=empty, + arguments=arguments, + ) + + +class TestEntryPoints(TestCase): + + def test_rondtrip(self): + + def function() -> int: + x = 0 + for i in range(2): + x += i + return x, i + + scfg = AST2SCFG(function) + scfg.restructure() + ast_ = SCFG2AST(function, scfg) + + exec_locals = {} + exec(ast.unparse(ast_), {}, exec_locals) + transformed = exec_locals["transformed_function"] + assert function() == transformed() if __name__ == "__main__": From ea3ef44ac5201a67a519221ba2ed4b2003e4f09b Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 May 2024 20:35:53 +0200 Subject: [PATCH 02/12] avoid use of break and continue As title --- .../core/datastructures/ast_transforms.py | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index 17a652b..5149ccd 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -762,11 +762,16 @@ def codegen_view() -> list[Any]: # A loop region gives rise to a Python while True loop. We # recursively visit the body. rval = [ + ast.Assign( + [ast.Name("__loop_cont__")], + ast.Constant(True), + lineno=0, + ), ast.While( - test=ast.Constant(value=True), + test=ast.Name("__loop_cont__"), body=codegen_view(), orelse=[], - ) + ), ] else: raise NotImplementedError @@ -791,28 +796,17 @@ def codegen_view() -> list[Any]: # special reserved variable. return [ast.Return(ast.Name("__return_value__"))] elif type(block) is SyntheticExitingLatch: - # The synthetic exiting latch much create a query on the variable - # it holds and then insert a Python if that will either break or - # continue. This effectively generates the backedge for the looping - # region. + # The synthetic exiting latch simply assigns the negated value of + # the exit variable to '__loop_cont__'. assert len(block.jump_targets) == 1 assert len(block.backedges) == 1 - compare_value = [ - i - for i, v in block.branch_value_table.items() - if v == block.backedges[0] - ][0] - if_beak_node_test = ast.Compare( - left=ast.Name(block.variable), - ops=[ast.Eq()], - comparators=[ast.Constant(compare_value)], - ) - if_break_node = ast.If( - test=if_beak_node_test, - body=[ast.Continue()], - orelse=[ast.Break()], - ) - return [if_break_node] + return [ + ast.Assign( + [ast.Name("__loop_cont__")], + ast.UnaryOp(ast.Not(), ast.Name(block.variable)), + lineno=0, + ) + ] elif type(block) in (SyntheticExitBranch, SyntheticHead): # Both the Synthetic exit branch and the synthetic head contain a # branching statement with potentially multiple outgoing branches. From 6b31686322626d6f8f1b87dd8e2e0292bfa0a1c3 Mon Sep 17 00:00:00 2001 From: esc Date: Wed, 29 May 2024 20:40:37 +0200 Subject: [PATCH 03/12] update docs As title --- numba_rvsdg/core/datastructures/ast_transforms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index 5149ccd..a51ba8d 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -759,8 +759,9 @@ def codegen_view() -> list[Any]: elif block.kind == "branch": rval = codegen_view() elif block.kind == "loop": - # A loop region gives rise to a Python while True loop. We - # recursively visit the body. + # A loop region gives rise to a Python while __loop_cont__ + # loop. We recursively visit the body. The exiting latch will + # update __loop_continue__. rval = [ ast.Assign( [ast.Name("__loop_cont__")], From f971cc01d7e1360701c1f29d0d715f7930a3e33f Mon Sep 17 00:00:00 2001 From: esc Date: Fri, 31 May 2024 08:51:43 +0200 Subject: [PATCH 04/12] fix running functions during test, increase coverage As title --- numba_rvsdg/tests/test_ast_transforms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/numba_rvsdg/tests/test_ast_transforms.py b/numba_rvsdg/tests/test_ast_transforms.py index 3337926..3c58cbc 100644 --- a/numba_rvsdg/tests/test_ast_transforms.py +++ b/numba_rvsdg/tests/test_ast_transforms.py @@ -37,11 +37,13 @@ def compare( ): # Execute function with first argument, if given. Ensure function is # sane and make sure it's picked up by coverage. - if type(function) is Callable: + try: if arguments: function(*arguments[0]) else: function() + except Exception: + pass # First, test against the expected CFG... ast2scfg_transformer = AST2SCFGTransformer(function) astcfg = ast2scfg_transformer.transform_to_ASTCFG() From 24b57b0cef92cf5de1dece086bd3d44a99a300a3 Mon Sep 17 00:00:00 2001 From: esc Date: Fri, 31 May 2024 08:54:36 +0200 Subject: [PATCH 05/12] make sure the tracers picked something up As title --- numba_rvsdg/tests/test_ast_transforms.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/numba_rvsdg/tests/test_ast_transforms.py b/numba_rvsdg/tests/test_ast_transforms.py index 3c58cbc..50029c1 100644 --- a/numba_rvsdg/tests/test_ast_transforms.py +++ b/numba_rvsdg/tests/test_ast_transforms.py @@ -99,6 +99,9 @@ def compare( else: transformed_results = [temporary_transformed_function()] + assert len(original_callback.lines) > 0 + assert len(transformed_callback.lines) > 0 + # Check call results assert original_results == transformed_results From ab0b3372ab3ae615819023c69c39f86b9384b993 Mon Sep 17 00:00:00 2001 From: esc Date: Fri, 31 May 2024 08:58:36 +0200 Subject: [PATCH 06/12] fix coverage for test functions, run all branches As title --- numba_rvsdg/tests/test_ast_transforms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/numba_rvsdg/tests/test_ast_transforms.py b/numba_rvsdg/tests/test_ast_transforms.py index 50029c1..02d12af 100644 --- a/numba_rvsdg/tests/test_ast_transforms.py +++ b/numba_rvsdg/tests/test_ast_transforms.py @@ -39,9 +39,10 @@ def compare( # sane and make sure it's picked up by coverage. try: if arguments: - function(*arguments[0]) + for a in arguments: + _ = function(*a) else: - function() + _ = function() except Exception: pass # First, test against the expected CFG... From 57ca469d55c2ae8efb85d05ae068d106ceef58b2 Mon Sep 17 00:00:00 2001 From: esc Date: Fri, 31 May 2024 09:03:22 +0200 Subject: [PATCH 07/12] secure codegen by not returning a default value As title --- numba_rvsdg/core/datastructures/ast_transforms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index a51ba8d..410ae2b 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -787,7 +787,7 @@ def codegen_view() -> list[Any]: ] elif type(block) is SyntheticTail: # Synthetic tails do nothing. - pass + return [] elif type(block) is SyntheticFill: # Synthetic fills must have a pass statement to main syntactical # correctness of the final program. @@ -861,7 +861,8 @@ def if_cascade(jump_targets: list[str]) -> list[ast.AST]: return if_cascade(list(block.jump_targets[::-1])) else: raise NotImplementedError - return [] + + raise NotImplementedError("unreachable") def AST2SCFG(code: str | list[ast.FunctionDef] | Callable[..., Any]) -> SCFG: From 3da4ecea0694d532e579748e16e907d73e15b21f Mon Sep 17 00:00:00 2001 From: esc Date: Sat, 1 Jun 2024 15:42:53 +0200 Subject: [PATCH 08/12] fix codegen_view to return non-nested AST As title --- .../core/datastructures/ast_transforms.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index 410ae2b..a642d50 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -744,19 +744,13 @@ def codegen(self, block: Any) -> list[ast.AST]: # blocks with multiple jump targets and all other regions must be # visited linearly. def codegen_view() -> list[Any]: - return [ - self.codegen(b) - for b in block.subregion.concealed_region_view.values() # type: ignore # noqa - if not (type(b) is RegionBlock and b.kind == "branch") - ] + r = [] + for b in block.subregion.concealed_region_view.values(): # type: ignore # noqa + if not (type(b) is RegionBlock and b.kind == "branch"): + r.extend(self.codegen(b)) + return r - # Head, tail and branch regions themselves to use the custom view - # above. - if block.kind == "head": - rval = codegen_view() - elif block.kind == "tail": - rval = codegen_view() - elif block.kind == "branch": + if block.kind in ("head", "tail", "branch"): rval = codegen_view() elif block.kind == "loop": # A loop region gives rise to a Python while __loop_cont__ From 682aff70b1b89fd2d684a8003811e7a04cbfc8b5 Mon Sep 17 00:00:00 2001 From: esc Date: Sat, 1 Jun 2024 16:00:29 +0200 Subject: [PATCH 09/12] cleanup codegen_view As title --- numba_rvsdg/core/datastructures/ast_transforms.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index a642d50..a7648db 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -1,5 +1,6 @@ import ast import inspect +import itertools from typing import Callable, Any, MutableMapping import textwrap from dataclasses import dataclass @@ -744,11 +745,13 @@ def codegen(self, block: Any) -> list[ast.AST]: # blocks with multiple jump targets and all other regions must be # visited linearly. def codegen_view() -> list[Any]: - r = [] - for b in block.subregion.concealed_region_view.values(): # type: ignore # noqa - if not (type(b) is RegionBlock and b.kind == "branch"): - r.extend(self.codegen(b)) - return r + return list( + itertools.chain.from_iterable( + self.codegen(b) + for b in block.subregion.concealed_region_view.values() # type: ignore # noqa + if not (type(b) is RegionBlock and b.kind == "branch") + ) + ) if block.kind in ("head", "tail", "branch"): rval = codegen_view() From 40b9daabcaa42c8de150758ee404a6ee36ae026d Mon Sep 17 00:00:00 2001 From: esc Date: Mon, 3 Jun 2024 18:30:29 +0200 Subject: [PATCH 10/12] prefix all SCFG variables with __scfg_ As title --- .../core/datastructures/ast_transforms.py | 37 +++++---- numba_rvsdg/core/datastructures/scfg.py | 4 +- numba_rvsdg/tests/test_ast_transforms.py | 80 +++++++++---------- numba_rvsdg/tests/test_figures.py | 26 +++--- numba_rvsdg/tests/test_rendering.py | 14 ++-- 5 files changed, 82 insertions(+), 79 deletions(-) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index a7648db..6c32500 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -586,8 +586,8 @@ def function(a: int) -> None # the CFG. target = ast.unparse(node.target) iter_setup = ast.unparse(node.iter) - iter_assign = f"__iterator_{head_index}__" - last_target_value = f"__iter_last_{head_index}__" + iter_assign = f"__scfg_iterator_{head_index}__" + last_target_value = f"__scfg_iter_last_{head_index}__" # Emit iterator setup to pre-header. preheader_code = textwrap.dedent( @@ -605,14 +605,14 @@ def function(a: int) -> None # Emit header instructions. This first makes a backup of the iteration # target and then checks if the iterator is exhausted and if the loop - # should continue. The '__sentinel__' is an singleton style marker, so - # it need not be versioned. + # should continue. The '_scfg__sentinel__' is an singleton style + # marker, so it need not be versioned. header_code = textwrap.dedent( f""" {last_target_value} = {target} - {target} = next({iter_assign}, "__sentinel__") - {target} != "__sentinel__" + {target} = next({iter_assign}, "__scfg_sentinel__") + {target} != "__scfg_sentinel__" """ ) self.codegen(ast.parse(header_code).body) @@ -720,12 +720,13 @@ def codegen(self, block: Any) -> list[ast.AST]: # The value of the ast.Return could be either None or an # ast.AST type. In the case of None, this refers to a plain # 'return', which is implicitly 'return None'. So, if it is - # None, we assign the __return_value__ a ast.Constant(None) and - # whatever the ast.AST node is otherwise. + # None, we assign the __scfg_return_value__ an + # ast.Constant(None) and whatever the ast.AST node is + # otherwise. val = block.tree[-1].value return block.tree[:-1] + [ ast.Assign( - [ast.Name("__return_value__")], + [ast.Name("__scfg_return_value__")], (ast.Constant(None) if val is None else val), lineno=0, ) @@ -756,17 +757,17 @@ def codegen_view() -> list[Any]: if block.kind in ("head", "tail", "branch"): rval = codegen_view() elif block.kind == "loop": - # A loop region gives rise to a Python while __loop_cont__ + # A loop region gives rise to a Python while __scfg_loop_cont__ # loop. We recursively visit the body. The exiting latch will - # update __loop_continue__. + # update __scfg_loop_continue__. rval = [ ast.Assign( - [ast.Name("__loop_cont__")], + [ast.Name("__scfg_loop_cont__")], ast.Constant(True), lineno=0, ), ast.While( - test=ast.Name("__loop_cont__"), + test=ast.Name("__scfg_loop_cont__"), body=codegen_view(), orelse=[], ), @@ -779,7 +780,9 @@ def codegen_view() -> list[Any]: # Synthetic assignments just create Python assignments, one for # each variable.. return [ - ast.Assign([ast.Name(t)], ast.Constant(v), lineno=0) + ast.Assign( + [ast.Name(t)], ast.Constant(v), lineno=0 + ) for t, v in block.variable_assignment.items() ] elif type(block) is SyntheticTail: @@ -792,15 +795,15 @@ def codegen_view() -> list[Any]: elif type(block) is SyntheticReturn: # Synthetic return blocks must re-assigne the return value to a # special reserved variable. - return [ast.Return(ast.Name("__return_value__"))] + return [ast.Return(ast.Name("__scfg_return_value__"))] elif type(block) is SyntheticExitingLatch: # The synthetic exiting latch simply assigns the negated value of - # the exit variable to '__loop_cont__'. + # the exit variable to '__scfg_loop_cont__'. assert len(block.jump_targets) == 1 assert len(block.backedges) == 1 return [ ast.Assign( - [ast.Name("__loop_cont__")], + [ast.Name("__scfg_loop_cont__")], ast.UnaryOp(ast.Not(), ast.Name(block.variable)), lineno=0, ) diff --git a/numba_rvsdg/core/datastructures/scfg.py b/numba_rvsdg/core/datastructures/scfg.py index f598b36..016622e 100644 --- a/numba_rvsdg/core/datastructures/scfg.py +++ b/numba_rvsdg/core/datastructures/scfg.py @@ -138,11 +138,11 @@ def new_var_name(self, kind: str) -> str: """ if kind in self.kinds.keys(): idx = self.kinds[kind] - name = str(kind) + "_var_" + str(idx) + name = "__scfg_" + str(kind) + "_var_" + str(idx) + "__" self.kinds[kind] = idx + 1 else: idx = 0 - name = str(kind) + "_var_" + str(idx) + name = "__scfg_" + str(kind) + "_var_" + str(idx) + "__" self.kinds[kind] = idx + 1 return name diff --git a/numba_rvsdg/tests/test_ast_transforms.py b/numba_rvsdg/tests/test_ast_transforms.py index 02d12af..bfc7dc0 100644 --- a/numba_rvsdg/tests/test_ast_transforms.py +++ b/numba_rvsdg/tests/test_ast_transforms.py @@ -769,7 +769,7 @@ def function() -> int: "0": { "instructions": [ "x = 0", - "__iterator_1__ = iter(range(10))", + "__scfg_iterator_1__ = iter(range(10))", "i = None", ], "jump_targets": ["1"], @@ -777,9 +777,9 @@ def function() -> int: }, "1": { "instructions": [ - "__iter_last_1__ = i", - "i = next(__iterator_1__, '__sentinel__')", - "i != '__sentinel__'", + "__scfg_iter_last_1__ = i", + "i = next(__scfg_iterator_1__, '__scfg_sentinel__')", + "i != '__scfg_sentinel__'", ], "jump_targets": ["2", "3"], "name": "1", @@ -790,7 +790,7 @@ def function() -> int: "name": "2", }, "3": { - "instructions": ["i = __iter_last_1__"], + "instructions": ["i = __scfg_iter_last_1__"], "jump_targets": ["4"], "name": "3", }, @@ -815,7 +815,7 @@ def function() -> int: "0": { "instructions": [ "x = 0", - "__iterator_1__ = iter(range(3))", + "__scfg_iterator_1__ = iter(range(3))", "i = None", ], "jump_targets": ["1"], @@ -823,9 +823,9 @@ def function() -> int: }, "1": { "instructions": [ - "__iter_last_1__ = i", - "i = next(__iterator_1__, '__sentinel__')", - "i != '__sentinel__'", + "__scfg_iter_last_1__ = i", + "i = next(__scfg_iterator_1__, '__scfg_sentinel__')", + "i != '__scfg_sentinel__'", ], "jump_targets": ["2", "3"], "name": "1", @@ -833,14 +833,14 @@ def function() -> int: "2": { "instructions": [ "x += i", - "__iterator_5__ = iter(range(3))", + "__scfg_iterator_5__ = iter(range(3))", "j = None", ], "jump_targets": ["5"], "name": "2", }, "3": { - "instructions": ["i = __iter_last_1__"], + "instructions": ["i = __scfg_iter_last_1__"], "jump_targets": ["4"], "name": "3", }, @@ -851,9 +851,9 @@ def function() -> int: }, "5": { "instructions": [ - "__iter_last_5__ = j", - "j = next(__iterator_5__, '__sentinel__')", - "j != '__sentinel__'", + "__scfg_iter_last_5__ = j", + "j = next(__scfg_iterator_5__, '__scfg_sentinel__')", + "j != '__scfg_sentinel__'", ], "jump_targets": ["6", "7"], "name": "5", @@ -864,7 +864,7 @@ def function() -> int: "name": "6", }, "7": { - "instructions": ["j = __iter_last_5__"], + "instructions": ["j = __scfg_iter_last_5__"], "jump_targets": ["1"], "name": "7", }, @@ -887,7 +887,7 @@ def function(x: int, y: int) -> int: expected = { "0": { "instructions": [ - "__iterator_1__ = iter(range(2))", + "__scfg_iterator_1__ = iter(range(2))", "i = None", ], "jump_targets": ["1"], @@ -895,9 +895,9 @@ def function(x: int, y: int) -> int: }, "1": { "instructions": [ - "__iter_last_1__ = i", - "i = next(__iterator_1__, '__sentinel__')", - "i != '__sentinel__'", + "__scfg_iter_last_1__ = i", + "i = next(__scfg_iterator_1__, '__scfg_sentinel__')", + "i != '__scfg_sentinel__'", ], "jump_targets": ["2", "3"], "name": "1", @@ -908,7 +908,7 @@ def function(x: int, y: int) -> int: "name": "2", }, "3": { - "instructions": ["i = __iter_last_1__"], + "instructions": ["i = __scfg_iter_last_1__"], "jump_targets": ["4"], "name": "3", }, @@ -957,7 +957,7 @@ def function(y: int): "0": { "instructions": [ "x = 0", - "__iterator_1__ = iter(range(10))", + "__scfg_iterator_1__ = iter(range(10))", "i = None", ], "jump_targets": ["1"], @@ -965,9 +965,9 @@ def function(y: int): }, "1": { "instructions": [ - "__iter_last_1__ = i", - "i = next(__iterator_1__, '__sentinel__')", - "i != '__sentinel__'", + "__scfg_iter_last_1__ = i", + "i = next(__scfg_iterator_1__, '__scfg_sentinel__')", + "i != '__scfg_sentinel__'", ], "jump_targets": ["2", "3"], "name": "1", @@ -978,7 +978,7 @@ def function(y: int): "name": "2", }, "3": { - "instructions": ["i = __iter_last_1__", "y == 0"], + "instructions": ["i = __scfg_iter_last_1__", "y == 0"], "jump_targets": ["5", "6"], "name": "3", }, @@ -1023,7 +1023,7 @@ def function(y: int) -> int: "0": { "instructions": [ "x = 1", - "__iterator_1__ = iter(range(1))", + "__scfg_iterator_1__ = iter(range(1))", "i = None", ], "jump_targets": ["1"], @@ -1031,23 +1031,23 @@ def function(y: int) -> int: }, "1": { "instructions": [ - "__iter_last_1__ = i", - "i = next(__iterator_1__, '__sentinel__')", - "i != '__sentinel__'", + "__scfg_iter_last_1__ = i", + "i = next(__scfg_iterator_1__, '__scfg_sentinel__')", + "i != '__scfg_sentinel__'", ], "jump_targets": ["2", "3"], "name": "1", }, "2": { "instructions": [ - "__iterator_5__ = iter(range(1))", + "__scfg_iterator_5__ = iter(range(1))", "j = None", ], "jump_targets": ["5"], "name": "2", }, "3": { - "instructions": ["i = __iter_last_1__", "x *= 9"], + "instructions": ["i = __scfg_iter_last_1__", "x *= 9"], "jump_targets": ["4"], "name": "3", }, @@ -1058,9 +1058,9 @@ def function(y: int) -> int: }, "5": { "instructions": [ - "__iter_last_5__ = j", - "j = next(__iterator_5__, '__sentinel__')", - "j != '__sentinel__'", + "__scfg_iter_last_5__ = j", + "j = next(__scfg_iterator_5__, '__scfg_sentinel__')", + "j != '__scfg_sentinel__'", ], "jump_targets": ["6", "7"], "name": "5", @@ -1071,7 +1071,7 @@ def function(y: int) -> int: "name": "6", }, "7": { - "instructions": ["j = __iter_last_5__", "x *= 5"], + "instructions": ["j = __scfg_iter_last_5__", "x *= 5"], "jump_targets": ["1"], "name": "7", }, @@ -1122,7 +1122,7 @@ def function(x: int) -> int: expected = { "0": { "instructions": [ - "__iterator_1__ = iter(range(2))", + "__scfg_iterator_1__ = iter(range(2))", "i = None", ], "jump_targets": ["1"], @@ -1130,9 +1130,9 @@ def function(x: int) -> int: }, "1": { "instructions": [ - "__iter_last_1__ = i", - "i = next(__iterator_1__, '__sentinel__')", - "i != '__sentinel__'", + "__scfg_iter_last_1__ = i", + "i = next(__scfg_iterator_1__, '__scfg_sentinel__')", + "i != '__scfg_sentinel__'", ], "jump_targets": ["2", "3"], "name": "1", @@ -1188,7 +1188,7 @@ def function(x: int) -> int: "name": "25", }, "3": { - "instructions": ["i = __iter_last_1__"], + "instructions": ["i = __scfg_iter_last_1__"], "jump_targets": ["4"], "name": "3", }, diff --git a/numba_rvsdg/tests/test_figures.py b/numba_rvsdg/tests/test_figures.py index 9476634..7a3a569 100644 --- a/numba_rvsdg/tests/test_figures.py +++ b/numba_rvsdg/tests/test_figures.py @@ -145,34 +145,34 @@ end: 22 synth_asign_block_0: type: synth_asign - variable_assignment: {'control_var_0': 0} + variable_assignment: {'__scfg_control_var_0__': 0} synth_asign_block_1: type: synth_asign - variable_assignment: {'control_var_0': 1} + variable_assignment: {'__scfg_control_var_0__': 1} synth_asign_block_2: type: synth_asign - variable_assignment: {'control_var_0': 0, 'backedge_var_0': 1} + variable_assignment: {'__scfg_control_var_0__': 0, '__scfg_backedge_var_0__': 1} synth_asign_block_3: type: synth_asign - variable_assignment: {'backedge_var_0': 0, 'control_var_0': 1} + variable_assignment: {'__scfg_backedge_var_0__': 0, '__scfg_control_var_0__': 1} synth_asign_block_4: type: synth_asign - variable_assignment: {'control_var_0': 1, 'backedge_var_0': 1} + variable_assignment: {'__scfg_control_var_0__': 1, '__scfg_backedge_var_0__': 1} synth_asign_block_5: type: synth_asign - variable_assignment: {'backedge_var_0': 0, 'control_var_0': 0} + variable_assignment: {'__scfg_backedge_var_0__': 0, '__scfg_control_var_0__': 0} synth_exit_block_0: type: synth_exit_branch branch_value_table: {0: 'branch_region_2', 1: 'branch_region_3'} - variable: control_var_0 + variable: __scfg_control_var_0__ synth_exit_latch_block_0: type: synth_exit_latch branch_value_table: {1: 'synth_exit_block_0', 0: 'head_region_2'} - variable: backedge_var_0 + variable: __scfg_backedge_var_0__ synth_head_block_0: type: synth_head branch_value_table: {0: 'branch_region_4', 1: 'branch_region_5'} - variable: control_var_0 + variable: __scfg_control_var_0__ synth_tail_block_0: type: synth_tail synth_tail_block_1: @@ -345,19 +345,19 @@ end: 20 synth_asign_block_0: type: synth_asign - variable_assignment: {'control_var_0': 0} + variable_assignment: {'__scfg_control_var_0__': 0} synth_asign_block_1: type: synth_asign - variable_assignment: {'control_var_0': 1} + variable_assignment: {'__scfg_control_var_0__': 1} synth_asign_block_2: type: synth_asign - variable_assignment: {'control_var_0': 2} + variable_assignment: {'__scfg_control_var_0__': 2} synth_fill_block_0: type: synth_fill synth_head_block_0: type: synth_head branch_value_table: {0: 'branch_region_4', 2: 'branch_region_4', 1: 'branch_region_5'} # noqa - variable: control_var_0 + variable: __scfg_control_var_0__ synth_tail_block_0: type: synth_tail tail_region_0: diff --git a/numba_rvsdg/tests/test_rendering.py b/numba_rvsdg/tests/test_rendering.py index 1149fae..7f21d63 100644 --- a/numba_rvsdg/tests/test_rendering.py +++ b/numba_rvsdg/tests/test_rendering.py @@ -48,7 +48,7 @@ color=green label="branch_region_0 jump targets: ('tail_region_0',) back edges: ()" - synth_asign_block_0 [label="synth_asign_block_0\lcontrol_var_0 = 0 + synth_asign_block_0 [label="synth_asign_block_0\l__scfg_control_var_0__ = 0 jump targets: ('tail_region_0',) back edges: ()" shape=rect] } @@ -56,7 +56,7 @@ color=green label="branch_region_1 jump targets: ('tail_region_0',) back edges: ()" - synth_asign_block_1 [label="synth_asign_block_1\lcontrol_var_0 = 1 + synth_asign_block_1 [label="synth_asign_block_1\l__scfg_control_var_0__ = 1 jump targets: ('tail_region_0',) back edges: ()" shape=rect] } @@ -75,7 +75,7 @@ color=red label="head_region_1 jump targets: ('branch_region_2', 'branch_region_3') back edges: ()" - synth_head_block_0 [label="synth_head_block_0\lvariable: control_var_0\l0=>branch_region_2\l1=>branch_region_3 + synth_head_block_0 [label="synth_head_block_0\lvariable: __scfg_control_var_0__\l0=>branch_region_2\l1=>branch_region_3 jump targets: ('branch_region_2', 'branch_region_3') back edges: ()" shape=rect] } @@ -98,7 +98,7 @@ color=green label="branch_region_4 jump targets: ('tail_region_2',) back edges: ()" - synth_asign_block_2 [label="synth_asign_block_2\lbackedge_var_0 = 0\lcontrol_var_0 = 1 + synth_asign_block_2 [label="synth_asign_block_2\l__scfg_backedge_var_0__ = 0\l__scfg_control_var_0__ = 1 jump targets: ('tail_region_2',) back edges: ()" shape=rect] } @@ -106,7 +106,7 @@ color=green label="branch_region_5 jump targets: ('tail_region_2',) back edges: ()" - synth_asign_block_3 [label="synth_asign_block_3\lbackedge_var_0 = 1 + synth_asign_block_3 [label="synth_asign_block_3\l__scfg_backedge_var_0__ = 1 jump targets: ('tail_region_2',) back edges: ()" shape=rect] } @@ -129,7 +129,7 @@ 4 [label="4\l jump targets: ('synth_asign_block_4',) back edges: ()" shape=rect] - synth_asign_block_4 [label="synth_asign_block_4\lbackedge_var_0 = 0\lcontrol_var_0 = 0 + synth_asign_block_4 [label="synth_asign_block_4\l__scfg_backedge_var_0__ = 0\l__scfg_control_var_0__ = 0 jump targets: ('tail_region_1',) back edges: ()" shape=rect] } @@ -137,7 +137,7 @@ color=purple label="tail_region_1 jump targets: ('5',) back edges: ()" - synth_exit_latch_block_0 [label="synth_exit_latch_block_0\lvariable: backedge_var_0\l1=>5\l0=>head_region_1 + synth_exit_latch_block_0 [label="synth_exit_latch_block_0\lvariable: __scfg_backedge_var_0__\l1=>5\l0=>head_region_1 jump targets: ('5',) back edges: ('head_region_1',)" shape=rect] } From c520a29a9fb750756d32228fdead5f1c3b11e43f Mon Sep 17 00:00:00 2001 From: esc Date: Mon, 3 Jun 2024 18:33:02 +0200 Subject: [PATCH 11/12] fix typo in docs As title --- numba_rvsdg/core/datastructures/ast_transforms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index 6c32500..cd8694c 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -605,7 +605,7 @@ def function(a: int) -> None # Emit header instructions. This first makes a backup of the iteration # target and then checks if the iterator is exhausted and if the loop - # should continue. The '_scfg__sentinel__' is an singleton style + # should continue. The '__scfg__sentinel__' is an singleton style # marker, so it need not be versioned. header_code = textwrap.dedent( From 74b5a0502c5d9a54174267380417be765bece2c2 Mon Sep 17 00:00:00 2001 From: esc Date: Mon, 3 Jun 2024 18:44:29 +0200 Subject: [PATCH 12/12] black the code As title --- numba_rvsdg/core/datastructures/ast_transforms.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/numba_rvsdg/core/datastructures/ast_transforms.py b/numba_rvsdg/core/datastructures/ast_transforms.py index cd8694c..34edea1 100644 --- a/numba_rvsdg/core/datastructures/ast_transforms.py +++ b/numba_rvsdg/core/datastructures/ast_transforms.py @@ -780,9 +780,7 @@ def codegen_view() -> list[Any]: # Synthetic assignments just create Python assignments, one for # each variable.. return [ - ast.Assign( - [ast.Name(t)], ast.Constant(v), lineno=0 - ) + ast.Assign([ast.Name(t)], ast.Constant(v), lineno=0) for t, v in block.variable_assignment.items() ] elif type(block) is SyntheticTail: