From c81f310590cb23c4141419f661f5527b50e5e0d0 Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Sun, 7 Jul 2024 16:08:50 -0700 Subject: [PATCH 01/14] WIP Initial steps towards Python AST compat checking. --- parser/ast_python.py | 102 ++++ parser/src/parser/compat.rs | 943 ++++++++++++++++++++++++++++++++++++ parser/src/parser/mod.rs | 1 + 3 files changed, 1046 insertions(+) create mode 100644 parser/ast_python.py create mode 100644 parser/src/parser/compat.rs diff --git a/parser/ast_python.py b/parser/ast_python.py new file mode 100644 index 00000000..e9720532 --- /dev/null +++ b/parser/ast_python.py @@ -0,0 +1,102 @@ +import sys +import ast +from _ast import AST # Python internals I guess? +import argparse +import pathlib +import codecs +import json + +arg_parser = argparse.ArgumentParser( + description="Parse a Python program to AST." +) +arg_parser.add_argument("--input-file", help="Read and parse input file.") +arg_parser.add_argument("--stdin", action="store_true", help="Read and parse input from stdin.") +arg_parser.add_argument("--type-comments", action="store_true", help="Produce an AST with type comments.") +args = arg_parser.parse_args() + +if args.input_file is not None: + source = pathlib.Path(args.input_file).read_text() +elif args.stdin: + source = sys.stdin.read() +else: + print("Missing input parameter. Please specify one of --input-file or --stdin.", file=sys.stderr) + sys.exit(1) + +# ----- Begin inline dependency ------------------------------------------------------------------- +# https://github.com/YoloSwagTeam/ast2json + +# Copyright (c) 2013, Laurent Peuch +# +# All rights reserved. +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the University of California, Berkeley nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +BUILTIN_PURE = (int, float, bool) +BUILTIN_BYTES = (bytearray, bytes) +BUILTIN_STR = (str) + +def decode_str(value): + return value + +def decode_bytes(value): + try: + return value.decode('utf-8') + except: + return codecs.getencoder('hex_codec')(value)[0].decode('utf-8') + +def ast2json(node): + assert isinstance(node, AST) + to_return = dict() + to_return["_type"] = node.__class__.__name__ + for attr in dir(node): + if attr.startswith("_") or attr == "n" or attr == "s": + continue + to_return[attr] = get_value(getattr(node, attr)) + return to_return + +def get_value(attr_value): + if attr_value is None: + return attr_value + if isinstance(attr_value, BUILTIN_PURE): + return attr_value + if isinstance(attr_value, BUILTIN_BYTES): + return decode_bytes(attr_value) + if isinstance(attr_value, BUILTIN_STR): + return decode_str(attr_value) + if isinstance(attr_value, complex): + return str(attr_value) + if isinstance(attr_value, list): + return [get_value(x) for x in attr_value] + if isinstance(attr_value, AST): + return ast2json(attr_value) + if isinstance(attr_value, type(Ellipsis)): + return "..." + else: + raise Exception("Unknown case for '%s' of type '%s'" % (attr_value, type(attr_value))) + +# -------------------------------------------------------------------- End inline dependency ------ + + +tree = ast.parse(source, filename=args.input_file or "stdin", mode="exec", type_comments=args.type_comments) +tree_json = ast2json(tree) +print(json.dumps(tree_json, indent=4)) diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs new file mode 100644 index 00000000..327316da --- /dev/null +++ b/parser/src/parser/compat.rs @@ -0,0 +1,943 @@ +use std::io::Write; +use std::convert::From; +// use serde::{Deserialize, Serialize}; +use miette::{bail, IntoDiagnostic, Result}; +use serde_json::{json, Value}; +// use enderpy_python_ast::*; +// use enderpy_python_ast::visitor::TraversalVisitor; + +use crate::Parser; +use crate::ast::*; +use crate::runpython::{default_python_path, spawn_python_script_command}; + +type ToRowColFn = dyn Fn(u32) -> (u32, u32); + +fn parse_python_source(source: &str) -> Result { + let mut process = spawn_python_script_command( + "parser/ast_python.py", + vec!["--stdin"], + default_python_path()?, + )?; + + // Get process stdin and write the input string. + if let Some(mut stdin) = process.stdin.take() { + stdin.write_all(source.as_bytes()).into_diagnostic()?; + } else { + bail!("Failed to open stdin when running `parser/ast_python.py`"); + } + // Get process stdout and parse result. + let output = process.wait_with_output().into_diagnostic()?; + let ast: Value = serde_json::from_str(String::from_utf8_lossy(&output.stdout).as_ref()).into_diagnostic()?; + Ok(ast) +} + +fn parse_enderpy_source(source: &str) -> Result { + let mut parser = Parser::new(source, "string"); + let typed_ast = parser.parse().into_diagnostic()?; + todo!(); +} + +trait IntoPythonCompat { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value; +} + +trait IntoNullablePythonCompat { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value; +} + +impl IntoNullablePythonCompat for Option { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + if let Some(v) = self { + v.into_python_compat(to_row_col) + } else { + json!(null) + } + } +} + +macro_rules! json_python_compat_node { + ($name:literal, $instance:ident, $to_row_col:ident, $other_fields:tt) => { + { + let mut node = json!($other_fields); + let (start_row, start_col) = $to_row_col($instance.node.start); + let (end_row, end_col) = $to_row_col($instance.node.end); + node["_type"] = json!($name); + node["lineno"] = json!(start_row); + node["col_offset"] = json!(start_col); + node["end_lineno"] = json!(end_row); + node["end_col_offset"] = json!(end_col); + node + } + }; +} + +impl IntoPythonCompat for Module { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json!({ + "_type": "Module", + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Statement { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + match self { + Statement::ExpressionStatement(e) => e.into_python_compat(to_row_col), + Statement::Import(i) => i.into_python_compat(to_row_col), + Statement::ImportFrom(i) => i.into_python_compat(to_row_col), + Statement::AssignStatement(a) => a.into_python_compat(to_row_col), + Statement::AnnAssignStatement(a) => a.into_python_compat(to_row_col), + Statement::AugAssignStatement(a) => a.into_python_compat(to_row_col), + Statement::Assert(a) => a.into_python_compat(to_row_col), + Statement::Pass(p) => p.into_python_compat(to_row_col), + Statement::Delete(d) => d.into_python_compat(to_row_col), + Statement::Return(r) => r.into_python_compat(to_row_col), + Statement::Raise(r) => r.into_python_compat(to_row_col), + Statement::Break(b) => b.into_python_compat(to_row_col), + Statement::Continue(c) => c.into_python_compat(to_row_col), + Statement::Global(g) => g.into_python_compat(to_row_col), + Statement::Nonlocal(n) => n.into_python_compat(to_row_col), + Statement::IfStatement(i) => i.into_python_compat(to_row_col), + Statement::WhileStatement(w) => w.into_python_compat(to_row_col), + Statement::ForStatement(f) => f.into_python_compat(to_row_col), + Statement::WithStatement(w) => w.into_python_compat(to_row_col), + Statement::TryStatement(t) => t.into_python_compat(to_row_col), + Statement::TryStarStatement(t) => t.into_python_compat(to_row_col), + Statement::FunctionDef(f) => f.into_python_compat(to_row_col), + Statement::ClassDef(c) => c.into_python_compat(to_row_col), + Statement::Match(m) => m.into_python_compat(to_row_col), + Statement::AsyncForStatement(f) => f.into_python_compat(to_row_col), + Statement::AsyncWithStatement(w) => w.into_python_compat(to_row_col), + Statement::AsyncFunctionDef(f) => f.into_python_compat(to_row_col), + Statement::TypeAlias(t) => t.into_python_compat(to_row_col), + } + } +} + +impl IntoPythonCompat for Assign { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Assign", self, to_row_col, { + "targets": self.targets.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for AnnAssign { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("AnnAssign", self, to_row_col, { + "target": self.target.into_python_compat(to_row_col), + "annotation": self.annotation.into_python_compat(to_row_col), + "value": self.value.into_python_compat(to_row_col), + "simple": self.simple, + }) + } +} + +impl IntoPythonCompat for AugAssign { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("AugAssign", self, to_row_col, { + "target": self.target.into_python_compat(to_row_col), + "op": self.op.into_python_compat(to_row_col), + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for AugAssignOp { + fn into_python_compat(&self, _: &ToRowColFn) -> Value { + match self { + AugAssignOp::Add => json!({"_type": "Add"}), + AugAssignOp::Sub => json!({"_type": "Sub"}), + AugAssignOp::Mult => json!({"_type": "Mult"}), + AugAssignOp::MatMult => json!({"_type": "MatMult"}), + AugAssignOp::Div => json!({"_type": "Div"}), + AugAssignOp::Mod => json!({"_type": "Mod"}), + AugAssignOp::Pow => json!({"_type": "Pow"}), + AugAssignOp::LShift => json!({"_type": "LShift"}), + AugAssignOp::RShift => json!({"_type": "RShift"}), + AugAssignOp::BitOr => json!({"_type": "BitOr"}), + AugAssignOp::BitXor => json!({"_type": "BitXor"}), + AugAssignOp::BitAnd => json!({"_type": "BitAnd"}), + AugAssignOp::FloorDiv => json!({"_type": "FloorDiv"}), + } + } +} + +impl IntoPythonCompat for Assert { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Assert", self, to_row_col, { + "test": self.test.into_python_compat(to_row_col), + "msg": self.msg.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Pass { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Pass", self, to_row_col, {}) + } +} + +impl IntoPythonCompat for Delete { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Delete", self, to_row_col, { + "targets": self.targets.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Return { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Return", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Raise { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Raise", self, to_row_col, { + "exc": self.exc.into_python_compat(to_row_col), + "cause": self.cause.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Break { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Break", self, to_row_col, {}) + } +} + +impl IntoPythonCompat for Continue { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Continue", self, to_row_col, {}) + } +} + +impl IntoPythonCompat for Import { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Import", self, to_row_col, { + "names": self.names.iter().map(|alias| alias.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Alias { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Alias", self, to_row_col, { + "name": self.name, + "asname": self.asname, + }) + } +} + +impl IntoPythonCompat for ImportFrom { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("ImportFrom", self, to_row_col, { + "module": self.module, + "names": self.names.iter().map(|alias| alias.into_python_compat(to_row_col)).collect::>(), + "level": self.level, + }) + } +} + +impl IntoPythonCompat for Global { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Global", self, to_row_col, { + "names": self.names, + }) + } +} + +impl IntoPythonCompat for Nonlocal { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Nonlocal", self, to_row_col, { + "names": self.names, + }) + } +} + +impl IntoPythonCompat for Expression { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + match self { + Expression::Constant(c) => c.into_python_compat(to_row_col), + Expression::List(l) => l.into_python_compat(to_row_col), + Expression::Tuple(t) => t.into_python_compat(to_row_col), + Expression::Dict(d) => d.into_python_compat(to_row_col), + Expression::Set(s) => s.into_python_compat(to_row_col), + Expression::Name(n) => n.into_python_compat(to_row_col), + Expression::BoolOp(b) => b.into_python_compat(to_row_col), + Expression::UnaryOp(u) => u.into_python_compat(to_row_col), + Expression::BinOp(b) => b.into_python_compat(to_row_col), + Expression::NamedExpr(n) => n.into_python_compat(to_row_col), + Expression::Yield(y) => y.into_python_compat(to_row_col), + Expression::YieldFrom(y) => y.into_python_compat(to_row_col), + Expression::Starred(s) => s.into_python_compat(to_row_col), + Expression::Generator(g) => g.into_python_compat(to_row_col), + Expression::ListComp(l) => l.into_python_compat(to_row_col), + Expression::SetComp(s) => s.into_python_compat(to_row_col), + Expression::DictComp(d) => d.into_python_compat(to_row_col), + Expression::Attribute(a) => a.into_python_compat(to_row_col), + Expression::Subscript(s) => s.into_python_compat(to_row_col), + Expression::Slice(s) => s.into_python_compat(to_row_col), + Expression::Call(c) => c.into_python_compat(to_row_col), + Expression::Await(a) => a.into_python_compat(to_row_col), + Expression::Compare(c) => c.into_python_compat(to_row_col), + Expression::Lambda(l) => l.into_python_compat(to_row_col), + Expression::IfExp(i) => i.into_python_compat(to_row_col), + Expression::JoinedStr(j) => j.into_python_compat(to_row_col), + Expression::FormattedValue(f) => f.into_python_compat(to_row_col), + } + } +} + +impl IntoPythonCompat for Name { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Name", self, to_row_col, { + "id": self.id, + }) + } +} + +impl IntoPythonCompat for Constant { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Constant", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for ConstantValue { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + match self { + ConstantValue::None => json!(null), + ConstantValue::Ellipsis => json!("..."), + ConstantValue::Bool(v) => json!(v), + ConstantValue::Str(v) => json!(v), + ConstantValue::Bytes(v) => json!(v), + ConstantValue::Tuple(v) => json!(v.iter().map(|cons| cons.into_python_compat(to_row_col)).collect::>()), + ConstantValue::Int(v) => json!(v), + ConstantValue::Float(v) => json!(v), + ConstantValue::Complex { real, imaginary } => json!({"real": real, "imaginary": imaginary}), + } + } +} + +impl IntoPythonCompat for List { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("List", self, to_row_col, { + "elements": self.elements.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Tuple { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Tuple", self, to_row_col, { + "elements": self.elements.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Dict { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Dict", self, to_row_col, { + "keys": self.keys.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "values": self.values.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Set { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Set", self, to_row_col, { + "elements": self.elements.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for BoolOperation { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("BoolOperation", self, to_row_col, { + "op": self.op.into_python_compat(to_row_col), + "values": self.values.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for BooleanOperator { + fn into_python_compat(&self, _: &ToRowColFn) -> Value { + match self { + BooleanOperator::And => json!({"_type": "And"}), + BooleanOperator::Or => json!({"_type": "Or"}), + } + } +} + +impl IntoPythonCompat for UnaryOperation { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("UnaryOperation", self, to_row_col, { + "op": self.op.into_python_compat(to_row_col), + "operand": self.operand.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for UnaryOperator { + fn into_python_compat(&self, _: &ToRowColFn) -> Value { + match self { + UnaryOperator::Not => json!({"_type": "Not"}), + UnaryOperator::Invert => json!({"_type": "Invert"}), + UnaryOperator::UAdd => json!({"_type": "UAdd"}), + UnaryOperator::USub => json!({"_type": "USub"}), + } + } +} + +impl IntoPythonCompat for BinOp { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("BinOp", self, to_row_col, { + "op": self.op.into_python_compat(to_row_col), + "left": self.left.into_python_compat(to_row_col), + "right": self.right.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for BinaryOperator { + fn into_python_compat(&self, _: &ToRowColFn) -> Value { + match self { + BinaryOperator::Add => json!({"_type": "Add"}), + BinaryOperator::Sub => json!({"_type": "Sub"}), + BinaryOperator::Mult => json!({"_type": "Mult"}), + BinaryOperator::MatMult => json!({"_type": "MatMult"}), + BinaryOperator::Div => json!({"_type": "Div"}), + BinaryOperator::Mod => json!({"_type": "Mod"}), + BinaryOperator::Pow => json!({"_type": "Pow"}), + BinaryOperator::LShift => json!({"_type": "LShift"}), + BinaryOperator::RShift => json!({"_type": "RShift"}), + BinaryOperator::BitOr => json!({"_type": "BitOr"}), + BinaryOperator::BitXor => json!({"_type": "BitXor"}), + BinaryOperator::BitAnd => json!({"_type": "BitAnd"}), + BinaryOperator::FloorDiv => json!({"_type": "FloorDiv"}), + } + } +} + +impl IntoPythonCompat for NamedExpression { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("NamedExpression", self, to_row_col, { + "target": self.target.into_python_compat(to_row_col), + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Yield { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Yield", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for YieldFrom { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("YieldForm", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Starred { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Starred", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Generator { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Generator", self, to_row_col, { + "element": self.element.into_python_compat(to_row_col), + "generators": self.generators.iter().map(|gen| gen.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for ListComp { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("ListComp", self, to_row_col, { + "element": self.element.into_python_compat(to_row_col), + "generators": self.generators.iter().map(|gen| gen.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for SetComp { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("SetComp", self, to_row_col, { + "element": self.element.into_python_compat(to_row_col), + "generators": self.generators.iter().map(|gen| gen.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for DictComp { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("DictComp", self, to_row_col, { + "key": self.key.into_python_compat(to_row_col), + "value": self.value.into_python_compat(to_row_col), + "generators": self.generators.iter().map(|gen| gen.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Comprehension { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Comprehension", self, to_row_col, { + "target": self.target.into_python_compat(to_row_col), + "iter": self.iter.into_python_compat(to_row_col), + "ifs": self.ifs.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "is_async": self.is_async, + }) + } +} + +impl IntoPythonCompat for Attribute { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Attribute", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + "attr": self.attr, + }) + } +} + +impl IntoPythonCompat for Subscript { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Subscript", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + "slice": self.slice.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Slice { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Slice", self, to_row_col, { + "lower": self.lower.into_python_compat(to_row_col), + "upper": self.upper.into_python_compat(to_row_col), + "step": self.step.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Call { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Call", self, to_row_col, { + "func": self.func.into_python_compat(to_row_col), + "args": self.args.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "keywords": self.keywords.iter().map(|kw| kw.into_python_compat(to_row_col)).collect::>(), + "starargs": self.starargs.into_python_compat(to_row_col), + "kwargs": self.kwargs.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Keyword { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Keyword", self, to_row_col, { + "arg": self.arg.as_ref().map_or(json!(null), |s| json!(s)), + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Await { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Await", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Compare { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Compar", self, to_row_col, { + "left": self.left.into_python_compat(to_row_col), + "ops": self.ops.iter().map(|op| op.into_python_compat(to_row_col)).collect::>(), + "comparators": self.comparators.iter().map(|op| op.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for ComparisonOperator { + fn into_python_compat(&self, _: &ToRowColFn) -> Value { + match self { + ComparisonOperator::Eq => json!({"_type": "Eq"}), + ComparisonOperator::NotEq => json!({"_type": "NotEq"}), + ComparisonOperator::Lt => json!({"_type": "Lt"}), + ComparisonOperator::LtE => json!({"_type": "LtE"}), + ComparisonOperator::Gt => json!({"_type": "Gt"}), + ComparisonOperator::GtE => json!({"_type": "GtE"}), + ComparisonOperator::Is => json!({"_type": "Is"}), + ComparisonOperator::IsNot => json!({"_type": "IsNot"}), + ComparisonOperator::In => json!({"_type": "In"}), + ComparisonOperator::NotIn => json!({"_type": "NotIn"}), + } + } +} + +impl IntoPythonCompat for Lambda { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Lambda", self, to_row_col, { + "args": self.args.into_python_compat(to_row_col), + "body": self.body.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for crate::ast::Arguments { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Arguments", self, to_row_col, { + "posonlyargs": self.posonlyargs.iter().map(|arg| arg.into_python_compat(to_row_col)).collect::>(), + "args": self.args.iter().map(|arg| arg.into_python_compat(to_row_col)).collect::>(), + "vararg": self.vararg.into_python_compat(to_row_col), + "kwonlyargs": self.kwonlyargs.iter().map(|arg| arg.into_python_compat(to_row_col)).collect::>(), + "kw_defaults": self.kw_defaults.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "kwarg": self.kwarg.into_python_compat(to_row_col), + "defaults": self.defaults.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Arg { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Arg", self, to_row_col, { + "arg": self.arg, + "annotation": self.annotation.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for IfExp { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("IfExp", self, to_row_col, { + "test": self.test.into_python_compat(to_row_col), + "body": self.body.into_python_compat(to_row_col), + "orelse": self.orelse.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for FormattedValue { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("FormattedValue", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + "conversion": self.conversion, + "format_spec": self.format_spec.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for JoinedStr { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("JoinedStr", self, to_row_col, { + "values": self.values.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for If { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("If", self, to_row_col, { + "test": self.test.into_python_compat(to_row_col), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for While { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("While", self, to_row_col, { + "test": self.test.into_python_compat(to_row_col), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for For { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("For", self, to_row_col, { + "target": self.target.into_python_compat(to_row_col), + "iter": self.iter.into_python_compat(to_row_col), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for AsyncFor { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("AsyncFor", self, to_row_col, { + "target": self.target.into_python_compat(to_row_col), + "iter": self.iter.into_python_compat(to_row_col), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for With { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("With", self, to_row_col, { + "items": self.items.iter().map(|wi| wi.into_python_compat(to_row_col)).collect::>(), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for AsyncWith { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("AsyncWith", self, to_row_col, { + "items": self.items.iter().map(|wi| wi.into_python_compat(to_row_col)).collect::>(), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for WithItem { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("WithItem", self, to_row_col, { + "context_expr": self.context_expr.into_python_compat(to_row_col), + "optional_vars": self.optional_vars.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for Try { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Try", self, to_row_col, { + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "handlers": self.handlers.iter().map(|hndl| hndl.into_python_compat(to_row_col)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "finalbody": self.finalbody.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for TryStar { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("TryStar", self, to_row_col, { + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "handlers": self.handlers.iter().map(|hndl| hndl.into_python_compat(to_row_col)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "finalbody": self.finalbody.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for ExceptHandler { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("ExceptHandler", self, to_row_col, { + "type": self.typ.into_python_compat(to_row_col), + "name": self.name.as_ref().map_or(json!(null), |s| json!(s)), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for FunctionDef { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("FunctionDef", self, to_row_col, { + "name": self.name, + "args": self.args.into_python_compat(to_row_col), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "decorator_list": self.decorator_list.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "returns": self.returns.into_python_compat(to_row_col), + "type_comment": self.type_comment.as_ref().map_or(json!(null), |s| json!(s)), + "type_params": self.type_params.iter().map(|tp| tp.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for AsyncFunctionDef { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("AsyncFunctionDef", self, to_row_col, { + "name": self.name, + "args": self.args.into_python_compat(to_row_col), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "decorator_list": self.decorator_list.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "returns": self.returns.into_python_compat(to_row_col), + "type_comment": self.type_comment.as_ref().map_or(json!(null), |s| json!(s)), + "type_params": self.type_params.iter().map(|tp| tp.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for ClassDef { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("ClassDef", self, to_row_col, { + "name": self.name, + "bases": self.bases.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "keywords": self.keywords.iter().map(|kw| kw.into_python_compat(to_row_col)).collect::>(), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "decorator_list": self.decorator_list.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "type_params": self.type_params.iter().map(|tp| tp.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for Match { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("Match", self, to_row_col, { + "subject": self.subject.into_python_compat(to_row_col), + "cases": self.cases.iter().map(|mc| mc.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for MatchCase { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("MatchCase", self, to_row_col, { + "pattern": self.pattern.into_python_compat(to_row_col), + "guard": self.guard.into_python_compat(to_row_col), + "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for MatchPattern { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + match self { + MatchPattern::MatchValue(val) => val.into_python_compat(to_row_col), + MatchPattern::MatchSingleton(expr) => expr.into_python_compat(to_row_col), + MatchPattern::MatchSequence(pats) => json!(pats.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>()), + MatchPattern::MatchStar(expr) => expr.into_python_compat(to_row_col), + MatchPattern::MatchMapping(map) => map.into_python_compat(to_row_col), + MatchPattern::MatchAs(mas) => mas.into_python_compat(to_row_col), + MatchPattern::MatchClass(cls) => cls.into_python_compat(to_row_col), + MatchPattern::MatchOr(pats) => json!(pats.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>()), + } + } +} + +impl IntoPythonCompat for MatchValue { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("MatchValue", self, to_row_col, { + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for MatchAs { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("MatchAs", self, to_row_col, { + "name": self.name.as_ref().map_or(json!(null), |s| json!(s)), + "pattern": self.pattern.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for MatchMapping { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("MatchMapping", self, to_row_col, { + "keys": self.keys.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), + "patterns": self.patterns.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>(), + "rest": self.rest.as_ref().map_or(json!(null), |s| json!(s)), + }) + } +} + +impl IntoPythonCompat for MatchClass { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("MatchClass", self, to_row_col, { + "cls": self.cls.into_python_compat(to_row_col), + "patterns": self.patterns.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>(), + "kwd_attrs": self.kwd_attrs, + "kwd_patterns": self.kwd_patterns.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>(), + }) + } +} + +impl IntoPythonCompat for TypeParam { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + match self { + TypeParam::TypeVar(var) => var.into_python_compat(to_row_col), + TypeParam::ParamSpec(spec) => spec.into_python_compat(to_row_col), + TypeParam::TypeVarTuple(tup) => tup.into_python_compat(to_row_col), + } + } +} + +impl IntoPythonCompat for TypeVar { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("TypeVar", self, to_row_col, { + "name": self.name, + "bound": self.bound.into_python_compat(to_row_col), + }) + } +} + +impl IntoPythonCompat for ParamSpec { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("ParamSpec", self, to_row_col, { + "name": self.name, + }) + } +} + +impl IntoPythonCompat for TypeVarTuple { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("TypeVarTuple", self, to_row_col, { + "name": self.name, + }) + } +} + +impl IntoPythonCompat for TypeAlias { + fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { + json_python_compat_node!("TypeAlias", self, to_row_col, { + "name": self.name, + "type_params": self.type_params.iter().map(|tp| tp.into_python_compat(to_row_col)).collect::>(), + "value": self.value.into_python_compat(to_row_col), + }) + } +} + +#[cfg(test)] +mod tests { + use serde_json::Value; + use miette::{bail, IntoDiagnostic, Result}; + use crate::Parser; + use crate::ast::*; + + use super::parse_python_source; + + #[test] + fn test_simple_compat() { + let source = r#" +a: int = 1 +print(a) +"#; + let mut parser = Parser::new(source, "string"); + let enderpy_ast = parser.parse().into_diagnostic().unwrap(); + let python_ast = parse_python_source(source).unwrap(); + + assert_ast_eq(python_ast, enderpy_ast); + + // let mut lexer = Lexer::new(source); + // let enderpy_tokens = lexer.lex(); + // let python_tokens = lex_python_source(source).unwrap(); + // assert_tokens_eq(python_tokens, enderpy_tokens, &lexer); + } + + fn assert_ast_eq(python_ast: Value, enderpy_ast: Module) { + } +} diff --git a/parser/src/parser/mod.rs b/parser/src/parser/mod.rs index a6c6106d..a3f34954 100644 --- a/parser/src/parser/mod.rs +++ b/parser/src/parser/mod.rs @@ -1,4 +1,5 @@ pub mod ast; +pub mod compat; #[allow(clippy::module_inception)] pub mod parser; use crate::token::{Kind, Token}; From 14b345cc8bbb58055b194ed0084eea431f4125aa Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Sun, 7 Jul 2024 16:41:53 -0700 Subject: [PATCH 02/14] A touch of refactoring to appease borrow checker + clippy. --- parser/src/lexer/mod.rs | 2 +- parser/src/parser/compat.rs | 914 ++++++++++++++++++------------------ parser/src/parser/parser.rs | 4 + 3 files changed, 457 insertions(+), 463 deletions(-) diff --git a/parser/src/lexer/mod.rs b/parser/src/lexer/mod.rs index ed982c15..df4ee883 100644 --- a/parser/src/lexer/mod.rs +++ b/parser/src/lexer/mod.rs @@ -1049,7 +1049,7 @@ impl<'a> Lexer<'a> { count } - fn to_row_col(&self, source_offset: u32) -> (u32, u32) { + pub fn to_row_col(&self, source_offset: u32) -> (u32, u32) { let (line_row, line_offset) = match self.line_starts.binary_search(&source_offset) { Ok(idx) => (idx, self.line_starts[idx]), Err(idx) => (idx - 1, self.line_starts[idx - 1]), diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index 327316da..9aaadea3 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -1,16 +1,13 @@ use std::io::Write; use std::convert::From; -// use serde::{Deserialize, Serialize}; use miette::{bail, IntoDiagnostic, Result}; use serde_json::{json, Value}; -// use enderpy_python_ast::*; -// use enderpy_python_ast::visitor::TraversalVisitor; use crate::Parser; use crate::ast::*; use crate::runpython::{default_python_path, spawn_python_script_command}; -type ToRowColFn = dyn Fn(u32) -> (u32, u32); +// type Parser = dyn Fn(u32) -> (u32, u32); fn parse_python_source(source: &str) -> Result { let mut process = spawn_python_script_command( @@ -34,21 +31,22 @@ fn parse_python_source(source: &str) -> Result { fn parse_enderpy_source(source: &str) -> Result { let mut parser = Parser::new(source, "string"); let typed_ast = parser.parse().into_diagnostic()?; - todo!(); + let ast = typed_ast.as_python_compat(&parser); + Ok(ast) } -trait IntoPythonCompat { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value; +trait AsPythonCompat { + fn as_python_compat(&self, parser: &Parser) -> Value; } -trait IntoNullablePythonCompat { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value; +trait AsNullablePythonCompat { + fn as_python_compat(&self, parser: &Parser) -> Value; } -impl IntoNullablePythonCompat for Option { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { +impl AsNullablePythonCompat for Option { + fn as_python_compat(&self, parser: &Parser) -> Value { if let Some(v) = self { - v.into_python_compat(to_row_col) + v.as_python_compat(parser) } else { json!(null) } @@ -56,11 +54,11 @@ impl IntoNullablePythonCompat for Option { } macro_rules! json_python_compat_node { - ($name:literal, $instance:ident, $to_row_col:ident, $other_fields:tt) => { + ($name:literal, $instance:ident, $parser:ident, $other_fields:tt) => { { let mut node = json!($other_fields); - let (start_row, start_col) = $to_row_col($instance.node.start); - let (end_row, end_col) = $to_row_col($instance.node.end); + let (start_row, start_col) = $parser.to_row_col($instance.node.start); + let (end_row, end_col) = $parser.to_row_col($instance.node.end); node["_type"] = json!($name); node["lineno"] = json!(start_row); node["col_offset"] = json!(start_col); @@ -71,82 +69,82 @@ macro_rules! json_python_compat_node { }; } -impl IntoPythonCompat for Module { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { +impl AsPythonCompat for Module { + fn as_python_compat(&self, parser: &Parser) -> Value { json!({ "_type": "Module", - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Statement { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { +impl AsPythonCompat for Statement { + fn as_python_compat(&self, parser: &Parser) -> Value { match self { - Statement::ExpressionStatement(e) => e.into_python_compat(to_row_col), - Statement::Import(i) => i.into_python_compat(to_row_col), - Statement::ImportFrom(i) => i.into_python_compat(to_row_col), - Statement::AssignStatement(a) => a.into_python_compat(to_row_col), - Statement::AnnAssignStatement(a) => a.into_python_compat(to_row_col), - Statement::AugAssignStatement(a) => a.into_python_compat(to_row_col), - Statement::Assert(a) => a.into_python_compat(to_row_col), - Statement::Pass(p) => p.into_python_compat(to_row_col), - Statement::Delete(d) => d.into_python_compat(to_row_col), - Statement::Return(r) => r.into_python_compat(to_row_col), - Statement::Raise(r) => r.into_python_compat(to_row_col), - Statement::Break(b) => b.into_python_compat(to_row_col), - Statement::Continue(c) => c.into_python_compat(to_row_col), - Statement::Global(g) => g.into_python_compat(to_row_col), - Statement::Nonlocal(n) => n.into_python_compat(to_row_col), - Statement::IfStatement(i) => i.into_python_compat(to_row_col), - Statement::WhileStatement(w) => w.into_python_compat(to_row_col), - Statement::ForStatement(f) => f.into_python_compat(to_row_col), - Statement::WithStatement(w) => w.into_python_compat(to_row_col), - Statement::TryStatement(t) => t.into_python_compat(to_row_col), - Statement::TryStarStatement(t) => t.into_python_compat(to_row_col), - Statement::FunctionDef(f) => f.into_python_compat(to_row_col), - Statement::ClassDef(c) => c.into_python_compat(to_row_col), - Statement::Match(m) => m.into_python_compat(to_row_col), - Statement::AsyncForStatement(f) => f.into_python_compat(to_row_col), - Statement::AsyncWithStatement(w) => w.into_python_compat(to_row_col), - Statement::AsyncFunctionDef(f) => f.into_python_compat(to_row_col), - Statement::TypeAlias(t) => t.into_python_compat(to_row_col), + Statement::ExpressionStatement(e) => e.as_python_compat(parser), + Statement::Import(i) => i.as_python_compat(parser), + Statement::ImportFrom(i) => i.as_python_compat(parser), + Statement::AssignStatement(a) => a.as_python_compat(parser), + Statement::AnnAssignStatement(a) => a.as_python_compat(parser), + Statement::AugAssignStatement(a) => a.as_python_compat(parser), + Statement::Assert(a) => a.as_python_compat(parser), + Statement::Pass(p) => p.as_python_compat(parser), + Statement::Delete(d) => d.as_python_compat(parser), + Statement::Return(r) => r.as_python_compat(parser), + Statement::Raise(r) => r.as_python_compat(parser), + Statement::Break(b) => b.as_python_compat(parser), + Statement::Continue(c) => c.as_python_compat(parser), + Statement::Global(g) => g.as_python_compat(parser), + Statement::Nonlocal(n) => n.as_python_compat(parser), + Statement::IfStatement(i) => i.as_python_compat(parser), + Statement::WhileStatement(w) => w.as_python_compat(parser), + Statement::ForStatement(f) => f.as_python_compat(parser), + Statement::WithStatement(w) => w.as_python_compat(parser), + Statement::TryStatement(t) => t.as_python_compat(parser), + Statement::TryStarStatement(t) => t.as_python_compat(parser), + Statement::FunctionDef(f) => f.as_python_compat(parser), + Statement::ClassDef(c) => c.as_python_compat(parser), + Statement::Match(m) => m.as_python_compat(parser), + Statement::AsyncForStatement(f) => f.as_python_compat(parser), + Statement::AsyncWithStatement(w) => w.as_python_compat(parser), + Statement::AsyncFunctionDef(f) => f.as_python_compat(parser), + Statement::TypeAlias(t) => t.as_python_compat(parser), } } } -impl IntoPythonCompat for Assign { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Assign", self, to_row_col, { - "targets": self.targets.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for Assign { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Assign", self, parser, { + "targets": self.targets.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for AnnAssign { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("AnnAssign", self, to_row_col, { - "target": self.target.into_python_compat(to_row_col), - "annotation": self.annotation.into_python_compat(to_row_col), - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for AnnAssign { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("AnnAssign", self, parser, { + "target": self.target.as_python_compat(parser), + "annotation": self.annotation.as_python_compat(parser), + "value": self.value.as_python_compat(parser), "simple": self.simple, }) } } -impl IntoPythonCompat for AugAssign { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("AugAssign", self, to_row_col, { - "target": self.target.into_python_compat(to_row_col), - "op": self.op.into_python_compat(to_row_col), - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for AugAssign { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("AugAssign", self, parser, { + "target": self.target.as_python_compat(parser), + "op": self.op.as_python_compat(parser), + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for AugAssignOp { - fn into_python_compat(&self, _: &ToRowColFn) -> Value { +impl AsPythonCompat for AugAssignOp { + fn as_python_compat(&self, _: &Parser) -> Value { match self { AugAssignOp::Add => json!({"_type": "Add"}), AugAssignOp::Sub => json!({"_type": "Sub"}), @@ -165,160 +163,160 @@ impl IntoPythonCompat for AugAssignOp { } } -impl IntoPythonCompat for Assert { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Assert", self, to_row_col, { - "test": self.test.into_python_compat(to_row_col), - "msg": self.msg.into_python_compat(to_row_col), +impl AsPythonCompat for Assert { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Assert", self, parser, { + "test": self.test.as_python_compat(parser), + "msg": self.msg.as_python_compat(parser), }) } } -impl IntoPythonCompat for Pass { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Pass", self, to_row_col, {}) +impl AsPythonCompat for Pass { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Pass", self, parser, {}) } } -impl IntoPythonCompat for Delete { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Delete", self, to_row_col, { - "targets": self.targets.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Delete { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Delete", self, parser, { + "targets": self.targets.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Return { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Return", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for Return { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Return", self, parser, { + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for Raise { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Raise", self, to_row_col, { - "exc": self.exc.into_python_compat(to_row_col), - "cause": self.cause.into_python_compat(to_row_col), +impl AsPythonCompat for Raise { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Raise", self, parser, { + "exc": self.exc.as_python_compat(parser), + "cause": self.cause.as_python_compat(parser), }) } } -impl IntoPythonCompat for Break { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Break", self, to_row_col, {}) +impl AsPythonCompat for Break { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Break", self, parser, {}) } } -impl IntoPythonCompat for Continue { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Continue", self, to_row_col, {}) +impl AsPythonCompat for Continue { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Continue", self, parser, {}) } } -impl IntoPythonCompat for Import { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Import", self, to_row_col, { - "names": self.names.iter().map(|alias| alias.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Import { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Import", self, parser, { + "names": self.names.iter().map(|alias| alias.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Alias { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Alias", self, to_row_col, { +impl AsPythonCompat for Alias { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Alias", self, parser, { "name": self.name, "asname": self.asname, }) } } -impl IntoPythonCompat for ImportFrom { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("ImportFrom", self, to_row_col, { +impl AsPythonCompat for ImportFrom { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("ImportFrom", self, parser, { "module": self.module, - "names": self.names.iter().map(|alias| alias.into_python_compat(to_row_col)).collect::>(), + "names": self.names.iter().map(|alias| alias.as_python_compat(parser)).collect::>(), "level": self.level, }) } } -impl IntoPythonCompat for Global { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Global", self, to_row_col, { +impl AsPythonCompat for Global { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Global", self, parser, { "names": self.names, }) } } -impl IntoPythonCompat for Nonlocal { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Nonlocal", self, to_row_col, { +impl AsPythonCompat for Nonlocal { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Nonlocal", self, parser, { "names": self.names, }) } } -impl IntoPythonCompat for Expression { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { +impl AsPythonCompat for Expression { + fn as_python_compat(&self, parser: &Parser) -> Value { match self { - Expression::Constant(c) => c.into_python_compat(to_row_col), - Expression::List(l) => l.into_python_compat(to_row_col), - Expression::Tuple(t) => t.into_python_compat(to_row_col), - Expression::Dict(d) => d.into_python_compat(to_row_col), - Expression::Set(s) => s.into_python_compat(to_row_col), - Expression::Name(n) => n.into_python_compat(to_row_col), - Expression::BoolOp(b) => b.into_python_compat(to_row_col), - Expression::UnaryOp(u) => u.into_python_compat(to_row_col), - Expression::BinOp(b) => b.into_python_compat(to_row_col), - Expression::NamedExpr(n) => n.into_python_compat(to_row_col), - Expression::Yield(y) => y.into_python_compat(to_row_col), - Expression::YieldFrom(y) => y.into_python_compat(to_row_col), - Expression::Starred(s) => s.into_python_compat(to_row_col), - Expression::Generator(g) => g.into_python_compat(to_row_col), - Expression::ListComp(l) => l.into_python_compat(to_row_col), - Expression::SetComp(s) => s.into_python_compat(to_row_col), - Expression::DictComp(d) => d.into_python_compat(to_row_col), - Expression::Attribute(a) => a.into_python_compat(to_row_col), - Expression::Subscript(s) => s.into_python_compat(to_row_col), - Expression::Slice(s) => s.into_python_compat(to_row_col), - Expression::Call(c) => c.into_python_compat(to_row_col), - Expression::Await(a) => a.into_python_compat(to_row_col), - Expression::Compare(c) => c.into_python_compat(to_row_col), - Expression::Lambda(l) => l.into_python_compat(to_row_col), - Expression::IfExp(i) => i.into_python_compat(to_row_col), - Expression::JoinedStr(j) => j.into_python_compat(to_row_col), - Expression::FormattedValue(f) => f.into_python_compat(to_row_col), + Expression::Constant(c) => c.as_python_compat(parser), + Expression::List(l) => l.as_python_compat(parser), + Expression::Tuple(t) => t.as_python_compat(parser), + Expression::Dict(d) => d.as_python_compat(parser), + Expression::Set(s) => s.as_python_compat(parser), + Expression::Name(n) => n.as_python_compat(parser), + Expression::BoolOp(b) => b.as_python_compat(parser), + Expression::UnaryOp(u) => u.as_python_compat(parser), + Expression::BinOp(b) => b.as_python_compat(parser), + Expression::NamedExpr(n) => n.as_python_compat(parser), + Expression::Yield(y) => y.as_python_compat(parser), + Expression::YieldFrom(y) => y.as_python_compat(parser), + Expression::Starred(s) => s.as_python_compat(parser), + Expression::Generator(g) => g.as_python_compat(parser), + Expression::ListComp(l) => l.as_python_compat(parser), + Expression::SetComp(s) => s.as_python_compat(parser), + Expression::DictComp(d) => d.as_python_compat(parser), + Expression::Attribute(a) => a.as_python_compat(parser), + Expression::Subscript(s) => s.as_python_compat(parser), + Expression::Slice(s) => s.as_python_compat(parser), + Expression::Call(c) => c.as_python_compat(parser), + Expression::Await(a) => a.as_python_compat(parser), + Expression::Compare(c) => c.as_python_compat(parser), + Expression::Lambda(l) => l.as_python_compat(parser), + Expression::IfExp(i) => i.as_python_compat(parser), + Expression::JoinedStr(j) => j.as_python_compat(parser), + Expression::FormattedValue(f) => f.as_python_compat(parser), } } } -impl IntoPythonCompat for Name { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Name", self, to_row_col, { +impl AsPythonCompat for Name { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Name", self, parser, { "id": self.id, }) } } -impl IntoPythonCompat for Constant { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Constant", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for Constant { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Constant", self, parser, { + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for ConstantValue { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { +impl AsPythonCompat for ConstantValue { + fn as_python_compat(&self, parser: &Parser) -> Value { match self { ConstantValue::None => json!(null), ConstantValue::Ellipsis => json!("..."), ConstantValue::Bool(v) => json!(v), ConstantValue::Str(v) => json!(v), ConstantValue::Bytes(v) => json!(v), - ConstantValue::Tuple(v) => json!(v.iter().map(|cons| cons.into_python_compat(to_row_col)).collect::>()), + ConstantValue::Tuple(v) => json!(v.iter().map(|cons| cons.as_python_compat(parser)).collect::>()), ConstantValue::Int(v) => json!(v), ConstantValue::Float(v) => json!(v), ConstantValue::Complex { real, imaginary } => json!({"real": real, "imaginary": imaginary}), @@ -326,50 +324,50 @@ impl IntoPythonCompat for ConstantValue { } } -impl IntoPythonCompat for List { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("List", self, to_row_col, { - "elements": self.elements.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for List { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("List", self, parser, { + "elements": self.elements.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Tuple { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Tuple", self, to_row_col, { - "elements": self.elements.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Tuple { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Tuple", self, parser, { + "elements": self.elements.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Dict { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Dict", self, to_row_col, { - "keys": self.keys.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "values": self.values.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Dict { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Dict", self, parser, { + "keys": self.keys.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "values": self.values.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Set { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Set", self, to_row_col, { - "elements": self.elements.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Set { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Set", self, parser, { + "elements": self.elements.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for BoolOperation { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("BoolOperation", self, to_row_col, { - "op": self.op.into_python_compat(to_row_col), - "values": self.values.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for BoolOperation { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("BoolOperation", self, parser, { + "op": self.op.as_python_compat(parser), + "values": self.values.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for BooleanOperator { - fn into_python_compat(&self, _: &ToRowColFn) -> Value { +impl AsPythonCompat for BooleanOperator { + fn as_python_compat(&self, _: &Parser) -> Value { match self { BooleanOperator::And => json!({"_type": "And"}), BooleanOperator::Or => json!({"_type": "Or"}), @@ -377,17 +375,17 @@ impl IntoPythonCompat for BooleanOperator { } } -impl IntoPythonCompat for UnaryOperation { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("UnaryOperation", self, to_row_col, { - "op": self.op.into_python_compat(to_row_col), - "operand": self.operand.into_python_compat(to_row_col), +impl AsPythonCompat for UnaryOperation { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("UnaryOperation", self, parser, { + "op": self.op.as_python_compat(parser), + "operand": self.operand.as_python_compat(parser), }) } } -impl IntoPythonCompat for UnaryOperator { - fn into_python_compat(&self, _: &ToRowColFn) -> Value { +impl AsPythonCompat for UnaryOperator { + fn as_python_compat(&self, _: &Parser) -> Value { match self { UnaryOperator::Not => json!({"_type": "Not"}), UnaryOperator::Invert => json!({"_type": "Invert"}), @@ -397,18 +395,18 @@ impl IntoPythonCompat for UnaryOperator { } } -impl IntoPythonCompat for BinOp { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("BinOp", self, to_row_col, { - "op": self.op.into_python_compat(to_row_col), - "left": self.left.into_python_compat(to_row_col), - "right": self.right.into_python_compat(to_row_col), +impl AsPythonCompat for BinOp { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("BinOp", self, parser, { + "op": self.op.as_python_compat(parser), + "left": self.left.as_python_compat(parser), + "right": self.right.as_python_compat(parser), }) } } -impl IntoPythonCompat for BinaryOperator { - fn into_python_compat(&self, _: &ToRowColFn) -> Value { +impl AsPythonCompat for BinaryOperator { + fn as_python_compat(&self, _: &Parser) -> Value { match self { BinaryOperator::Add => json!({"_type": "Add"}), BinaryOperator::Sub => json!({"_type": "Sub"}), @@ -427,156 +425,156 @@ impl IntoPythonCompat for BinaryOperator { } } -impl IntoPythonCompat for NamedExpression { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("NamedExpression", self, to_row_col, { - "target": self.target.into_python_compat(to_row_col), - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for NamedExpression { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("NamedExpression", self, parser, { + "target": self.target.as_python_compat(parser), + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for Yield { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Yield", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for Yield { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Yield", self, parser, { + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for YieldFrom { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("YieldForm", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for YieldFrom { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("YieldForm", self, parser, { + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for Starred { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Starred", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for Starred { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Starred", self, parser, { + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for Generator { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Generator", self, to_row_col, { - "element": self.element.into_python_compat(to_row_col), - "generators": self.generators.iter().map(|gen| gen.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Generator { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Generator", self, parser, { + "element": self.element.as_python_compat(parser), + "generators": self.generators.iter().map(|gen| gen.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for ListComp { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("ListComp", self, to_row_col, { - "element": self.element.into_python_compat(to_row_col), - "generators": self.generators.iter().map(|gen| gen.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for ListComp { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("ListComp", self, parser, { + "element": self.element.as_python_compat(parser), + "generators": self.generators.iter().map(|gen| gen.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for SetComp { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("SetComp", self, to_row_col, { - "element": self.element.into_python_compat(to_row_col), - "generators": self.generators.iter().map(|gen| gen.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for SetComp { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("SetComp", self, parser, { + "element": self.element.as_python_compat(parser), + "generators": self.generators.iter().map(|gen| gen.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for DictComp { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("DictComp", self, to_row_col, { - "key": self.key.into_python_compat(to_row_col), - "value": self.value.into_python_compat(to_row_col), - "generators": self.generators.iter().map(|gen| gen.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for DictComp { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("DictComp", self, parser, { + "key": self.key.as_python_compat(parser), + "value": self.value.as_python_compat(parser), + "generators": self.generators.iter().map(|gen| gen.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Comprehension { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Comprehension", self, to_row_col, { - "target": self.target.into_python_compat(to_row_col), - "iter": self.iter.into_python_compat(to_row_col), - "ifs": self.ifs.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Comprehension { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Comprehension", self, parser, { + "target": self.target.as_python_compat(parser), + "iter": self.iter.as_python_compat(parser), + "ifs": self.ifs.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), "is_async": self.is_async, }) } } -impl IntoPythonCompat for Attribute { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Attribute", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for Attribute { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Attribute", self, parser, { + "value": self.value.as_python_compat(parser), "attr": self.attr, }) } } -impl IntoPythonCompat for Subscript { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Subscript", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), - "slice": self.slice.into_python_compat(to_row_col), +impl AsPythonCompat for Subscript { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Subscript", self, parser, { + "value": self.value.as_python_compat(parser), + "slice": self.slice.as_python_compat(parser), }) } } -impl IntoPythonCompat for Slice { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Slice", self, to_row_col, { - "lower": self.lower.into_python_compat(to_row_col), - "upper": self.upper.into_python_compat(to_row_col), - "step": self.step.into_python_compat(to_row_col), +impl AsPythonCompat for Slice { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Slice", self, parser, { + "lower": self.lower.as_python_compat(parser), + "upper": self.upper.as_python_compat(parser), + "step": self.step.as_python_compat(parser), }) } } -impl IntoPythonCompat for Call { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Call", self, to_row_col, { - "func": self.func.into_python_compat(to_row_col), - "args": self.args.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "keywords": self.keywords.iter().map(|kw| kw.into_python_compat(to_row_col)).collect::>(), - "starargs": self.starargs.into_python_compat(to_row_col), - "kwargs": self.kwargs.into_python_compat(to_row_col), +impl AsPythonCompat for Call { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Call", self, parser, { + "func": self.func.as_python_compat(parser), + "args": self.args.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "keywords": self.keywords.iter().map(|kw| kw.as_python_compat(parser)).collect::>(), + "starargs": self.starargs.as_python_compat(parser), + "kwargs": self.kwargs.as_python_compat(parser), }) } } -impl IntoPythonCompat for Keyword { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Keyword", self, to_row_col, { +impl AsPythonCompat for Keyword { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Keyword", self, parser, { "arg": self.arg.as_ref().map_or(json!(null), |s| json!(s)), - "value": self.value.into_python_compat(to_row_col), + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for Await { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Await", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for Await { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Await", self, parser, { + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for Compare { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Compar", self, to_row_col, { - "left": self.left.into_python_compat(to_row_col), - "ops": self.ops.iter().map(|op| op.into_python_compat(to_row_col)).collect::>(), - "comparators": self.comparators.iter().map(|op| op.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Compare { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Compar", self, parser, { + "left": self.left.as_python_compat(parser), + "ops": self.ops.iter().map(|op| op.as_python_compat(parser)).collect::>(), + "comparators": self.comparators.iter().map(|op| op.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for ComparisonOperator { - fn into_python_compat(&self, _: &ToRowColFn) -> Value { +impl AsPythonCompat for ComparisonOperator { + fn as_python_compat(&self, _: &Parser) -> Value { match self { ComparisonOperator::Eq => json!({"_type": "Eq"}), ComparisonOperator::NotEq => json!({"_type": "NotEq"}), @@ -592,321 +590,321 @@ impl IntoPythonCompat for ComparisonOperator { } } -impl IntoPythonCompat for Lambda { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Lambda", self, to_row_col, { - "args": self.args.into_python_compat(to_row_col), - "body": self.body.into_python_compat(to_row_col), +impl AsPythonCompat for Lambda { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Lambda", self, parser, { + "args": self.args.as_python_compat(parser), + "body": self.body.as_python_compat(parser), }) } } -impl IntoPythonCompat for crate::ast::Arguments { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Arguments", self, to_row_col, { - "posonlyargs": self.posonlyargs.iter().map(|arg| arg.into_python_compat(to_row_col)).collect::>(), - "args": self.args.iter().map(|arg| arg.into_python_compat(to_row_col)).collect::>(), - "vararg": self.vararg.into_python_compat(to_row_col), - "kwonlyargs": self.kwonlyargs.iter().map(|arg| arg.into_python_compat(to_row_col)).collect::>(), - "kw_defaults": self.kw_defaults.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "kwarg": self.kwarg.into_python_compat(to_row_col), - "defaults": self.defaults.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for crate::ast::Arguments { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Arguments", self, parser, { + "posonlyargs": self.posonlyargs.iter().map(|arg| arg.as_python_compat(parser)).collect::>(), + "args": self.args.iter().map(|arg| arg.as_python_compat(parser)).collect::>(), + "vararg": self.vararg.as_python_compat(parser), + "kwonlyargs": self.kwonlyargs.iter().map(|arg| arg.as_python_compat(parser)).collect::>(), + "kw_defaults": self.kw_defaults.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "kwarg": self.kwarg.as_python_compat(parser), + "defaults": self.defaults.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Arg { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Arg", self, to_row_col, { +impl AsPythonCompat for Arg { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Arg", self, parser, { "arg": self.arg, - "annotation": self.annotation.into_python_compat(to_row_col), + "annotation": self.annotation.as_python_compat(parser), }) } } -impl IntoPythonCompat for IfExp { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("IfExp", self, to_row_col, { - "test": self.test.into_python_compat(to_row_col), - "body": self.body.into_python_compat(to_row_col), - "orelse": self.orelse.into_python_compat(to_row_col), +impl AsPythonCompat for IfExp { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("IfExp", self, parser, { + "test": self.test.as_python_compat(parser), + "body": self.body.as_python_compat(parser), + "orelse": self.orelse.as_python_compat(parser), }) } } -impl IntoPythonCompat for FormattedValue { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("FormattedValue", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for FormattedValue { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("FormattedValue", self, parser, { + "value": self.value.as_python_compat(parser), "conversion": self.conversion, - "format_spec": self.format_spec.into_python_compat(to_row_col), + "format_spec": self.format_spec.as_python_compat(parser), }) } } -impl IntoPythonCompat for JoinedStr { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("JoinedStr", self, to_row_col, { - "values": self.values.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for JoinedStr { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("JoinedStr", self, parser, { + "values": self.values.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for If { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("If", self, to_row_col, { - "test": self.test.into_python_compat(to_row_col), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for If { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("If", self, parser, { + "test": self.test.as_python_compat(parser), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for While { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("While", self, to_row_col, { - "test": self.test.into_python_compat(to_row_col), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for While { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("While", self, parser, { + "test": self.test.as_python_compat(parser), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for For { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("For", self, to_row_col, { - "target": self.target.into_python_compat(to_row_col), - "iter": self.iter.into_python_compat(to_row_col), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for For { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("For", self, parser, { + "target": self.target.as_python_compat(parser), + "iter": self.iter.as_python_compat(parser), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for AsyncFor { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("AsyncFor", self, to_row_col, { - "target": self.target.into_python_compat(to_row_col), - "iter": self.iter.into_python_compat(to_row_col), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for AsyncFor { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("AsyncFor", self, parser, { + "target": self.target.as_python_compat(parser), + "iter": self.iter.as_python_compat(parser), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for With { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("With", self, to_row_col, { - "items": self.items.iter().map(|wi| wi.into_python_compat(to_row_col)).collect::>(), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for With { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("With", self, parser, { + "items": self.items.iter().map(|wi| wi.as_python_compat(parser)).collect::>(), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for AsyncWith { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("AsyncWith", self, to_row_col, { - "items": self.items.iter().map(|wi| wi.into_python_compat(to_row_col)).collect::>(), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for AsyncWith { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("AsyncWith", self, parser, { + "items": self.items.iter().map(|wi| wi.as_python_compat(parser)).collect::>(), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for WithItem { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("WithItem", self, to_row_col, { - "context_expr": self.context_expr.into_python_compat(to_row_col), - "optional_vars": self.optional_vars.into_python_compat(to_row_col), +impl AsPythonCompat for WithItem { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("WithItem", self, parser, { + "context_expr": self.context_expr.as_python_compat(parser), + "optional_vars": self.optional_vars.as_python_compat(parser), }) } } -impl IntoPythonCompat for Try { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Try", self, to_row_col, { - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "handlers": self.handlers.iter().map(|hndl| hndl.into_python_compat(to_row_col)).collect::>(), - "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "finalbody": self.finalbody.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Try { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Try", self, parser, { + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "handlers": self.handlers.iter().map(|hndl| hndl.as_python_compat(parser)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "finalbody": self.finalbody.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for TryStar { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("TryStar", self, to_row_col, { - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "handlers": self.handlers.iter().map(|hndl| hndl.into_python_compat(to_row_col)).collect::>(), - "orelse": self.orelse.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "finalbody": self.finalbody.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for TryStar { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("TryStar", self, parser, { + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "handlers": self.handlers.iter().map(|hndl| hndl.as_python_compat(parser)).collect::>(), + "orelse": self.orelse.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "finalbody": self.finalbody.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for ExceptHandler { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("ExceptHandler", self, to_row_col, { - "type": self.typ.into_python_compat(to_row_col), +impl AsPythonCompat for ExceptHandler { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("ExceptHandler", self, parser, { + "type": self.typ.as_python_compat(parser), "name": self.name.as_ref().map_or(json!(null), |s| json!(s)), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for FunctionDef { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("FunctionDef", self, to_row_col, { +impl AsPythonCompat for FunctionDef { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("FunctionDef", self, parser, { "name": self.name, - "args": self.args.into_python_compat(to_row_col), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "decorator_list": self.decorator_list.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "returns": self.returns.into_python_compat(to_row_col), + "args": self.args.as_python_compat(parser), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "decorator_list": self.decorator_list.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "returns": self.returns.as_python_compat(parser), "type_comment": self.type_comment.as_ref().map_or(json!(null), |s| json!(s)), - "type_params": self.type_params.iter().map(|tp| tp.into_python_compat(to_row_col)).collect::>(), + "type_params": self.type_params.iter().map(|tp| tp.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for AsyncFunctionDef { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("AsyncFunctionDef", self, to_row_col, { +impl AsPythonCompat for AsyncFunctionDef { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("AsyncFunctionDef", self, parser, { "name": self.name, - "args": self.args.into_python_compat(to_row_col), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "decorator_list": self.decorator_list.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "returns": self.returns.into_python_compat(to_row_col), + "args": self.args.as_python_compat(parser), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "decorator_list": self.decorator_list.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "returns": self.returns.as_python_compat(parser), "type_comment": self.type_comment.as_ref().map_or(json!(null), |s| json!(s)), - "type_params": self.type_params.iter().map(|tp| tp.into_python_compat(to_row_col)).collect::>(), + "type_params": self.type_params.iter().map(|tp| tp.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for ClassDef { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("ClassDef", self, to_row_col, { +impl AsPythonCompat for ClassDef { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("ClassDef", self, parser, { "name": self.name, - "bases": self.bases.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "keywords": self.keywords.iter().map(|kw| kw.into_python_compat(to_row_col)).collect::>(), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), - "decorator_list": self.decorator_list.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "type_params": self.type_params.iter().map(|tp| tp.into_python_compat(to_row_col)).collect::>(), + "bases": self.bases.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "keywords": self.keywords.iter().map(|kw| kw.as_python_compat(parser)).collect::>(), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + "decorator_list": self.decorator_list.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "type_params": self.type_params.iter().map(|tp| tp.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for Match { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("Match", self, to_row_col, { - "subject": self.subject.into_python_compat(to_row_col), - "cases": self.cases.iter().map(|mc| mc.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for Match { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("Match", self, parser, { + "subject": self.subject.as_python_compat(parser), + "cases": self.cases.iter().map(|mc| mc.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for MatchCase { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("MatchCase", self, to_row_col, { - "pattern": self.pattern.into_python_compat(to_row_col), - "guard": self.guard.into_python_compat(to_row_col), - "body": self.body.iter().map(|stmt| stmt.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for MatchCase { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("MatchCase", self, parser, { + "pattern": self.pattern.as_python_compat(parser), + "guard": self.guard.as_python_compat(parser), + "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for MatchPattern { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { +impl AsPythonCompat for MatchPattern { + fn as_python_compat(&self, parser: &Parser) -> Value { match self { - MatchPattern::MatchValue(val) => val.into_python_compat(to_row_col), - MatchPattern::MatchSingleton(expr) => expr.into_python_compat(to_row_col), - MatchPattern::MatchSequence(pats) => json!(pats.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>()), - MatchPattern::MatchStar(expr) => expr.into_python_compat(to_row_col), - MatchPattern::MatchMapping(map) => map.into_python_compat(to_row_col), - MatchPattern::MatchAs(mas) => mas.into_python_compat(to_row_col), - MatchPattern::MatchClass(cls) => cls.into_python_compat(to_row_col), - MatchPattern::MatchOr(pats) => json!(pats.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>()), + MatchPattern::MatchValue(val) => val.as_python_compat(parser), + MatchPattern::MatchSingleton(expr) => expr.as_python_compat(parser), + MatchPattern::MatchSequence(pats) => json!(pats.iter().map(|pat| pat.as_python_compat(parser)).collect::>()), + MatchPattern::MatchStar(expr) => expr.as_python_compat(parser), + MatchPattern::MatchMapping(map) => map.as_python_compat(parser), + MatchPattern::MatchAs(mas) => mas.as_python_compat(parser), + MatchPattern::MatchClass(cls) => cls.as_python_compat(parser), + MatchPattern::MatchOr(pats) => json!(pats.iter().map(|pat| pat.as_python_compat(parser)).collect::>()), } } } -impl IntoPythonCompat for MatchValue { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("MatchValue", self, to_row_col, { - "value": self.value.into_python_compat(to_row_col), +impl AsPythonCompat for MatchValue { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("MatchValue", self, parser, { + "value": self.value.as_python_compat(parser), }) } } -impl IntoPythonCompat for MatchAs { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("MatchAs", self, to_row_col, { +impl AsPythonCompat for MatchAs { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("MatchAs", self, parser, { "name": self.name.as_ref().map_or(json!(null), |s| json!(s)), - "pattern": self.pattern.into_python_compat(to_row_col), + "pattern": self.pattern.as_python_compat(parser), }) } } -impl IntoPythonCompat for MatchMapping { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("MatchMapping", self, to_row_col, { - "keys": self.keys.iter().map(|expr| expr.into_python_compat(to_row_col)).collect::>(), - "patterns": self.patterns.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for MatchMapping { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("MatchMapping", self, parser, { + "keys": self.keys.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "patterns": self.patterns.iter().map(|pat| pat.as_python_compat(parser)).collect::>(), "rest": self.rest.as_ref().map_or(json!(null), |s| json!(s)), }) } } -impl IntoPythonCompat for MatchClass { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("MatchClass", self, to_row_col, { - "cls": self.cls.into_python_compat(to_row_col), - "patterns": self.patterns.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>(), +impl AsPythonCompat for MatchClass { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("MatchClass", self, parser, { + "cls": self.cls.as_python_compat(parser), + "patterns": self.patterns.iter().map(|pat| pat.as_python_compat(parser)).collect::>(), "kwd_attrs": self.kwd_attrs, - "kwd_patterns": self.kwd_patterns.iter().map(|pat| pat.into_python_compat(to_row_col)).collect::>(), + "kwd_patterns": self.kwd_patterns.iter().map(|pat| pat.as_python_compat(parser)).collect::>(), }) } } -impl IntoPythonCompat for TypeParam { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { +impl AsPythonCompat for TypeParam { + fn as_python_compat(&self, parser: &Parser) -> Value { match self { - TypeParam::TypeVar(var) => var.into_python_compat(to_row_col), - TypeParam::ParamSpec(spec) => spec.into_python_compat(to_row_col), - TypeParam::TypeVarTuple(tup) => tup.into_python_compat(to_row_col), + TypeParam::TypeVar(var) => var.as_python_compat(parser), + TypeParam::ParamSpec(spec) => spec.as_python_compat(parser), + TypeParam::TypeVarTuple(tup) => tup.as_python_compat(parser), } } } -impl IntoPythonCompat for TypeVar { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("TypeVar", self, to_row_col, { +impl AsPythonCompat for TypeVar { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("TypeVar", self, parser, { "name": self.name, - "bound": self.bound.into_python_compat(to_row_col), + "bound": self.bound.as_python_compat(parser), }) } } -impl IntoPythonCompat for ParamSpec { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("ParamSpec", self, to_row_col, { +impl AsPythonCompat for ParamSpec { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("ParamSpec", self, parser, { "name": self.name, }) } } -impl IntoPythonCompat for TypeVarTuple { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("TypeVarTuple", self, to_row_col, { +impl AsPythonCompat for TypeVarTuple { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("TypeVarTuple", self, parser, { "name": self.name, }) } } -impl IntoPythonCompat for TypeAlias { - fn into_python_compat(&self, to_row_col: &ToRowColFn) -> Value { - json_python_compat_node!("TypeAlias", self, to_row_col, { +impl AsPythonCompat for TypeAlias { + fn as_python_compat(&self, parser: &Parser) -> Value { + json_python_compat_node!("TypeAlias", self, parser, { "name": self.name, - "type_params": self.type_params.iter().map(|tp| tp.into_python_compat(to_row_col)).collect::>(), - "value": self.value.into_python_compat(to_row_col), + "type_params": self.type_params.iter().map(|tp| tp.as_python_compat(parser)).collect::>(), + "value": self.value.as_python_compat(parser), }) } } @@ -914,10 +912,9 @@ impl IntoPythonCompat for TypeAlias { #[cfg(test)] mod tests { use serde_json::Value; - use miette::{bail, IntoDiagnostic, Result}; - use crate::Parser; - use crate::ast::*; + // use miette::{bail, IntoDiagnostic, Result}; + use super::parse_enderpy_source; use super::parse_python_source; #[test] @@ -926,18 +923,11 @@ mod tests { a: int = 1 print(a) "#; - let mut parser = Parser::new(source, "string"); - let enderpy_ast = parser.parse().into_diagnostic().unwrap(); + let enderpy_ast = parse_enderpy_source(source).unwrap(); let python_ast = parse_python_source(source).unwrap(); - assert_ast_eq(python_ast, enderpy_ast); - - // let mut lexer = Lexer::new(source); - // let enderpy_tokens = lexer.lex(); - // let python_tokens = lex_python_source(source).unwrap(); - // assert_tokens_eq(python_tokens, enderpy_tokens, &lexer); } - fn assert_ast_eq(python_ast: Value, enderpy_ast: Module) { + fn assert_ast_eq(python_ast: Value, enderpy_ast: Value) { } } diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index ed789bfd..b2a32f56 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -3494,6 +3494,10 @@ impl<'a> Parser<'a> { } } } + + pub fn to_row_col(&self, source_offset: u32) -> (u32, u32) { + self.lexer.to_row_col(source_offset) + } } #[cfg(test)] From d4ca780fc09edf16347db9b2b5d7fe48e0a30049 Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Sun, 7 Jul 2024 22:28:03 -0700 Subject: [PATCH 03/14] Begun working my way through various mismatches between the AST parser libraries. --- parser/Cargo.toml | 1 + parser/src/parser/compat.rs | 125 ++++++++++++++++++++++++++++-------- parser/src/parser/parser.rs | 8 ++- 3 files changed, 107 insertions(+), 27 deletions(-) diff --git a/parser/Cargo.toml b/parser/Cargo.toml index a6d25b95..e2c79b24 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -30,6 +30,7 @@ reqwest = { version= "0.12.4", features = ["blocking"] } tokio.workspace = true tabled = "0.15" terminal_size = "0.3" +assert-json-diff = "2.0" [lib] bench = false diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index 9aaadea3..bf4f5013 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -1,14 +1,14 @@ use std::io::Write; use std::convert::From; +use std::str::FromStr; use miette::{bail, IntoDiagnostic, Result}; +use serde_json::Number; use serde_json::{json, Value}; use crate::Parser; use crate::ast::*; use crate::runpython::{default_python_path, spawn_python_script_command}; -// type Parser = dyn Fn(u32) -> (u32, u32); - fn parse_python_source(source: &str) -> Result { let mut process = spawn_python_script_command( "parser/ast_python.py", @@ -24,10 +24,31 @@ fn parse_python_source(source: &str) -> Result { } // Get process stdout and parse result. let output = process.wait_with_output().into_diagnostic()?; - let ast: Value = serde_json::from_str(String::from_utf8_lossy(&output.stdout).as_ref()).into_diagnostic()?; + let mut ast = serde_json::from_str(String::from_utf8_lossy(&output.stdout).as_ref()).into_diagnostic()?; + remove_unimplemented_attributes(&mut ast); Ok(ast) } +fn remove_unimplemented_attributes(value: &mut Value) { + match value { + Value::Object(map) => { + // TODO ast_python: Adjust these ignored values as Enderpy adds support. + map.retain(|key, _| !matches!(key.as_str(), "ctx" | "type_ignores" | "kind")); + for (_, v) in map.iter_mut() { + remove_unimplemented_attributes(v); + } + }, + Value::Array(vec) => { + for v in vec.iter_mut() { + remove_unimplemented_attributes(v); + } + }, + _ => { + // Nothing to do for other value types. + }, + }; +} + fn parse_enderpy_source(source: &str) -> Result { let mut parser = Parser::new(source, "string"); let typed_ast = parser.parse().into_diagnostic()?; @@ -60,9 +81,9 @@ macro_rules! json_python_compat_node { let (start_row, start_col) = $parser.to_row_col($instance.node.start); let (end_row, end_col) = $parser.to_row_col($instance.node.end); node["_type"] = json!($name); - node["lineno"] = json!(start_row); + node["lineno"] = json!(start_row + 1); node["col_offset"] = json!(start_col); - node["end_lineno"] = json!(end_row); + node["end_lineno"] = json!(end_row + 1); node["end_col_offset"] = json!(end_col); node } @@ -118,6 +139,7 @@ impl AsPythonCompat for Assign { json_python_compat_node!("Assign", self, parser, { "targets": self.targets.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), "value": self.value.as_python_compat(parser), + "type_comment": json!(null), }) } } @@ -317,8 +339,8 @@ impl AsPythonCompat for ConstantValue { ConstantValue::Str(v) => json!(v), ConstantValue::Bytes(v) => json!(v), ConstantValue::Tuple(v) => json!(v.iter().map(|cons| cons.as_python_compat(parser)).collect::>()), - ConstantValue::Int(v) => json!(v), - ConstantValue::Float(v) => json!(v), + ConstantValue::Int(v) => Value::Number(Number::from_str(v).unwrap()), + ConstantValue::Float(v) => Value::Number(Number::from_str(v).unwrap()), ConstantValue::Complex { real, imaginary } => json!({"real": real, "imaginary": imaginary}), } } @@ -327,7 +349,7 @@ impl AsPythonCompat for ConstantValue { impl AsPythonCompat for List { fn as_python_compat(&self, parser: &Parser) -> Value { json_python_compat_node!("List", self, parser, { - "elements": self.elements.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "elts": self.elements.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } @@ -536,13 +558,27 @@ impl AsPythonCompat for Slice { impl AsPythonCompat for Call { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("Call", self, parser, { + let mut node = json_python_compat_node!("Call", self, parser, { "func": self.func.as_python_compat(parser), "args": self.args.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), "keywords": self.keywords.iter().map(|kw| kw.as_python_compat(parser)).collect::>(), - "starargs": self.starargs.as_python_compat(parser), - "kwargs": self.kwargs.as_python_compat(parser), - }) + }); + if let Some(expr) = &self.starargs { + node["starargs"] = expr.as_python_compat(parser); + } + if let Some(expr) = &self.kwargs { + node["kwargs"] = expr.as_python_compat(parser); + } + // NOTE: Python wraps print calls in an extra Expr node. + // Don't ask me why the parser does this. + if let Expression::Name(name) = &self.func { + if matches!(name.id.as_str(), "print") { + return json_python_compat_node!("Expr", self, parser, { + "value": node, + }) + } + } + node } } @@ -601,7 +637,9 @@ impl AsPythonCompat for Lambda { impl AsPythonCompat for crate::ast::Arguments { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("Arguments", self, parser, { + // NOTE: Arguments is kinda weird in Python. Feels like legacy support. + json!({ + "_type": "arguments", "posonlyargs": self.posonlyargs.iter().map(|arg| arg.as_python_compat(parser)).collect::>(), "args": self.args.iter().map(|arg| arg.as_python_compat(parser)).collect::>(), "vararg": self.vararg.as_python_compat(parser), @@ -615,9 +653,12 @@ impl AsPythonCompat for crate::ast::Arguments { impl AsPythonCompat for Arg { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("Arg", self, parser, { + // NOTE: Python doesn't use TitleCase for Arg. 🤦 + json_python_compat_node!("arg", self, parser, { "arg": self.arg, "annotation": self.annotation.as_python_compat(parser), + // TODO ast_python: Support for type_comment in arg. + "type_comment": json!(null), }) } } @@ -911,23 +952,57 @@ impl AsPythonCompat for TypeAlias { #[cfg(test)] mod tests { + use assert_json_diff::assert_json_matches_no_panic; use serde_json::Value; - // use miette::{bail, IntoDiagnostic, Result}; - - use super::parse_enderpy_source; - use super::parse_python_source; + use tabled::{ + builder::Builder, + settings::peaker::PriorityMax, + settings::{Style, Width}, + }; + use terminal_size::{terminal_size, Width as TerminalWidth}; + use super::{parse_enderpy_source, parse_python_source}; #[test] fn test_simple_compat() { let source = r#" -a: int = 1 -print(a) +def x(a: int) -> int: + return 1 + 1 +b = x(1) +print(b) "#; let enderpy_ast = parse_enderpy_source(source).unwrap(); - let python_ast = parse_python_source(source).unwrap(); - assert_ast_eq(python_ast, enderpy_ast); - } - - fn assert_ast_eq(python_ast: Value, enderpy_ast: Value) { + let mut python_ast = parse_python_source(source).unwrap(); + assert_ast_eq(&mut python_ast, &enderpy_ast); + } + + fn assert_ast_eq(python_ast: &mut Value, enderpy_ast: &Value) { + if let Err(message) = assert_json_matches_no_panic( + &python_ast, + &enderpy_ast, + assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict), + ) { + let mut table_builder = Builder::default(); + table_builder.push_record(["Python AST", "Enderpy AST"]); + table_builder.push_record([ + serde_json::to_string_pretty(&python_ast).unwrap(), + serde_json::to_string_pretty(&enderpy_ast).unwrap(), + ]); + let mut table = table_builder.build(); + table.with(Style::modern()); + // If run in a terminal, don't expand table beyond terminal width. + if let Some((TerminalWidth(width), _)) = terminal_size() { + table + .with( + Width::wrap(width as usize) + .keep_words() + .priority::(), + ) + .with(Width::increase(width as usize)); + } + panic!( + "Enderpy AST does not match Python AST.\n{}\n{}", + table, message + ); + } } } diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index b2a32f56..c0f8297a 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -101,6 +101,10 @@ impl<'a> Parser<'a> { Node::new(node.start, self.prev_token_end) } + fn finish_node_chomped(&self, node: Node) -> Node { + Node::new(node.start, self.prev_token_end - 1) + } + pub(crate) fn cur_token(&self) -> &Token { &self.cur_token } @@ -624,7 +628,7 @@ impl<'a> Parser<'a> { let body = self.parse_suite()?; if is_async { Ok(Statement::AsyncFunctionDef(Box::new(AsyncFunctionDef { - node: self.finish_node(node), + node: self.finish_node_chomped(node), name, args, body, @@ -635,7 +639,7 @@ impl<'a> Parser<'a> { }))) } else { Ok(Statement::FunctionDef(Box::new(FunctionDef { - node: self.finish_node(node), + node: self.finish_node_chomped(node), name, args, body, From 77e61381504a378cecc71f524a186840b31b1a3b Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Sun, 7 Jul 2024 22:34:02 -0700 Subject: [PATCH 04/14] Added option to include source in output. --- parser/src/parser/compat.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index bf4f5013..53505d58 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -972,10 +972,10 @@ print(b) "#; let enderpy_ast = parse_enderpy_source(source).unwrap(); let mut python_ast = parse_python_source(source).unwrap(); - assert_ast_eq(&mut python_ast, &enderpy_ast); + assert_ast_eq(&mut python_ast, &enderpy_ast, source); } - fn assert_ast_eq(python_ast: &mut Value, enderpy_ast: &Value) { + fn assert_ast_eq(python_ast: &mut Value, enderpy_ast: &Value, source: &str) { if let Err(message) = assert_json_matches_no_panic( &python_ast, &enderpy_ast, @@ -999,9 +999,15 @@ print(b) ) .with(Width::increase(width as usize)); } + let include_source = std::env::var("INCLUDE_SOURCE").is_ok(); + let formatted_source = if include_source { + format!("\nSource:\n{}", source) + } else { + "".to_string() + }; panic!( - "Enderpy AST does not match Python AST.\n{}\n{}", - table, message + "Enderpy AST does not match Python AST.\n{}{}\n{}", + formatted_source, table, message ); } } From 67b5a2e83cdd033aee658f1007f39e139ae064ef Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Mon, 8 Jul 2024 23:05:30 -0700 Subject: [PATCH 05/14] Fixed up the string based tests, mostly. Added plenty of TODOs. --- parser/src/parser/compat.rs | 382 +++++++++++++++++++++++++++++++++--- parser/src/parser/parser.rs | 30 ++- 2 files changed, 381 insertions(+), 31 deletions(-) diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index 53505d58..678e3715 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -102,7 +102,17 @@ impl AsPythonCompat for Module { impl AsPythonCompat for Statement { fn as_python_compat(&self, parser: &Parser) -> Value { match self { - Statement::ExpressionStatement(e) => e.as_python_compat(parser), + Statement::ExpressionStatement(e) => { + let expr = e.as_python_compat(parser); + json!({ + "_type": "Expr", + "lineno": expr["lineno"], + "col_offset": expr["col_offset"], + "end_lineno": expr["end_lineno"], + "end_col_offset": expr["end_col_offset"], + "value": expr, + }) + }, Statement::Import(i) => i.as_python_compat(parser), Statement::ImportFrom(i) => i.as_python_compat(parser), Statement::AssignStatement(a) => a.as_python_compat(parser), @@ -139,6 +149,7 @@ impl AsPythonCompat for Assign { json_python_compat_node!("Assign", self, parser, { "targets": self.targets.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), "value": self.value.as_python_compat(parser), + // TODO ast_python: Support for type_comment. "type_comment": json!(null), }) } @@ -341,7 +352,7 @@ impl AsPythonCompat for ConstantValue { ConstantValue::Tuple(v) => json!(v.iter().map(|cons| cons.as_python_compat(parser)).collect::>()), ConstantValue::Int(v) => Value::Number(Number::from_str(v).unwrap()), ConstantValue::Float(v) => Value::Number(Number::from_str(v).unwrap()), - ConstantValue::Complex { real, imaginary } => json!({"real": real, "imaginary": imaginary}), + ConstantValue::Complex { real: _real, imaginary } => json!(imaginary), } } } @@ -357,7 +368,9 @@ impl AsPythonCompat for List { impl AsPythonCompat for Tuple { fn as_python_compat(&self, parser: &Parser) -> Value { json_python_compat_node!("Tuple", self, parser, { - "elements": self.elements.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + // Yes, these are repeated. + "dims": self.elements.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), + "elts": self.elements.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) } } @@ -381,7 +394,7 @@ impl AsPythonCompat for Set { impl AsPythonCompat for BoolOperation { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("BoolOperation", self, parser, { + json_python_compat_node!("BoolOp", self, parser, { "op": self.op.as_python_compat(parser), "values": self.values.iter().map(|expr| expr.as_python_compat(parser)).collect::>(), }) @@ -399,7 +412,7 @@ impl AsPythonCompat for BooleanOperator { impl AsPythonCompat for UnaryOperation { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("UnaryOperation", self, parser, { + json_python_compat_node!("UnaryOp", self, parser, { "op": self.op.as_python_compat(parser), "operand": self.operand.as_python_compat(parser), }) @@ -466,7 +479,7 @@ impl AsPythonCompat for Yield { impl AsPythonCompat for YieldFrom { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("YieldForm", self, parser, { + json_python_compat_node!("YieldFrom", self, parser, { "value": self.value.as_python_compat(parser), }) } @@ -569,22 +582,13 @@ impl AsPythonCompat for Call { if let Some(expr) = &self.kwargs { node["kwargs"] = expr.as_python_compat(parser); } - // NOTE: Python wraps print calls in an extra Expr node. - // Don't ask me why the parser does this. - if let Expression::Name(name) = &self.func { - if matches!(name.id.as_str(), "print") { - return json_python_compat_node!("Expr", self, parser, { - "value": node, - }) - } - } node } } impl AsPythonCompat for Keyword { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("Keyword", self, parser, { + json_python_compat_node!("keyword", self, parser, { "arg": self.arg.as_ref().map_or(json!(null), |s| json!(s)), "value": self.value.as_python_compat(parser), }) @@ -601,7 +605,7 @@ impl AsPythonCompat for Await { impl AsPythonCompat for Compare { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("Compar", self, parser, { + json_python_compat_node!("Compare", self, parser, { "left": self.left.as_python_compat(parser), "ops": self.ops.iter().map(|op| op.as_python_compat(parser)).collect::>(), "comparators": self.comparators.iter().map(|op| op.as_python_compat(parser)).collect::>(), @@ -657,7 +661,7 @@ impl AsPythonCompat for Arg { json_python_compat_node!("arg", self, parser, { "arg": self.arg, "annotation": self.annotation.as_python_compat(parser), - // TODO ast_python: Support for type_comment in arg. + // TODO ast_python: Support for type_comment. "type_comment": json!(null), }) } @@ -971,11 +975,345 @@ b = x(1) print(b) "#; let enderpy_ast = parse_enderpy_source(source).unwrap(); - let mut python_ast = parse_python_source(source).unwrap(); - assert_ast_eq(&mut python_ast, &enderpy_ast, source); + let python_ast = parse_python_source(source).unwrap(); + assert_ast_eq(&python_ast, &enderpy_ast, source); + } + + fn python_parser_test_ast(inputs: &[&str]) { + for test_input in inputs.iter() { + let enderpy_ast = parse_enderpy_source(test_input).unwrap(); + let python_ast = parse_python_source(test_input).unwrap(); + assert_ast_eq(&python_ast, &enderpy_ast, test_input); + } + } + + #[test] + fn test_parse_assignment() { + python_parser_test_ast(&[ + "a = 1", + "a = None", + "a = True", + "a = False", + "a = 1j", + // TODO ast_python: Python does not evaluate bytes. + // "a = b'1'", + // "a = rb'1'", + // "a = br'1'", + "a = \"a\"", + "a = '''a'''", + "a = \"\"\"a\"\"\"", + "a = 'a'", + "a = 1, 2", + "a = 1, 2, ", + "a = b = 1", + "a,b = c,d = 1,2", + // augmented assignment + "a += 1", + "a -= 1", + "a *= 1", + "a /= 1", + "a //= 1", + "a %= 1", + "a **= 1", + "a <<= 1", + "a >>= 1", + "a &= 1", + "a ^= 1", + "a |= 1", + // annotated assignment + ]); + } + + #[test] + fn test_parse_assert_stmt() { + python_parser_test_ast(&["assert a", "assert a, b", "assert True, 'fancy message'"]); + } + + #[test] + fn test_pass_stmt() { + python_parser_test_ast(&["pass", "pass ", "pass\n"]); + } + + #[test] + fn test_parse_del_stmt() { + python_parser_test_ast(&["del a", "del a, b", "del a, b, "]); + } + + #[test] + fn parse_yield_statement() { + python_parser_test_ast(&["yield", "yield a", "yield a, b", "yield a, b, "]); + } + + #[test] + fn test_raise_statement() { + python_parser_test_ast(&["raise", "raise a", "raise a from c"]); + } + + #[test] + fn test_parse_break_continue() { + python_parser_test_ast(&["break", "continue"]); + } + + #[test] + fn test_parse_bool_op() { + python_parser_test_ast(&[ + "a or b", + "a and b", + // TODO ast_python: Python parses this as a BoolOp with 3 values. + // i.e. {"op": "or", "values": ["a", "b", "c"]} + // Enderpy parses this as a nested set of BoolOps. + // i.e. {"op": "or", "values": ["a", {"op": "or", "values": ["b", "c"]}]} + // I'm not sure which is correct. + "a and b or c", + ]); + } + + #[test] + fn test_parse_unary_op() { + python_parser_test_ast(&["not a", "+ a", "~ a", "-a"]); + } + + #[test] + fn test_named_expression() { + // TODO ast_python: Enderpy chokes on this. + // python_parser_test_ast(&["(a := b)"]); + } + + #[test] + fn test_tuple() { + python_parser_test_ast(&[ + "(a, b, c)", + // TODO ast_python: Enderpy doesn't handle newlines within a nested context. + // "(a, + // b, c)", + // "(a + // , b, c)", + // "(a, + // b, + // c)", +// "(a, +// )", + "(a, b, c,)", + ]); + } + + #[test] + fn test_yield_expression() { + python_parser_test_ast(&["yield", "yield a", "yield from a"]); + } + + #[test] + fn test_starred() { + // TODO ast_python: Enderpy chokes on this. + // python_parser_test_ast(&["(*a)"]); + } + + #[test] + fn test_await_expression() { + python_parser_test_ast(&["await a"]); + } + + #[test] + fn test_attribute_ref() { + python_parser_test_ast(&["a.b", "a.b.c", "a.b_c", "a.b.c.d"]); + } + + #[test] + fn parse_call() { + python_parser_test_ast(&[ + "a()", + "a(b)", + "a(b, c)", + "func(b=c)", + "func(a, b=c, d=e)", + "func(a, b=c, d=e, *f)", + "func(a, b=c, d=e, *f, **g)", + "func(a,)", + ]); + } + + #[test] + fn test_lambda() { + python_parser_test_ast(&[ + "lambda: a", + "lambda a: a", + "lambda a, b: a", + "lambda a, b, c: a", + "lambda a, *b: a", + "lambda a, *b, c: a", + "lambda a, *b, c, **d: a", + "lambda a=1 : a", + "lambda a=1 : a,", + ]); + } + + #[test] + fn test_conditional_expression() { + python_parser_test_ast(&["a if b else c if d else e"]); + } + + #[test] + fn test_string_literal_concatenation() { + python_parser_test_ast(&[ + "'a' 'b'", + // TODO ast_python: Python evaluates this as "ab". + // "b'a' b'b'", + "'a' 'b'", + // TODO ast_python: Enderpy evaluates this as 'r"a"b'. This seems wrong. + // "r'a' 'b'", + // TODO ast_python: Enderpy doesn't handle newlines within a nested context. + // "('a' + // 'b')", + // "('a' + // 'b', 'c')", +// "('a' +// 'b' +// 'c')", + // TODO ast_python: Python evaluates this as "ac". Enderpy creates 2 constants. + // "f'a' 'c'", + // TODO ast_python: Python evaluates this as "abc". Enderpy creates 3 constants. + // "f'a' 'b' 'c'", + // TODO ast_python: Python evaluates this as "dab". Enderpy creates 3 constants. + // "'d' f'a' 'b'", + "f'a_{1}' 'b' ", + ]); + } + + #[test] + fn test_fstring() { + python_parser_test_ast(&[ + "f'a'", + "f'hello_{a}'", + "f'hello_{a} {b}'", + "f'hello_{a} {b} {c}'", + // unsupported + // "f'hello_{f'''{a}'''}'", + ]); + } + + #[test] + fn test_comparison() { + python_parser_test_ast(&[ + "a == b", + "a != b", + "a > b", + "a < b", + "a >= b", + "a <= b", + "a is b", + "a is not b", + "a in b", + "a not in b", + "a < b < c", + ]); + } + + #[test] + fn test_while_statement() { + python_parser_test_ast(&[ + "while a: pass", + "while a: + pass", + "while a: + a = 1 +else: + b = 1 +", + ]); + } + + #[test] + fn test_try_statement() { + python_parser_test_ast(&[ + "try: + pass +except: + pass", + "try: + pass +except Exception: + pass", + "try: + pass +except Exception as e: + pass", + "try: + pass +except Exception as e: + pass +else: + pass", + "try: + pass +except Exception as e: + pass +else: + pass +finally: + pass", + "try: + pass +except *Exception as e: + pass +", + ]); + } + + #[test] + fn test_ellipsis_statement() { + python_parser_test_ast(&[ + "def a(): ...", + "def a(): + ...", + "a = ...", + "... + 1", + ]); + } + + macro_rules! parser_test { + ($test_name:ident, $test_file:expr) => { + #[test] + fn $test_name() { + let test_case = std::fs::read_to_string($test_file).unwrap(); + python_parser_test_ast(&[test_case.as_str()]); + } + } + } - fn assert_ast_eq(python_ast: &mut Value, enderpy_ast: &Value, source: &str) { + // parser_test!(test_functions, "test_data/inputs/functions.py"); + // parser_test!(test_if, "test_data/inputs/if.py"); + // parser_test!(test_indentation, "test_data/inputs/indentation.py"); + // parser_test!( + // test_separate_statements, + // "test_data/inputs/separate_statements.py" + // ); + // parser_test!(test_try, "test_data/inputs/try.py"); + // parser_test!( + // annotated_assignment, + // "test_data/inputs/annotated_assignment.py" + // ); + // parser_test!(binary_op, "test_data/inputs/binary_op.py"); + // parser_test!(class, "test_data/inputs/class.py"); + // parser_test!(dict, "test_data/inputs/dict.py"); + // parser_test!(test_for, "test_data/inputs/for.py"); + // parser_test!(from_import, "test_data/inputs/from_import.py"); + // parser_test!(function_def, "test_data/inputs/function_def.py"); + // parser_test!( + // generator_expressions, + // "test_data/inputs/generator_expressions.py" + // ); + // parser_test!(lists, "test_data/inputs/lists.py"); + // parser_test!(test_match, "test_data/inputs/match.py"); + // parser_test!(sets, "test_data/inputs/sets.py"); + // parser_test!(string, "test_data/inputs/string.py"); + // parser_test!(subscript, "test_data/inputs/subscript.py"); + // parser_test!(with, "test_data/inputs/with.py"); + // parser_test!(newlines, "test_data/inputs/newlines.py"); + // parser_test!(comments, "test_data/inputs/comments.py"); + // parser_test!(types_alias, "test_data/inputs/type_alias.py"); + + fn assert_ast_eq(python_ast: &Value, enderpy_ast: &Value, source: &str) { if let Err(message) = assert_json_matches_no_panic( &python_ast, &enderpy_ast, @@ -1001,7 +1339,7 @@ print(b) } let include_source = std::env::var("INCLUDE_SOURCE").is_ok(); let formatted_source = if include_source { - format!("\nSource:\n{}", source) + format!("\nSource:\n{}\n", source) } else { "".to_string() }; diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index c0f8297a..7ab1efa2 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -24,6 +24,7 @@ pub struct Parser<'a> { lexer: Lexer<'a>, cur_token: Token, prev_token_end: u32, + prev_token_was_whitespace: bool, // This var keeps track of how many levels deep we are in a list, tuple or set // expression. This is used to determine if we should parse comma separated // expressions as tuple or not. @@ -61,6 +62,7 @@ impl<'a> Parser<'a> { lexer, cur_token, prev_token_end, + prev_token_was_whitespace: false, nested_expression_list, curr_line_string: String::new(), path, @@ -102,7 +104,12 @@ impl<'a> Parser<'a> { } fn finish_node_chomped(&self, node: Node) -> Node { - Node::new(node.start, self.prev_token_end - 1) + let end = if self.prev_token_was_whitespace { + self.prev_token_end - 1 + } else { + self.prev_token_end + }; + Node::new(node.start, end) } pub(crate) fn cur_token(&self) -> &Token { @@ -171,6 +178,7 @@ impl<'a> Parser<'a> { } self.prev_token_end = self.cur_token.end; + self.prev_token_was_whitespace = matches!(self.cur_token.kind, Kind::NewLine | Kind::Dedent); self.cur_token = token; if self.cur_kind() == Kind::Comment { self.advance(); @@ -410,7 +418,7 @@ impl<'a> Parser<'a> { }; Ok(Statement::WhileStatement(Box::new(While { - node: self.finish_node(node), + node: self.finish_node_chomped(node), test, body, orelse, @@ -551,7 +559,7 @@ impl<'a> Parser<'a> { if is_try_star { Ok(Statement::TryStarStatement(Box::new(TryStar { - node: self.finish_node(node), + node: self.finish_node_chomped(node), body, handlers, orelse, @@ -591,7 +599,7 @@ impl<'a> Parser<'a> { let body = self.parse_suite()?; handlers.push(ExceptHandler { - node: self.finish_node(node), + node: self.finish_node_chomped(node), typ, name, body, @@ -1574,13 +1582,14 @@ impl<'a> Parser<'a> { // https://docs.python.org/3/reference/expressions.html#conditional-expressions fn parse_conditional_expression(&mut self) -> Result { + let node = self.start_node(); let or_test = self.parse_or_test(); if self.eat(Kind::If) { let test = self.parse_or_test()?; self.expect(Kind::Else)?; let or_else = self.parse_expression()?; return Ok(Expression::IfExp(Box::new(IfExp { - node: self.start_node(), + node: self.finish_node(node), test, body: or_test?, orelse: or_else, @@ -1701,7 +1710,6 @@ impl<'a> Parser<'a> { } let expr = self.parse_starred_expression(node, first_expr)?; - self.expect(Kind::RightParen)?; Ok(expr) } @@ -2304,19 +2312,21 @@ impl<'a> Parser<'a> { } else if self.at(Kind::Mul) { let star_arg_node = self.start_node(); self.bump(Kind::Mul); + let expr = self.parse_expression()?; let star_arg = Expression::Starred(Box::new(Starred { node: self.finish_node(star_arg_node), - value: self.parse_expression()?, + value: expr, })); positional_args.push(star_arg); } else if self.at(Kind::Pow) { let kwarg_node = self.start_node(); self.bump(Kind::Pow); seen_keyword = true; + let expr = self.parse_expression()?; let kwarg = Keyword { node: self.finish_node(kwarg_node), arg: None, - value: self.parse_expression()?, + value: expr, }; keyword_args.push(kwarg); } else { @@ -2953,6 +2963,7 @@ impl<'a> Parser<'a> { return Ok(expr); } } + self.expect(Kind::RightParen)?; Ok(Expression::Tuple(Box::new(Tuple { node: self.finish_node(node), elements, @@ -3316,6 +3327,7 @@ impl<'a> Parser<'a> { } else { None }; + let arg_node = self.finish_node(node); let default = if self.eat(Kind::Assign) { Some(self.parse_expression()?) } else { @@ -3323,7 +3335,7 @@ impl<'a> Parser<'a> { }; Ok(( Arg { - node: self.finish_node(node), + node: arg_node, arg, annotation, }, From bfaef2a57943735a4b4ed464bcd254a59ecfccc8 Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Tue, 9 Jul 2024 21:02:17 -0700 Subject: [PATCH 06/14] Fixed some more tests. --- parser/Cargo.toml | 1 + parser/src/parser/compat.rs | 31 ++++++++++++++++++++++--------- parser/src/parser/parser.rs | 19 ++++++++----------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/parser/Cargo.toml b/parser/Cargo.toml index e2c79b24..5284493b 100644 --- a/parser/Cargo.toml +++ b/parser/Cargo.toml @@ -31,6 +31,7 @@ tokio.workspace = true tabled = "0.15" terminal_size = "0.3" assert-json-diff = "2.0" +pretty_assertions = "1.4" [lib] bench = false diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index 678e3715..e8e0601d 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -742,6 +742,8 @@ impl AsPythonCompat for With { json_python_compat_node!("With", self, parser, { "items": self.items.iter().map(|wi| wi.as_python_compat(parser)).collect::>(), "body": self.body.iter().map(|stmt| stmt.as_python_compat(parser)).collect::>(), + // TODO ast_python: Support for type_comment. + "type_comment": json!(null), }) } } @@ -757,7 +759,8 @@ impl AsPythonCompat for AsyncWith { impl AsPythonCompat for WithItem { fn as_python_compat(&self, parser: &Parser) -> Value { - json_python_compat_node!("WithItem", self, parser, { + json!({ + "_type": "withitem", "context_expr": self.context_expr.as_python_compat(parser), "optional_vars": self.optional_vars.as_python_compat(parser), }) @@ -1281,7 +1284,7 @@ except *Exception as e: } - // parser_test!(test_functions, "test_data/inputs/functions.py"); + parser_test!(test_functions, "test_data/inputs/functions.py"); // parser_test!(test_if, "test_data/inputs/if.py"); // parser_test!(test_indentation, "test_data/inputs/indentation.py"); // parser_test!( @@ -1314,11 +1317,27 @@ except *Exception as e: // parser_test!(types_alias, "test_data/inputs/type_alias.py"); fn assert_ast_eq(python_ast: &Value, enderpy_ast: &Value, source: &str) { - if let Err(message) = assert_json_matches_no_panic( + let include_source = std::env::var("INCLUDE_SOURCE").is_ok(); + let side_by_side = std::env::var("SIDE_BY_SIDE").is_ok(); + + let formatted_source = if include_source { + format!("\nSource:\n{}\n", source) + } else { + "".to_string() + }; + if !side_by_side { + pretty_assertions::assert_eq!( + &python_ast, + &enderpy_ast, + "Enderpy AST does not match Python AST.\n{}\x1b[31mPython AST\x1b[0m / \x1b[32mEnderpy AST\x1b[0m", + formatted_source, + ); + } else if let Err(message) = assert_json_matches_no_panic( &python_ast, &enderpy_ast, assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict), ) { + let mut table_builder = Builder::default(); table_builder.push_record(["Python AST", "Enderpy AST"]); table_builder.push_record([ @@ -1337,12 +1356,6 @@ except *Exception as e: ) .with(Width::increase(width as usize)); } - let include_source = std::env::var("INCLUDE_SOURCE").is_ok(); - let formatted_source = if include_source { - format!("\nSource:\n{}\n", source) - } else { - "".to_string() - }; panic!( "Enderpy AST does not match Python AST.\n{}{}\n{}", formatted_source, table, message diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 7ab1efa2..7d799a92 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -24,7 +24,7 @@ pub struct Parser<'a> { lexer: Lexer<'a>, cur_token: Token, prev_token_end: u32, - prev_token_was_whitespace: bool, + prev_nonwhitespace_token_end: u32, // This var keeps track of how many levels deep we are in a list, tuple or set // expression. This is used to determine if we should parse comma separated // expressions as tuple or not. @@ -62,7 +62,7 @@ impl<'a> Parser<'a> { lexer, cur_token, prev_token_end, - prev_token_was_whitespace: false, + prev_nonwhitespace_token_end: prev_token_end, nested_expression_list, curr_line_string: String::new(), path, @@ -104,12 +104,7 @@ impl<'a> Parser<'a> { } fn finish_node_chomped(&self, node: Node) -> Node { - let end = if self.prev_token_was_whitespace { - self.prev_token_end - 1 - } else { - self.prev_token_end - }; - Node::new(node.start, end) + Node::new(node.start, self.prev_nonwhitespace_token_end) } pub(crate) fn cur_token(&self) -> &Token { @@ -178,7 +173,9 @@ impl<'a> Parser<'a> { } self.prev_token_end = self.cur_token.end; - self.prev_token_was_whitespace = matches!(self.cur_token.kind, Kind::NewLine | Kind::Dedent); + if !matches!(self.cur_token.kind, Kind::WhiteSpace | Kind::NewLine | Kind::Dedent) { + self.prev_nonwhitespace_token_end = self.prev_token_end; + } self.cur_token = token; if self.cur_kind() == Kind::Comment { self.advance(); @@ -481,13 +478,13 @@ impl<'a> Parser<'a> { if is_async { Ok(Statement::AsyncWithStatement(Box::new(AsyncWith { - node: self.finish_node(node), + node: self.finish_node_chomped(node), items, body, }))) } else { Ok(Statement::WithStatement(Box::new(With { - node: self.finish_node(node), + node: self.finish_node_chomped(node), items, body, }))) From 0eba53263a62b1f7e6f769b85476ea059acf9aa2 Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Tue, 9 Jul 2024 21:50:07 -0700 Subject: [PATCH 07/14] Fixed another statement. --- parser/src/parser/parser.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 7d799a92..e4a7cc5d 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -393,7 +393,7 @@ impl<'a> Parser<'a> { }; Ok(Statement::IfStatement(Box::new(If { - node: self.finish_node(node), + node: self.finish_node_chomped(node), test, body, orelse: or_else_vec, From 7b7d12f3a6d0bc50c73e1d0356bfc0fcbbfab9d0 Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Tue, 9 Jul 2024 21:52:14 -0700 Subject: [PATCH 08/14] Updated some snapshots. --- parser/src/parser/compat.rs | 4 +- ...rser__parser__parser__tests__comments.snap | 4 +- ...parser__tests__conditional_expression.snap | 8 +-- ...__parser__parser__tests__function_def.snap | 68 +++++++++---------- ...ser__parser__parser__tests__functions.snap | 14 ++-- ...hon_parser__parser__parser__tests__if.snap | 22 +++--- ...rser__parser__parser__tests__lambda-8.snap | 2 +- ...rser__parser__parser__tests__newlines.snap | 4 +- ...__parser__parser__tests__parse_call-6.snap | 2 +- ...r__parser__tests__separate_statements.snap | 2 +- ...n_parser__parser__parser__tests__sets.snap | 2 +- ...on_parser__parser__parser__tests__try.snap | 18 ++--- ...arser__parser__tests__try_statement-4.snap | 2 +- ..._parser__parser__parser__tests__tuple.snap | 2 +- ...ser__parser__tests__while_statement-3.snap | 2 +- ...n_parser__parser__parser__tests__with.snap | 14 ++-- 16 files changed, 86 insertions(+), 84 deletions(-) diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index e8e0601d..9e299aa2 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -971,12 +971,14 @@ mod tests { #[test] fn test_simple_compat() { + let source = r#" def x(a: int) -> int: return 1 + 1 b = x(1) print(b) "#; + let enderpy_ast = parse_enderpy_source(source).unwrap(); let python_ast = parse_python_source(source).unwrap(); assert_ast_eq(&python_ast, &enderpy_ast, source); @@ -1284,7 +1286,7 @@ except *Exception as e: } - parser_test!(test_functions, "test_data/inputs/functions.py"); + // parser_test!(test_functions, "test_data/inputs/functions.py"); // parser_test!(test_if, "test_data/inputs/if.py"); // parser_test!(test_indentation, "test_data/inputs/indentation.py"); // parser_test!( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__comments.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__comments.snap index e16a5bfb..ae04f4fa 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__comments.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__comments.snap @@ -12,7 +12,7 @@ Module { FunctionDef { node: Node { start: 17, - end: 34, + end: 33, }, name: "a", args: Arguments { @@ -51,7 +51,7 @@ Module { FunctionDef { node: Node { start: 35, - end: 95, + end: 94, }, name: "b", args: Arguments { diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__conditional_expression.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__conditional_expression.snap index 691ea8fa..0db07841 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__conditional_expression.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__conditional_expression.snap @@ -12,8 +12,8 @@ Module { IfExp( IfExp { node: Node { - start: 25, - end: 0, + start: 0, + end: 25, }, test: Name( Name { @@ -36,8 +36,8 @@ Module { orelse: IfExp( IfExp { node: Node { - start: 25, - end: 0, + start: 12, + end: 25, }, test: Name( Name { diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__function_def.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__function_def.snap index 69b17035..2ba3b693 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__function_def.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__function_def.snap @@ -12,7 +12,7 @@ Module { FunctionDef { node: Node { start: 0, - end: 14, + end: 13, }, name: "a", args: Arguments { @@ -48,7 +48,7 @@ Module { FunctionDef { node: Node { start: 15, - end: 34, + end: 32, }, name: "a", args: Arguments { @@ -84,7 +84,7 @@ Module { FunctionDef { node: Node { start: 34, - end: 55, + end: 54, }, name: "a", args: Arguments { @@ -145,7 +145,7 @@ Module { FunctionDef { node: Node { start: 56, - end: 80, + end: 79, }, name: "a", args: Arguments { @@ -208,7 +208,7 @@ Module { FunctionDef { node: Node { start: 81, - end: 102, + end: 101, }, name: "a", args: Arguments { @@ -269,7 +269,7 @@ Module { FunctionDef { node: Node { start: 103, - end: 124, + end: 123, }, name: "a", args: Arguments { @@ -315,7 +315,7 @@ Module { FunctionDef { node: Node { start: 125, - end: 188, + end: 187, }, name: "f", args: Arguments { @@ -346,7 +346,7 @@ Module { Arg { node: Node { start: 155, - end: 158, + end: 156, }, arg: "b", annotation: None, @@ -354,7 +354,7 @@ Module { Arg { node: Node { start: 160, - end: 163, + end: 161, }, arg: "c", annotation: None, @@ -382,7 +382,7 @@ Module { Arg { node: Node { start: 172, - end: 175, + end: 173, }, arg: "f", annotation: None, @@ -463,7 +463,7 @@ Module { FunctionDef { node: Node { start: 189, - end: 214, + end: 213, }, name: "func", args: Arguments { @@ -509,7 +509,7 @@ Module { AsyncFunctionDef { node: Node { start: 215, - end: 235, + end: 234, }, name: "a", args: Arguments { @@ -545,7 +545,7 @@ Module { AsyncFunctionDef { node: Node { start: 236, - end: 261, + end: 259, }, name: "a", args: Arguments { @@ -581,7 +581,7 @@ Module { AsyncFunctionDef { node: Node { start: 261, - end: 288, + end: 287, }, name: "a", args: Arguments { @@ -642,7 +642,7 @@ Module { FunctionDef { node: Node { start: 289, - end: 306, + end: 305, }, name: "a", args: Arguments { @@ -689,7 +689,7 @@ Module { FunctionDef { node: Node { start: 307, - end: 327, + end: 326, }, name: "a", args: Arguments { @@ -746,7 +746,7 @@ Module { FunctionDef { node: Node { start: 328, - end: 348, + end: 347, }, name: "a", args: Arguments { @@ -803,7 +803,7 @@ Module { FunctionDef { node: Node { start: 349, - end: 375, + end: 374, }, name: "a", args: Arguments { @@ -880,7 +880,7 @@ Module { FunctionDef { node: Node { start: 376, - end: 394, + end: 393, }, name: "a", args: Arguments { @@ -926,7 +926,7 @@ Module { FunctionDef { node: Node { start: 395, - end: 416, + end: 415, }, name: "a", args: Arguments { @@ -982,7 +982,7 @@ Module { FunctionDef { node: Node { start: 417, - end: 441, + end: 440, }, name: "a", args: Arguments { @@ -1048,7 +1048,7 @@ Module { FunctionDef { node: Node { start: 442, - end: 472, + end: 471, }, name: "a", args: Arguments { @@ -1134,7 +1134,7 @@ Module { FunctionDef { node: Node { start: 473, - end: 492, + end: 491, }, name: "a", args: Arguments { @@ -1180,7 +1180,7 @@ Module { FunctionDef { node: Node { start: 493, - end: 515, + end: 514, }, name: "a", args: Arguments { @@ -1236,7 +1236,7 @@ Module { FunctionDef { node: Node { start: 516, - end: 541, + end: 540, }, name: "a", args: Arguments { @@ -1302,7 +1302,7 @@ Module { FunctionDef { node: Node { start: 542, - end: 573, + end: 572, }, name: "a", args: Arguments { @@ -1388,7 +1388,7 @@ Module { FunctionDef { node: Node { start: 574, - end: 600, + end: 599, }, name: "a", args: Arguments { @@ -1453,7 +1453,7 @@ Module { FunctionDef { node: Node { start: 601, - end: 630, + end: 629, }, name: "a", args: Arguments { @@ -1528,7 +1528,7 @@ Module { FunctionDef { node: Node { start: 631, - end: 1013, + end: 1012, }, name: "dataclass_transform", args: Arguments { @@ -1543,7 +1543,7 @@ Module { Arg { node: Node { start: 667, - end: 690, + end: 683, }, arg: "eq_default", annotation: Some( @@ -1561,7 +1561,7 @@ Module { Arg { node: Node { start: 696, - end: 723, + end: 715, }, arg: "order_default", annotation: Some( @@ -1579,7 +1579,7 @@ Module { Arg { node: Node { start: 729, - end: 758, + end: 750, }, arg: "kw_only_default", annotation: Some( @@ -1597,7 +1597,7 @@ Module { Arg { node: Node { start: 764, - end: 792, + end: 784, }, arg: "frozen_default", annotation: Some( @@ -1615,7 +1615,7 @@ Module { Arg { node: Node { start: 900, - end: 965, + end: 960, }, arg: "field_specifiers", annotation: Some( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__functions.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__functions.snap index 12e704d4..0394950e 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__functions.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__functions.snap @@ -12,7 +12,7 @@ Module { FunctionDef { node: Node { start: 0, - end: 342, + end: 339, }, name: "_handle_ticker_index", args: Arguments { @@ -101,7 +101,7 @@ Module { If { node: Node { start: 96, - end: 320, + end: 315, }, test: Compare( Compare { @@ -189,7 +189,7 @@ Module { If { node: Node { start: 177, - end: 320, + end: 315, }, test: Compare( Compare { @@ -340,7 +340,7 @@ Module { FunctionDef { node: Node { start: 342, - end: 610, + end: 608, }, name: "_extract_ticker_client_types_data", args: Arguments { @@ -439,7 +439,7 @@ Module { With { node: Node { start: 468, - end: 561, + end: 556, }, items: [ WithItem { @@ -671,7 +671,7 @@ Module { FunctionDef { node: Node { start: 610, - end: 1153, + end: 1151, }, name: "common_process", args: Arguments { @@ -738,7 +738,7 @@ Module { If { node: Node { start: 663, - end: 759, + end: 754, }, test: Compare( Compare { diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__if.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__if.snap index 2be8c027..df91ee74 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__if.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__if.snap @@ -12,7 +12,7 @@ Module { If { node: Node { start: 0, - end: 16, + end: 14, }, test: Name( Name { @@ -40,7 +40,7 @@ Module { If { node: Node { start: 16, - end: 33, + end: 31, }, test: Name( Name { @@ -68,7 +68,7 @@ Module { If { node: Node { start: 33, - end: 44, + end: 43, }, test: Name( Name { @@ -96,7 +96,7 @@ Module { If { node: Node { start: 45, - end: 66, + end: 64, }, test: Name( Name { @@ -132,7 +132,7 @@ Module { If { node: Node { start: 66, - end: 95, + end: 93, }, test: Compare( Compare { @@ -182,7 +182,7 @@ Module { If { node: Node { start: 95, - end: 162, + end: 160, }, test: Compare( Compare { @@ -283,7 +283,7 @@ Module { If { node: Node { start: 162, - end: 257, + end: 254, }, test: Compare( Compare { @@ -393,7 +393,7 @@ Module { If { node: Node { start: 257, - end: 273, + end: 272, }, test: Name( Name { @@ -441,7 +441,7 @@ Module { If { node: Node { start: 273, - end: 290, + end: 288, }, test: Name( Name { @@ -489,7 +489,7 @@ Module { If { node: Node { start: 290, - end: 471, + end: 470, }, test: Attribute( Attribute { @@ -514,7 +514,7 @@ Module { If { node: Node { start: 310, - end: 436, + end: 435, }, test: Constant( Constant { diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__lambda-8.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__lambda-8.snap index ad856666..386019c7 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__lambda-8.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__lambda-8.snap @@ -25,7 +25,7 @@ Module { Arg { node: Node { start: 7, - end: 10, + end: 8, }, arg: "a", annotation: None, diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__newlines.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__newlines.snap index a4113fdd..6721dd74 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__newlines.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__newlines.snap @@ -29,7 +29,7 @@ Module { Tuple { node: Node { start: 4, - end: 20, + end: 21, }, elements: [ Constant( @@ -153,7 +153,7 @@ Module { FunctionDef { node: Node { start: 68, - end: 100, + end: 99, }, name: "hello", args: Arguments { diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__parse_call-6.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__parse_call-6.snap index 35c142e8..1dae3a66 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__parse_call-6.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__parse_call-6.snap @@ -38,7 +38,7 @@ Module { Starred { node: Node { start: 18, - end: 19, + end: 20, }, value: Name( Name { diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__separate_statements.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__separate_statements.snap index 8eb1c99e..ee85bcea 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__separate_statements.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__separate_statements.snap @@ -12,7 +12,7 @@ Module { FunctionDef { node: Node { start: 99, - end: 145, + end: 143, }, name: "foo", args: Arguments { diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__sets.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__sets.snap index 4b28f724..6f2689e0 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__sets.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__sets.snap @@ -862,7 +862,7 @@ Module { Tuple { node: Node { start: 302, - end: 306, + end: 307, }, elements: [ Name( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try.snap index 9e38a19c..20268d31 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try.snap @@ -87,14 +87,14 @@ Module { ExceptHandler { node: Node { start: 135, - end: 190, + end: 189, }, typ: Some( Tuple( Tuple { node: Node { start: 142, - end: 165, + end: 166, }, elements: [ Name( @@ -235,14 +235,14 @@ Module { ExceptHandler { node: Node { start: 233, - end: 287, + end: 286, }, typ: Some( Tuple( Tuple { node: Node { start: 240, - end: 263, + end: 264, }, elements: [ Name( @@ -371,7 +371,7 @@ Module { ExceptHandler { node: Node { start: 339, - end: 382, + end: 379, }, typ: Some( Name( @@ -432,7 +432,7 @@ Module { AsyncWith { node: Node { start: 391, - end: 914, + end: 913, }, items: [ WithItem { @@ -535,7 +535,7 @@ Module { If { node: Node { start: 483, - end: 914, + end: 913, }, test: Compare( Compare { @@ -1003,14 +1003,14 @@ Module { ExceptHandler { node: Node { start: 914, - end: 1078, + end: 1077, }, typ: Some( Tuple( Tuple { node: Node { start: 921, - end: 963, + end: 964, }, elements: [ Attribute( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-4.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-4.snap index 4b6577a6..5273faa4 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-4.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-4.snap @@ -28,7 +28,7 @@ Module { ExceptHandler { node: Node { start: 26, - end: 70, + end: 69, }, typ: Some( Name( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple.snap index 20dfed35..cabc4cc5 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple.snap @@ -13,7 +13,7 @@ Module { Tuple { node: Node { start: 0, - end: 8, + end: 9, }, elements: [ Name( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__while_statement-3.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__while_statement-3.snap index f7d681d0..c2736654 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__while_statement-3.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__while_statement-3.snap @@ -12,7 +12,7 @@ Module { While { node: Node { start: 0, - end: 43, + end: 42, }, test: Name( Name { diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__with.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__with.snap index 7c196c4c..eb63e1c5 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__with.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__with.snap @@ -12,7 +12,7 @@ Module { With { node: Node { start: 0, - end: 13, + end: 12, }, items: [ WithItem { @@ -48,7 +48,7 @@ Module { With { node: Node { start: 14, - end: 32, + end: 31, }, items: [ WithItem { @@ -94,7 +94,7 @@ Module { With { node: Node { start: 33, - end: 59, + end: 58, }, items: [ WithItem { @@ -166,7 +166,7 @@ Module { With { node: Node { start: 60, - end: 88, + end: 87, }, items: [ WithItem { @@ -238,7 +238,7 @@ Module { AsyncWith { node: Node { start: 89, - end: 113, + end: 112, }, items: [ WithItem { @@ -284,7 +284,7 @@ Module { AsyncWith { node: Node { start: 114, - end: 151, + end: 149, }, items: [ WithItem { @@ -356,7 +356,7 @@ Module { AsyncWith { node: Node { start: 151, - end: 200, + end: 199, }, items: [ WithItem { From fadf812878faa19f247cdadd2c74b9af7a99b9eb Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Tue, 9 Jul 2024 21:55:20 -0700 Subject: [PATCH 09/14] More snapshots. --- ...nderpy_python_parser__parser__parser__tests__lambda-9.snap | 2 +- ...py_python_parser__parser__parser__tests__parse_call-7.snap | 4 ++-- ...python_parser__parser__parser__tests__try_statement-5.snap | 2 +- ...python_parser__parser__parser__tests__try_statement-6.snap | 4 ++-- ...enderpy_python_parser__parser__parser__tests__tuple-2.snap | 2 +- ...enderpy_python_parser__parser__parser__tests__tuple-3.snap | 2 +- ...enderpy_python_parser__parser__parser__tests__tuple-4.snap | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__lambda-9.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__lambda-9.snap index 650581b3..1dfc571e 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__lambda-9.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__lambda-9.snap @@ -32,7 +32,7 @@ Module { Arg { node: Node { start: 7, - end: 10, + end: 8, }, arg: "a", annotation: None, diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__parse_call-7.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__parse_call-7.snap index 3d96315f..48a7b76a 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__parse_call-7.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__parse_call-7.snap @@ -38,7 +38,7 @@ Module { Starred { node: Node { start: 18, - end: 19, + end: 20, }, value: Name( Name { @@ -92,7 +92,7 @@ Module { Keyword { node: Node { start: 22, - end: 24, + end: 25, }, arg: None, value: Name( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-5.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-5.snap index bcba8eff..8439f00b 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-5.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-5.snap @@ -28,7 +28,7 @@ Module { ExceptHandler { node: Node { start: 26, - end: 70, + end: 69, }, typ: Some( Name( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-6.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-6.snap index 6916f93f..8f7604ad 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-6.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__try_statement-6.snap @@ -12,7 +12,7 @@ Module { TryStar { node: Node { start: 0, - end: 47, + end: 46, }, body: [ Pass( @@ -28,7 +28,7 @@ Module { ExceptHandler { node: Node { start: 14, - end: 47, + end: 46, }, typ: Some( Name( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-2.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-2.snap index a041e53a..870e40ab 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-2.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-2.snap @@ -13,7 +13,7 @@ Module { Tuple { node: Node { start: 0, - end: 20, + end: 21, }, elements: [ Name( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-3.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-3.snap index 59abf839..a3efa430 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-3.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-3.snap @@ -13,7 +13,7 @@ Module { Tuple { node: Node { start: 0, - end: 21, + end: 22, }, elements: [ Name( diff --git a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-4.snap b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-4.snap index 913facc6..b9416ddc 100644 --- a/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-4.snap +++ b/parser/test_data/output/enderpy_python_parser__parser__parser__tests__tuple-4.snap @@ -13,7 +13,7 @@ Module { Tuple { node: Node { start: 0, - end: 36, + end: 37, }, elements: [ Name( From 85d66be0c63d3a4fcf6680ecbffc140a8fcee522 Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Wed, 10 Jul 2024 00:04:39 -0700 Subject: [PATCH 10/14] Fixed a bug where subscripts with a dotted attribute ref had the wrong start/end. --- parser/src/parser/compat.rs | 8 +++++--- parser/src/parser/parser.rs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index 9e299aa2..71bc8988 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -971,14 +971,12 @@ mod tests { #[test] fn test_simple_compat() { - let source = r#" def x(a: int) -> int: return 1 + 1 b = x(1) print(b) "#; - let enderpy_ast = parse_enderpy_source(source).unwrap(); let python_ast = parse_python_source(source).unwrap(); assert_ast_eq(&python_ast, &enderpy_ast, source); @@ -1122,6 +1120,11 @@ print(b) fn test_attribute_ref() { python_parser_test_ast(&["a.b", "a.b.c", "a.b_c", "a.b.c.d"]); } + #[test] + fn test_subscript() { + python_parser_test_ast(&["a[1]", "a.b[1]"]); + } + #[test] fn parse_call() { @@ -1283,7 +1286,6 @@ except *Exception as e: python_parser_test_ast(&[test_case.as_str()]); } } - } // parser_test!(test_functions, "test_data/inputs/functions.py"); diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index e4a7cc5d..dc7ff48f 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -2290,7 +2290,7 @@ impl<'a> Parser<'a> { self.parse_attribute_ref(node, atom_or_primary) } else if self.at(Kind::LeftBrace) { // https://docs.python.org/3/reference/expressions.html#slicings - self.parse_subscript(node, atom_or_primary) + self.parse_subscript(atom_or_primary.get_node(), atom_or_primary) } else if self.eat(Kind::LeftParen) { // parse call // https://docs.python.org/3/reference/expressions.html#calls From 65034147e8ea1a87c45245db0548f2f9dcf7021f Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Wed, 10 Jul 2024 22:54:24 -0700 Subject: [PATCH 11/14] Removed some comments. --- parser/src/parser/compat.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index 71bc8988..acfd138e 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -641,7 +641,6 @@ impl AsPythonCompat for Lambda { impl AsPythonCompat for crate::ast::Arguments { fn as_python_compat(&self, parser: &Parser) -> Value { - // NOTE: Arguments is kinda weird in Python. Feels like legacy support. json!({ "_type": "arguments", "posonlyargs": self.posonlyargs.iter().map(|arg| arg.as_python_compat(parser)).collect::>(), @@ -657,7 +656,6 @@ impl AsPythonCompat for crate::ast::Arguments { impl AsPythonCompat for Arg { fn as_python_compat(&self, parser: &Parser) -> Value { - // NOTE: Python doesn't use TitleCase for Arg. 🤦 json_python_compat_node!("arg", self, parser, { "arg": self.arg, "annotation": self.annotation.as_python_compat(parser), @@ -1067,6 +1065,7 @@ print(b) // Enderpy parses this as a nested set of BoolOps. // i.e. {"op": "or", "values": ["a", {"op": "or", "values": ["b", "c"]}]} // I'm not sure which is correct. + // "a or b or c", "a and b or c", ]); } From c90eb84813b1c434af18cfcfa83ad426fc4ad2a0 Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Thu, 11 Jul 2024 00:22:40 -0700 Subject: [PATCH 12/14] Fixed lexer for nested tokens. --- parser/src/lexer/compat.rs | 20 ++++++++------------ parser/src/lexer/mod.rs | 24 ++++++++++++++++++++---- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/parser/src/lexer/compat.rs b/parser/src/lexer/compat.rs index da5ef146..3f685114 100644 --- a/parser/src/lexer/compat.rs +++ b/parser/src/lexer/compat.rs @@ -317,18 +317,14 @@ print(a) python_tokenize_test_lexer(&["import a", "import a.b", "import a.b.c", "import a from b"]); } - // TODO lex_python: Decide whether to keep this test or not. The Python lexer + Enderpy lexer - // handle newlines in a nested context slightly differently. - // - Python increments the row counter. - // - Enderpy appends them to the original row. - // #[test] - // fn test_lex_other() { - // python_tokenize_test_lexer( - // &["(a, - // - // )"], - // ); - // } + #[test] + fn test_lex_other() { + python_tokenize_test_lexer( + &["(a, + +)"], + ); + } #[test] fn test_lex_indentation() { diff --git a/parser/src/lexer/mod.rs b/parser/src/lexer/mod.rs index df4ee883..eb8f4031 100644 --- a/parser/src/lexer/mod.rs +++ b/parser/src/lexer/mod.rs @@ -255,7 +255,7 @@ impl<'a> Lexer<'a> { } fn next_kind(&mut self) -> Result { - if self.start_of_line && self.nesting == 0 { + if self.start_of_line { self.line_starts.push(self.current); if let Some(indent_kind) = self.match_indentation()? { self.start_of_line = false; // WHY!? @@ -914,6 +914,22 @@ impl<'a> Lexer<'a> { } } } + if self.nesting > 0 { + // Don't add indent/dedent tokens when in nested context. + // For example, in the following example + // ``` + // a = ( + // 1 + // ) + // ``` + // the indentation of "1" is completely inconsequential. To be technically correct, + // we'll return a whiteSpace token if any amount of whitespace was found. + if spaces_count > 0 { + return Ok(Some(Kind::WhiteSpace)); + } else { + return Ok(None); + } + } if spaces_count == 0 { // When there are no spaces and only a new line // like the following @@ -936,14 +952,14 @@ impl<'a> Lexer<'a> { // loop over indent stack from the top and check if this element matches the // new indentation level if nothing matches then it is an error // do not pop the element from the stack - let mut indentation_matches_outer_evel = false; + let mut indentation_matches_outer_level = false; for top in self.indent_stack.iter().rev() { if top == &spaces_count { - indentation_matches_outer_evel = true; + indentation_matches_outer_level = true; break; } } - if !indentation_matches_outer_evel { + if !indentation_matches_outer_level { return Err(LexError::UnindentDoesNotMatchAnyOuterIndentationLevel); } Ok(Some(Kind::Dedent)) From 9ee75fa29e6cf8bf24a7c74a3392acadcc15acf2 Mon Sep 17 00:00:00 2001 From: Daniel Woznicki Date: Sat, 10 Aug 2024 11:27:33 -0700 Subject: [PATCH 13/14] Fixed more failing tests. --- parser/src/parser/compat.rs | 23 ++++++++++++++--------- parser/src/parser/parser.rs | 4 +++- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index acfd138e..10745825 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -969,12 +969,17 @@ mod tests { #[test] fn test_simple_compat() { - let source = r#" -def x(a: int) -> int: - return 1 + 1 -b = x(1) -print(b) +// let source = r#" +// def x(a: int) -> int: +// return 1 + 1 +// b = x(1) +// print(b) +// "#; + + let source = r#"(a +, b, c) "#; + let enderpy_ast = parse_enderpy_source(source).unwrap(); let python_ast = parse_python_source(source).unwrap(); assert_ast_eq(&python_ast, &enderpy_ast, source); @@ -1086,10 +1091,10 @@ print(b) python_parser_test_ast(&[ "(a, b, c)", // TODO ast_python: Enderpy doesn't handle newlines within a nested context. - // "(a, - // b, c)", - // "(a - // , b, c)", + "(a, + b, c)", + "(a + , b, c)", // "(a, // b, // c)", diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 3b0d52a8..19aec922 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -177,6 +177,8 @@ impl<'a> Parser<'a> { self.prev_nonwhitespace_token_end = self.prev_token_end; } self.cur_token = token; + // println!("{:?} {:?} {:?}", self.cur_token, self.to_row_col(self.cur_token.start), self.to_row_col(self.cur_token.end)); + // println!("{}", std::backtrace::Backtrace::force_capture()); if self.cur_kind() == Kind::Comment { self.advance(); } @@ -2863,7 +2865,7 @@ impl<'a> Parser<'a> { let value = self.cur_token().value.to_string(); self.expect(Kind::Identifier)?; Ok(Expression::Name(Box::new(Name { - node: self.finish_node(node), + node: self.finish_node_chomped(node), id: value, parenthesized: false, }))) From a4bd1be91d3b6063125221373922912d755fcf00 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Sun, 11 Aug 2024 00:21:43 +0200 Subject: [PATCH 14/14] Fix to_row_col function --- parser/src/parser/compat.rs | 93 ++++++++++++++++++++----------------- parser/src/parser/parser.rs | 10 ++-- 2 files changed, 58 insertions(+), 45 deletions(-) diff --git a/parser/src/parser/compat.rs b/parser/src/parser/compat.rs index 10745825..c83fe257 100644 --- a/parser/src/parser/compat.rs +++ b/parser/src/parser/compat.rs @@ -1,13 +1,13 @@ -use std::io::Write; -use std::convert::From; -use std::str::FromStr; use miette::{bail, IntoDiagnostic, Result}; use serde_json::Number; use serde_json::{json, Value}; +use std::convert::From; +use std::io::Write; +use std::str::FromStr; -use crate::Parser; use crate::ast::*; use crate::runpython::{default_python_path, spawn_python_script_command}; +use crate::Parser; fn parse_python_source(source: &str) -> Result { let mut process = spawn_python_script_command( @@ -24,7 +24,8 @@ fn parse_python_source(source: &str) -> Result { } // Get process stdout and parse result. let output = process.wait_with_output().into_diagnostic()?; - let mut ast = serde_json::from_str(String::from_utf8_lossy(&output.stdout).as_ref()).into_diagnostic()?; + let mut ast = + serde_json::from_str(String::from_utf8_lossy(&output.stdout).as_ref()).into_diagnostic()?; remove_unimplemented_attributes(&mut ast); Ok(ast) } @@ -37,15 +38,15 @@ fn remove_unimplemented_attributes(value: &mut Value) { for (_, v) in map.iter_mut() { remove_unimplemented_attributes(v); } - }, + } Value::Array(vec) => { for v in vec.iter_mut() { remove_unimplemented_attributes(v); } - }, + } _ => { // Nothing to do for other value types. - }, + } }; } @@ -75,19 +76,17 @@ impl AsNullablePythonCompat for Option { } macro_rules! json_python_compat_node { - ($name:literal, $instance:ident, $parser:ident, $other_fields:tt) => { - { - let mut node = json!($other_fields); - let (start_row, start_col) = $parser.to_row_col($instance.node.start); - let (end_row, end_col) = $parser.to_row_col($instance.node.end); - node["_type"] = json!($name); - node["lineno"] = json!(start_row + 1); - node["col_offset"] = json!(start_col); - node["end_lineno"] = json!(end_row + 1); - node["end_col_offset"] = json!(end_col); - node - } - }; + ($name:literal, $instance:ident, $parser:ident, $other_fields:tt) => {{ + let mut node = json!($other_fields); + let (start_row, start_col, end_row, end_col) = + $parser.to_row_col($instance.node.start, $instance.node.end); + node["_type"] = json!($name); + node["lineno"] = json!(start_row + 1); + node["col_offset"] = json!(start_col); + node["end_lineno"] = json!(end_row + 1); + node["end_col_offset"] = json!(end_col); + node + }}; } impl AsPythonCompat for Module { @@ -112,7 +111,7 @@ impl AsPythonCompat for Statement { "end_col_offset": expr["end_col_offset"], "value": expr, }) - }, + } Statement::Import(i) => i.as_python_compat(parser), Statement::ImportFrom(i) => i.as_python_compat(parser), Statement::AssignStatement(a) => a.as_python_compat(parser), @@ -349,10 +348,16 @@ impl AsPythonCompat for ConstantValue { ConstantValue::Bool(v) => json!(v), ConstantValue::Str(v) => json!(v), ConstantValue::Bytes(v) => json!(v), - ConstantValue::Tuple(v) => json!(v.iter().map(|cons| cons.as_python_compat(parser)).collect::>()), + ConstantValue::Tuple(v) => json!(v + .iter() + .map(|cons| cons.as_python_compat(parser)) + .collect::>()), ConstantValue::Int(v) => Value::Number(Number::from_str(v).unwrap()), ConstantValue::Float(v) => Value::Number(Number::from_str(v).unwrap()), - ConstantValue::Complex { real: _real, imaginary } => json!(imaginary), + ConstantValue::Complex { + real: _real, + imaginary, + } => json!(imaginary), } } } @@ -862,12 +867,18 @@ impl AsPythonCompat for MatchPattern { match self { MatchPattern::MatchValue(val) => val.as_python_compat(parser), MatchPattern::MatchSingleton(expr) => expr.as_python_compat(parser), - MatchPattern::MatchSequence(pats) => json!(pats.iter().map(|pat| pat.as_python_compat(parser)).collect::>()), + MatchPattern::MatchSequence(pats) => json!(pats + .iter() + .map(|pat| pat.as_python_compat(parser)) + .collect::>()), MatchPattern::MatchStar(expr) => expr.as_python_compat(parser), MatchPattern::MatchMapping(map) => map.as_python_compat(parser), MatchPattern::MatchAs(mas) => mas.as_python_compat(parser), MatchPattern::MatchClass(cls) => cls.as_python_compat(parser), - MatchPattern::MatchOr(pats) => json!(pats.iter().map(|pat| pat.as_python_compat(parser)).collect::>()), + MatchPattern::MatchOr(pats) => json!(pats + .iter() + .map(|pat| pat.as_python_compat(parser)) + .collect::>()), } } } @@ -957,6 +968,7 @@ impl AsPythonCompat for TypeAlias { #[cfg(test)] mod tests { + use super::{parse_enderpy_source, parse_python_source}; use assert_json_diff::assert_json_matches_no_panic; use serde_json::Value; use tabled::{ @@ -965,16 +977,15 @@ mod tests { settings::{Style, Width}, }; use terminal_size::{terminal_size, Width as TerminalWidth}; - use super::{parse_enderpy_source, parse_python_source}; #[test] fn test_simple_compat() { -// let source = r#" -// def x(a: int) -> int: -// return 1 + 1 -// b = x(1) -// print(b) -// "#; + // let source = r#" + // def x(a: int) -> int: + // return 1 + 1 + // b = x(1) + // print(b) + // "#; let source = r#"(a , b, c) @@ -1067,7 +1078,7 @@ mod tests { "a and b", // TODO ast_python: Python parses this as a BoolOp with 3 values. // i.e. {"op": "or", "values": ["a", "b", "c"]} - // Enderpy parses this as a nested set of BoolOps. + // Enderpy parses this as a nested set of BoolOps. // i.e. {"op": "or", "values": ["a", {"op": "or", "values": ["b", "c"]}]} // I'm not sure which is correct. // "a or b or c", @@ -1098,8 +1109,8 @@ mod tests { // "(a, // b, // c)", -// "(a, -// )", + // "(a, + // )", "(a, b, c,)", ]); } @@ -1129,7 +1140,6 @@ mod tests { python_parser_test_ast(&["a[1]", "a.b[1]"]); } - #[test] fn parse_call() { python_parser_test_ast(&[ @@ -1178,9 +1188,9 @@ mod tests { // 'b')", // "('a' // 'b', 'c')", -// "('a' -// 'b' -// 'c')", + // "('a' + // 'b' + // 'c')", // TODO ast_python: Python evaluates this as "ac". Enderpy creates 2 constants. // "f'a' 'c'", // TODO ast_python: Python evaluates this as "abc". Enderpy creates 3 constants. @@ -1289,7 +1299,7 @@ except *Exception as e: let test_case = std::fs::read_to_string($test_file).unwrap(); python_parser_test_ast(&[test_case.as_str()]); } - } + }; } // parser_test!(test_functions, "test_data/inputs/functions.py"); @@ -1345,7 +1355,6 @@ except *Exception as e: &enderpy_ast, assert_json_diff::Config::new(assert_json_diff::CompareMode::Strict), ) { - let mut table_builder = Builder::default(); table_builder.push_record(["Python AST", "Enderpy AST"]); table_builder.push_record([ diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 66baae25..ab9a7e76 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -11,6 +11,7 @@ use miette::Result; use super::{concat_string_exprs, is_at_compound_statement, is_iterable, map_unary_operator}; use crate::{ error::ParsingError, + get_row_col_position, lexer::Lexer, parser::{ast::*, extract_string_inside}, token::{Kind, Token, TokenValue}, @@ -173,7 +174,10 @@ impl<'a> Parser<'a> { } self.prev_token_end = self.cur_token.end; - if !matches!(self.cur_token.kind, Kind::WhiteSpace | Kind::NewLine | Kind::Dedent) { + if !matches!( + self.cur_token.kind, + Kind::WhiteSpace | Kind::NewLine | Kind::Dedent + ) { self.prev_nonwhitespace_token_end = self.prev_token_end; } self.cur_token = token; @@ -3503,8 +3507,8 @@ impl<'a> Parser<'a> { } } - pub fn to_row_col(&self, source_offset: u32) -> (u32, u32) { - self.lexer.to_row_col(source_offset) + pub fn to_row_col(&self, start: u32, end: u32) -> (u32, u32, u32, u32) { + get_row_col_position(start, end, &self.lexer.line_starts) } }