From 4c2441ef09bf0dcdacefa0310803b285a8f9b8c6 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Wed, 15 Nov 2023 18:51:02 +0100 Subject: [PATCH 01/13] Replace let with const --- client/vscode/src/extension.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/vscode/src/extension.ts b/client/vscode/src/extension.ts index 294d4ee6..4016736f 100644 --- a/client/vscode/src/extension.ts +++ b/client/vscode/src/extension.ts @@ -28,7 +28,7 @@ export async function activate(_context: ExtensionContext) { run, debug: run, }; - let clientOptions: LanguageClientOptions = { + const clientOptions: LanguageClientOptions = { documentSelector: [{ scheme: "file", language: "python" }], traceOutputChannel, }; From db35159937119dba599640f48cedc32dd8f70438 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Sat, 9 Dec 2023 22:43:48 +0100 Subject: [PATCH 02/13] Fix bug in parsing () as expression --- parser/src/parser/parser.rs | 14 ++++++++++++++ parser/test_data/inputs/one_liners/function_def.py | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index cc2732fa..9c89ffc9 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -2115,6 +2115,15 @@ impl Parser { // https://docs.python.org/3/reference/expressions.html#conditional-expressions fn parse_expression_2(&mut self) -> Result { let node = self.start_node(); + // This is a hack to make this function parse () as tuple + if self.at(Kind::LeftParen) && matches!(self.peek_kind(), Ok(Kind::RightParen)) { + self.bump(Kind::LeftParen); + self.bump(Kind::RightParen); + return Ok(Expression::Tuple(Box::new(Tuple { + node: self.finish_node(node), + elements: vec![], + }))) + } if self.eat(Kind::Lambda) { let params_list = self.parse_parameters(true).expect("lambda params"); self.expect(Kind::Colon)?; @@ -2963,6 +2972,11 @@ impl Parser { // after seeing vararg the must_have_default is reset // until we see a default value again must_have_default = false; + // In this case this is not a vararg but only marks the end of positional arguments + // e.g. def foo(a, b, *, c, d) + if self.eat(Kind::Comma) { + continue; + } let (param, default) = self.parse_parameter(is_lambda)?; // default is not allowed for vararg if default.is_some() { diff --git a/parser/test_data/inputs/one_liners/function_def.py b/parser/test_data/inputs/one_liners/function_def.py index f97e0909..e7bac04f 100644 --- a/parser/test_data/inputs/one_liners/function_def.py +++ b/parser/test_data/inputs/one_liners/function_def.py @@ -53,3 +53,14 @@ def a[T: U, V: W, **X](): pass def a[T, *U, **V](): pass def a[T: U, *V, **W](): pass + +def dataclass_transform( + *, + eq_default: bool = True, + order_default: bool = False, + kw_only_default: bool = False, + frozen_default: bool = False, # on 3.11, runtime accepts it as part of kwargs + # this next annotation cannot be parsed need fix + field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), + **kwargs: Any, +) -> IdentityFunction: ... From 8632087dc0fcbfbb71b1088614954ef8803a4678 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Sat, 9 Dec 2023 22:44:32 +0100 Subject: [PATCH 03/13] Add module name to symbol table --- typechecker/src/semantic_analyzer.rs | 2 +- typechecker/src/settings.rs | 3 ++- typechecker/src/state.rs | 7 +++++-- typechecker/src/symbol_table.rs | 20 ++++++++++++++------ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/typechecker/src/semantic_analyzer.rs b/typechecker/src/semantic_analyzer.rs index c8a3ac8d..208721c4 100644 --- a/typechecker/src/semantic_analyzer.rs +++ b/typechecker/src/semantic_analyzer.rs @@ -37,7 +37,7 @@ pub struct SemanticAnalyzer { #[allow(unused)] impl SemanticAnalyzer { pub fn new(file: EnderpyFile, imports: HashMap) -> Self { - let globals = SymbolTable::global(); + let globals = SymbolTable::global(file.module_name()); log::debug!("Creating semantic analyzer for {}", file.module_name()); SemanticAnalyzer { globals, diff --git a/typechecker/src/settings.rs b/typechecker/src/settings.rs index 234574e1..1b32c4f5 100644 --- a/typechecker/src/settings.rs +++ b/typechecker/src/settings.rs @@ -41,13 +41,14 @@ impl Settings { } pub fn test_settings() -> Self { + let file_dir = env::current_dir().unwrap(); Settings { debug: false, root: PathBuf::from(""), follow_imports: FollowImports::All, import_discovery: ImportDiscovery { python_executable: None, - typeshed_path: None, + typeshed_path: Some(PathBuf::from(file_dir.parent().unwrap().join("typeshed"))), }, } } diff --git a/typechecker/src/state.rs b/typechecker/src/state.rs index ce742801..600f71c5 100644 --- a/typechecker/src/state.rs +++ b/typechecker/src/state.rs @@ -23,9 +23,10 @@ pub struct State { impl State { pub fn new(file: EnderpyFile) -> Self { + let module_name = file.module_name(); Self { file, - symbol_table: SymbolTable::global(), + symbol_table: SymbolTable::global(module_name), diagnostics: Vec::new(), imports: HashMap::new(), } @@ -36,7 +37,9 @@ impl State { for stmt in &self.file.body { sem_anal.visit_stmt(stmt) } - self.symbol_table = sem_anal.globals + // TODO: Hacky way to add the global scope to all scopes in symbol table after finishing + sem_anal.globals.exit_scope(); + self.symbol_table = sem_anal.globals; } pub fn get_symbol_table(&self) -> SymbolTable { diff --git a/typechecker/src/symbol_table.rs b/typechecker/src/symbol_table.rs index d49076b6..da28ea89 100644 --- a/typechecker/src/symbol_table.rs +++ b/typechecker/src/symbol_table.rs @@ -15,6 +15,9 @@ pub struct SymbolTable { /// The distance between the current scope and the scope where the symbol /// was defined _locals: HashMap, + + /// Name of the module that this symbol table is for + pub module_name: String, } #[derive(Debug, Clone)] @@ -205,7 +208,7 @@ pub enum SymbolScope { } impl SymbolTable { - pub fn global() -> Self { + pub fn global(module_name: String) -> Self { let mut builtin_scope = SymbolTableScope { id: get_id(), symbol_table_type: SymbolTableType::BUILTIN, @@ -293,6 +296,7 @@ impl SymbolTable { scopes: vec![builtin_scope, global_scope], all_scopes: vec![], _locals: HashMap::new(), + module_name, } } @@ -328,6 +332,7 @@ impl SymbolTable { Some(pos) => { let mut innermost_scope = self.innermost_scope(pos); while let Some(scope) = innermost_scope { + log::debug!("lookin in scope: {:?}", scope.name); if let Some(symbol) = scope.symbols.get(&lookup_request.name) { return Some(symbol); } @@ -346,7 +351,7 @@ impl SymbolTable { } } None => { - if let Some(symbol) = self.current_scope().symbols.get(&lookup_request.name) { + if let Some(symbol) = self.global_scope().symbols.get(&lookup_request.name) { return Some(symbol); } } @@ -367,7 +372,7 @@ impl SymbolTable { } pub fn global_scope(&self) -> &SymbolTableScope { - self.scopes + self.all_scopes .iter() .filter(|scope| scope.symbol_table_type == SymbolTableType::Module) .last() @@ -376,9 +381,12 @@ impl SymbolTable { pub fn exit_scope(&mut self) { let finished_scope = self.scopes.pop(); - match finished_scope { - Some(scope) => self.all_scopes.push(scope), - None => panic!("tried to exit non-existent scope"), + if let Some(scope) = finished_scope { + // Also pop the builtin scope if we are exiting the global scope + // if scope.symbol_table_type == SymbolTableType::Module { + // self.all_scopes.push(self.scopes.pop().unwrap()); + // } + self.all_scopes.push(scope); } } From b8bd475da366b8d0a09e22a0b60e06a6bbe0c919 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Sat, 9 Dec 2023 22:44:55 +0100 Subject: [PATCH 04/13] Draft: Resolve stdlib class names from typeshed --- typechecker/src/type_check/type_evaluator.rs | 221 +++++++++++++------ 1 file changed, 151 insertions(+), 70 deletions(-) diff --git a/typechecker/src/type_check/type_evaluator.rs b/typechecker/src/type_check/type_evaluator.rs index 6ed4712e..fa02d4fa 100755 --- a/typechecker/src/type_check/type_evaluator.rs +++ b/typechecker/src/type_check/type_evaluator.rs @@ -114,35 +114,51 @@ impl TypeEvaluator { ast::Expression::List(l) => { let final_elm_type = self.get_sequence_type_from_elements(&l.elements); let builtin_type = self.get_builtin_type(builtins::LIST_TYPE); - Ok(PythonType::Class(ClassType::new( - builtin_type, - vec![final_elm_type], - ))) + if let Some(builtin_type) = builtin_type { + Ok(PythonType::Class(ClassType::new( + builtin_type, + vec![final_elm_type], + ))) + } else { + Ok(PythonType::Unknown) + } } ast::Expression::Tuple(t) => { let elm_type = self.get_sequence_type_from_elements(&t.elements); let builtin_type = self.get_builtin_type(builtins::TUPLE_TYPE); - Ok(PythonType::Class(ClassType::new( - builtin_type, - vec![elm_type], - ))) + if let Some(builtin_type) = builtin_type { + Ok(PythonType::Class(ClassType::new( + builtin_type, + vec![elm_type], + ))) + } else { + Ok(PythonType::Unknown) + } } ast::Expression::Dict(d) => { let key_type = self.get_sequence_type_from_elements(&d.keys); let value_type = self.get_sequence_type_from_elements(&d.values); let builtin_type = self.get_builtin_type(builtins::DICT_TYPE); - Ok(PythonType::Class(ClassType::new( - builtin_type, - vec![key_type, value_type], - ))) + if let Some(builtin_type) = builtin_type { + Ok(PythonType::Class(ClassType::new( + builtin_type, + vec![key_type, value_type], + ))) + } else { + Ok(PythonType::Unknown) + } } ast::Expression::Set(s) => { let elm_type = self.get_sequence_type_from_elements(&s.elements); let builtin_type = self.get_builtin_type(builtins::SET_TYPE); - Ok(PythonType::Class(ClassType::new( - builtin_type, - vec![elm_type], - ))) + if let Some(builtin_type) = builtin_type { + Ok(PythonType::Class(ClassType::new( + builtin_type, + vec![elm_type], + ))) + } else { + Ok(PythonType::Unknown) + } } ast::Expression::BoolOp(_) => Ok(PythonType::Bool), ast::Expression::UnaryOp(u) => match u.op { @@ -228,59 +244,12 @@ impl TypeEvaluator { } } Expression::Subscript(s) => { + let type_parameters = vec![self.get_type_from_annotation(&s.slice)]; // This is a generic type - let typ = match *s.value.clone() { - Expression::Constant(_) => todo!(), - Expression::List(_) => todo!(), - Expression::Tuple(_) => todo!(), - Expression::Dict(_) => todo!(), - Expression::Set(_) => todo!(), - Expression::Name(n) => { - // TODO: handle builtins with enum - if self.is_literal(n.id.clone()) { - return self.handle_literal_type(s); - } - if self.is_union(n.id.clone()) { - match *s.slice.clone() { - Expression::Tuple(t) => { - return self.handle_union_type(t.elements); - } - expr @ Expression::Name(_) - | expr @ Expression::Constant(_) - | expr @ Expression::Subscript(_) - | expr @ Expression::Slice(_) => { - return self.handle_union_type(vec![expr]); - } - _ => panic!("Union type must have a tuple as parameter"), - } - } - self.get_builtin_type(n.id.as_str()) - } - Expression::BoolOp(_) => todo!(), - Expression::UnaryOp(_) => todo!(), - Expression::BinOp(_) => todo!(), - Expression::NamedExpr(_) => todo!(), - Expression::Yield(_) => todo!(), - Expression::YieldFrom(_) => todo!(), - Expression::Starred(_) => todo!(), - Expression::Generator(_) => todo!(), - Expression::ListComp(_) => todo!(), - Expression::SetComp(_) => todo!(), - Expression::DictComp(_) => todo!(), - Expression::Attribute(_) => todo!(), - Expression::Subscript(_) => todo!(), - Expression::Slice(_) => todo!(), - Expression::Call(_) => todo!(), - Expression::Await(_) => todo!(), - Expression::Compare(_) => todo!(), - Expression::Lambda(_) => todo!(), - Expression::IfExp(_) => todo!(), - Expression::JoinedStr(_) => todo!(), - Expression::FormattedValue(_) => todo!(), - }; + let typ = self.get_class_declaration(*s.value.clone(), None); PythonType::Class(ClassType { details: typ, - type_parameters: vec![self.get_type_from_annotation(&s.slice)], + type_parameters, }) } Expression::BinOp(b) => { @@ -420,10 +389,13 @@ impl TypeEvaluator { } /// Retrieves a pythoh type that is present in the builtin scope - fn get_builtin_type(&self, name: &str) -> symbol_table::Class { + fn get_builtin_type(&self, name: &str) -> Option { let builtin_symbol = self.symbol_table.lookup_in_builtin_scope(name); let cls_declaration = match builtin_symbol { - None => panic!("builtin type {} not found", name), + None => { + log::debug!("builtin type {} not found", name); + None + } Some(node) => { // get the declaration with type class node.declarations.iter().find_map(|decl| match decl { @@ -434,8 +406,8 @@ impl TypeEvaluator { }; match cls_declaration { - None => panic!("builtin type {} not found", name), - Some(Declaration::Class(c)) => c.clone(), + None => None, + Some(Declaration::Class(c)) => Some(c), _ => panic!("builtin type {} not found", name), } } @@ -730,6 +702,115 @@ impl TypeEvaluator { false } + + fn get_class_declaration(&self, value: Expression, symbbol_table: Option) -> symbol_table::Class { + let symbol_table = match symbbol_table { + Some(s) => s, + None => self.symbol_table.clone(), + }; + match value { + Expression::Constant(_) => todo!(), + Expression::List(_) => todo!(), + Expression::Tuple(_) => todo!(), + Expression::Dict(_) => todo!(), + Expression::Set(_) => todo!(), + // This is a Generic type with a param: Container[type, ...] + Expression::Name(n) => { + if let Some(builtin_type) = self.get_builtin_type(n.id.as_str()) { + builtin_type + // if it's not a builtin we want to get the class declaration form symbol table + // and find where this class is defined + } else { + let container_decl = + match symbol_table.lookup_in_scope(LookupSymbolRequest { + name: n.id.clone(), + position: Some(n.node.start), + }) { + Some(decl) => decl.last_declaration(), + None => panic!("Type {} has no declaration", n.id), + }; + match container_decl { + Some(Declaration::Class(c)) => c.clone(), + Some(Declaration::Alias(a)) => self.resolve_alias(a), + _ => panic!("Type {} not found {:?}", n.id, container_decl), + } + } + } + Expression::BoolOp(_) => todo!(), + Expression::UnaryOp(_) => todo!(), + Expression::BinOp(_) => todo!(), + Expression::NamedExpr(_) => todo!(), + Expression::Yield(_) => todo!(), + Expression::YieldFrom(_) => todo!(), + Expression::Starred(_) => todo!(), + Expression::Generator(_) => todo!(), + Expression::ListComp(_) => todo!(), + Expression::SetComp(_) => todo!(), + Expression::DictComp(_) => todo!(), + Expression::Attribute(_) => todo!(), + Expression::Subscript(_) => todo!(), + Expression::Slice(_) => todo!(), + Expression::Call(_) => todo!(), + Expression::Await(_) => todo!(), + Expression::Compare(_) => todo!(), + Expression::Lambda(_) => todo!(), + Expression::IfExp(_) => todo!(), + Expression::JoinedStr(_) => todo!(), + Expression::FormattedValue(_) => todo!(), + } + } + + // Follows Alias declaration and resolves it to a class declaration + // It searches through imported symbol tables for the module alias imports + // and resolves the alias to the class declaration + // TODO: refactor to resolve all aliases and not only classes + fn resolve_alias(&self, a: &symbol_table::Alias) -> symbol_table::Class { + let class_name = match a.symbol_name { + Some(ref name) => name.clone(), + None => panic!("Alias {:?} has no symbol name", a.import_node), + }; + let imported_symbol_tables = self.imported_symbol_tables.clone(); + log::debug!( + "Searching for alias {}", + class_name, + ); + for symbol_table in imported_symbol_tables { + // TODO: only check the imports from typing for now. + if symbol_table.module_name + != ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi" + { + continue; + } + if let Some(decl) = symbol_table.lookup_in_scope(LookupSymbolRequest { + name: class_name.clone(), + position: None, + }) { + match decl.last_declaration() { + Some(c) => { + log::debug!("Found declaration: {:?}", c); + match c { + Declaration::Class(c) => return c.clone(), + Declaration::Alias(a) => return self.resolve_alias(a), + Declaration::Variable(v) => { + let type_annotation = v.type_annotation.clone(); + log::debug!("Type annotation for class is {:?} now searching for {:?} in symbol table", type_annotation, type_annotation.clone()); + match type_annotation { + Some(t) => { + log::debug!("Type annotation for class is {:?} now searching for {:?} in symbol table", t, t.clone()); + return self.get_class_declaration(t, Some(symbol_table)); + } + None => continue, + } + } + _ => panic!("Cannot resolve alias to class"), + } + } + _ => panic!("Alias has no declaration"), + } + } + } + panic!("Alias {} not found", class_name); + } } impl TraversalVisitorImmutGeneric for TypeEvaluator { From 513e5126dc054672e70e405ef495cb692aac015a Mon Sep 17 00:00:00 2001 From: Glyphack Date: Mon, 11 Dec 2023 22:21:46 +0100 Subject: [PATCH 05/13] Draft --- ...t_lexer_and_errors@function_def.py-25.snap | 10 +- ...t_lexer_and_errors@function_def.py-26.snap | 431 ++++++++++++++++++ ..._tests__one_liners@function_def.py-25.snap | 6 +- ..._tests__one_liners@function_def.py-26.snap | 335 ++++++++++++++ typechecker/src/build.rs | 9 +- ...aluator__tests__type_evaluator@any.py.snap | 4 +- typechecker/src/type_check/type_evaluator.rs | 234 +++++----- ...sts__symbol_table@class_definition.py.snap | 56 +-- ...__symbol_table@function_definition.py.snap | 194 ++++---- ...build__tests__symbol_table@imports.py.snap | 187 ++++++-- ...symbol_table@simple_var_assignment.py.snap | 2 +- ...ld__tests__symbol_table@type_alias.py.snap | 2 +- ...ild__tests__symbol_table@variables.py.snap | 2 +- 13 files changed, 1167 insertions(+), 305 deletions(-) create mode 100644 parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py-26.snap create mode 100644 parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@function_def.py-26.snap diff --git a/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py-25.snap b/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py-25.snap index 0c5712e7..2cf05734 100644 --- a/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py-25.snap +++ b/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py-25.snap @@ -1,6 +1,6 @@ --- -source: parser/src/lexer/lexer.rs -description: "def a[T: U, *V, **W](): pass\n" +source: parser/src/lexer/mod.rs +description: "def a[T: U, *V, **W](): pass" input_file: parser/test_data/inputs/one_liners/function_def.py --- [ @@ -116,10 +116,4 @@ input_file: parser/test_data/inputs/one_liners/function_def.py start: 24, end: 28, }, - Token { - kind: NewLine, - value: None, - start: 28, - end: 29, - }, ] diff --git a/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py-26.snap b/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py-26.snap new file mode 100644 index 00000000..550fede6 --- /dev/null +++ b/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__snapshot_test_lexer_and_errors@function_def.py-26.snap @@ -0,0 +1,431 @@ +--- +source: parser/src/lexer/mod.rs +description: "def dataclass_transform(\n *,\n eq_default: bool = True,\n order_default: bool = False,\n kw_only_default: bool = False,\n frozen_default: bool = False, # on 3.11, runtime accepts it as part of kwargs\n # this next annotation cannot be parsed need fix\n field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (),\n **kwargs: Any,\n) -> IdentityFunction: ...\n" +input_file: parser/test_data/inputs/one_liners/function_def.py +--- +[ + Token { + kind: Def, + value: None, + start: 0, + end: 3, + }, + Token { + kind: Identifier, + value: Str( + "dataclass_transform", + ), + start: 4, + end: 23, + }, + Token { + kind: LeftParen, + value: None, + start: 23, + end: 24, + }, + Token { + kind: Mul, + value: None, + start: 29, + end: 30, + }, + Token { + kind: Comma, + value: None, + start: 30, + end: 31, + }, + Token { + kind: Identifier, + value: Str( + "eq_default", + ), + start: 36, + end: 46, + }, + Token { + kind: Colon, + value: None, + start: 46, + end: 47, + }, + Token { + kind: Identifier, + value: Str( + "bool", + ), + start: 48, + end: 52, + }, + Token { + kind: Assign, + value: None, + start: 53, + end: 54, + }, + Token { + kind: True, + value: None, + start: 55, + end: 59, + }, + Token { + kind: Comma, + value: None, + start: 59, + end: 60, + }, + Token { + kind: Identifier, + value: Str( + "order_default", + ), + start: 65, + end: 78, + }, + Token { + kind: Colon, + value: None, + start: 78, + end: 79, + }, + Token { + kind: Identifier, + value: Str( + "bool", + ), + start: 80, + end: 84, + }, + Token { + kind: Assign, + value: None, + start: 85, + end: 86, + }, + Token { + kind: False, + value: None, + start: 87, + end: 92, + }, + Token { + kind: Comma, + value: None, + start: 92, + end: 93, + }, + Token { + kind: Identifier, + value: Str( + "kw_only_default", + ), + start: 98, + end: 113, + }, + Token { + kind: Colon, + value: None, + start: 113, + end: 114, + }, + Token { + kind: Identifier, + value: Str( + "bool", + ), + start: 115, + end: 119, + }, + Token { + kind: Assign, + value: None, + start: 120, + end: 121, + }, + Token { + kind: False, + value: None, + start: 122, + end: 127, + }, + Token { + kind: Comma, + value: None, + start: 127, + end: 128, + }, + Token { + kind: Identifier, + value: Str( + "frozen_default", + ), + start: 133, + end: 147, + }, + Token { + kind: Colon, + value: None, + start: 147, + end: 148, + }, + Token { + kind: Identifier, + value: Str( + "bool", + ), + start: 149, + end: 153, + }, + Token { + kind: Assign, + value: None, + start: 154, + end: 155, + }, + Token { + kind: False, + value: None, + start: 156, + end: 161, + }, + Token { + kind: Comma, + value: None, + start: 161, + end: 162, + }, + Token { + kind: Comment, + value: Str( + "# on 3.11, runtime accepts it as part of kwargs", + ), + start: 164, + end: 211, + }, + Token { + kind: Comment, + value: Str( + "# this next annotation cannot be parsed need fix", + ), + start: 216, + end: 264, + }, + Token { + kind: Identifier, + value: Str( + "field_specifiers", + ), + start: 269, + end: 285, + }, + Token { + kind: Colon, + value: None, + start: 285, + end: 286, + }, + Token { + kind: Identifier, + value: Str( + "tuple", + ), + start: 287, + end: 292, + }, + Token { + kind: LeftBrace, + value: None, + start: 292, + end: 293, + }, + Token { + kind: Identifier, + value: Str( + "type", + ), + start: 293, + end: 297, + }, + Token { + kind: LeftBrace, + value: None, + start: 297, + end: 298, + }, + Token { + kind: Identifier, + value: Str( + "Any", + ), + start: 298, + end: 301, + }, + Token { + kind: RightBrace, + value: None, + start: 301, + end: 302, + }, + Token { + kind: BitOr, + value: None, + start: 303, + end: 304, + }, + Token { + kind: Identifier, + value: Str( + "Callable", + ), + start: 305, + end: 313, + }, + Token { + kind: LeftBrace, + value: None, + start: 313, + end: 314, + }, + Token { + kind: Ellipsis, + value: None, + start: 314, + end: 317, + }, + Token { + kind: Comma, + value: None, + start: 317, + end: 318, + }, + Token { + kind: Identifier, + value: Str( + "Any", + ), + start: 319, + end: 322, + }, + Token { + kind: RightBrace, + value: None, + start: 322, + end: 323, + }, + Token { + kind: Comma, + value: None, + start: 323, + end: 324, + }, + Token { + kind: Ellipsis, + value: None, + start: 325, + end: 328, + }, + Token { + kind: RightBrace, + value: None, + start: 328, + end: 329, + }, + Token { + kind: Assign, + value: None, + start: 330, + end: 331, + }, + Token { + kind: LeftParen, + value: None, + start: 332, + end: 333, + }, + Token { + kind: RightParen, + value: None, + start: 333, + end: 334, + }, + Token { + kind: Comma, + value: None, + start: 334, + end: 335, + }, + Token { + kind: Pow, + value: None, + start: 340, + end: 342, + }, + Token { + kind: Identifier, + value: Str( + "kwargs", + ), + start: 342, + end: 348, + }, + Token { + kind: Colon, + value: None, + start: 348, + end: 349, + }, + Token { + kind: Identifier, + value: Str( + "Any", + ), + start: 350, + end: 353, + }, + Token { + kind: Comma, + value: None, + start: 353, + end: 354, + }, + Token { + kind: RightParen, + value: None, + start: 355, + end: 356, + }, + Token { + kind: Arrow, + value: None, + start: 357, + end: 359, + }, + Token { + kind: Identifier, + value: Str( + "IdentityFunction", + ), + start: 360, + end: 376, + }, + Token { + kind: Colon, + value: None, + start: 376, + end: 377, + }, + Token { + kind: Ellipsis, + value: None, + start: 378, + end: 381, + }, + Token { + kind: NewLine, + value: None, + start: 381, + end: 382, + }, +] diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@function_def.py-25.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@function_def.py-25.snap index 298c64d5..8749c2a2 100644 --- a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@function_def.py-25.snap +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@function_def.py-25.snap @@ -1,19 +1,19 @@ --- source: parser/src/parser/parser.rs -description: "def a[T: U, *V, **W](): pass\n" +description: "def a[T: U, *V, **W](): pass" input_file: parser/test_data/inputs/one_liners/function_def.py --- Module { node: Node { start: 0, - end: 29, + end: 28, }, body: [ FunctionDef( FunctionDef { node: Node { start: 0, - end: 29, + end: 28, }, name: "a", args: Arguments { diff --git a/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@function_def.py-26.snap b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@function_def.py-26.snap new file mode 100644 index 00000000..3a3cfb1a --- /dev/null +++ b/parser/src/parser/snapshots/enderpy_python_parser__parser__parser__tests__one_liners@function_def.py-26.snap @@ -0,0 +1,335 @@ +--- +source: parser/src/parser/parser.rs +description: "def dataclass_transform(\n *,\n eq_default: bool = True,\n order_default: bool = False,\n kw_only_default: bool = False,\n frozen_default: bool = False, # on 3.11, runtime accepts it as part of kwargs\n # this next annotation cannot be parsed need fix\n field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (),\n **kwargs: Any,\n) -> IdentityFunction: ...\n" +input_file: parser/test_data/inputs/one_liners/function_def.py +--- +Module { + node: Node { + start: 0, + end: 382, + }, + body: [ + FunctionDef( + FunctionDef { + node: Node { + start: 0, + end: 382, + }, + name: "dataclass_transform", + args: Arguments { + node: Node { + start: 29, + end: 354, + }, + posonlyargs: [], + args: [], + vararg: None, + kwonlyargs: [ + Arg { + node: Node { + start: 36, + end: 59, + }, + arg: "eq_default", + annotation: Some( + Name( + Name { + node: Node { + start: 48, + end: 52, + }, + id: "bool", + }, + ), + ), + }, + Arg { + node: Node { + start: 65, + end: 92, + }, + arg: "order_default", + annotation: Some( + Name( + Name { + node: Node { + start: 80, + end: 84, + }, + id: "bool", + }, + ), + ), + }, + Arg { + node: Node { + start: 98, + end: 127, + }, + arg: "kw_only_default", + annotation: Some( + Name( + Name { + node: Node { + start: 115, + end: 119, + }, + id: "bool", + }, + ), + ), + }, + Arg { + node: Node { + start: 133, + end: 161, + }, + arg: "frozen_default", + annotation: Some( + Name( + Name { + node: Node { + start: 149, + end: 153, + }, + id: "bool", + }, + ), + ), + }, + Arg { + node: Node { + start: 269, + end: 334, + }, + arg: "field_specifiers", + annotation: Some( + Subscript( + Subscript { + node: Node { + start: 287, + end: 329, + }, + value: Name( + Name { + node: Node { + start: 287, + end: 292, + }, + id: "tuple", + }, + ), + slice: Tuple( + Tuple { + node: Node { + start: 293, + end: 329, + }, + elements: [ + BinOp( + BinOp { + node: Node { + start: 293, + end: 323, + }, + op: BitOr, + left: Subscript( + Subscript { + node: Node { + start: 293, + end: 302, + }, + value: Name( + Name { + node: Node { + start: 293, + end: 297, + }, + id: "type", + }, + ), + slice: Name( + Name { + node: Node { + start: 298, + end: 301, + }, + id: "Any", + }, + ), + }, + ), + right: Subscript( + Subscript { + node: Node { + start: 305, + end: 323, + }, + value: Name( + Name { + node: Node { + start: 305, + end: 313, + }, + id: "Callable", + }, + ), + slice: Tuple( + Tuple { + node: Node { + start: 314, + end: 323, + }, + elements: [ + Constant( + Constant { + node: Node { + start: 314, + end: 317, + }, + value: ..., + }, + ), + Name( + Name { + node: Node { + start: 319, + end: 322, + }, + id: "Any", + }, + ), + ], + }, + ), + }, + ), + }, + ), + Constant( + Constant { + node: Node { + start: 325, + end: 328, + }, + value: ..., + }, + ), + ], + }, + ), + }, + ), + ), + }, + ], + kw_defaults: [ + Some( + Constant( + Constant { + node: Node { + start: 55, + end: 59, + }, + value: true, + }, + ), + ), + Some( + Constant( + Constant { + node: Node { + start: 87, + end: 92, + }, + value: false, + }, + ), + ), + Some( + Constant( + Constant { + node: Node { + start: 122, + end: 127, + }, + value: false, + }, + ), + ), + Some( + Constant( + Constant { + node: Node { + start: 156, + end: 161, + }, + value: false, + }, + ), + ), + Some( + Tuple( + Tuple { + node: Node { + start: 332, + end: 334, + }, + elements: [], + }, + ), + ), + ], + kwarg: Some( + Arg { + node: Node { + start: 342, + end: 353, + }, + arg: "kwargs", + annotation: Some( + Name( + Name { + node: Node { + start: 350, + end: 353, + }, + id: "Any", + }, + ), + ), + }, + ), + defaults: [], + }, + body: [ + ExpressionStatement( + Constant( + Constant { + node: Node { + start: 378, + end: 381, + }, + value: ..., + }, + ), + ), + ], + decorator_list: [], + returns: Some( + Name( + Name { + node: Node { + start: 360, + end: 376, + }, + id: "IdentityFunction", + }, + ), + ), + type_comment: None, + type_params: [], + }, + ), + ], +} diff --git a/typechecker/src/build.rs b/typechecker/src/build.rs index aaf1b20c..f111601b 100644 --- a/typechecker/src/build.rs +++ b/typechecker/src/build.rs @@ -342,6 +342,7 @@ impl BuildManager { } } +// TODO: refactor type check tests to be like symbol table tests #[cfg(test)] mod tests { use std::fs; @@ -444,8 +445,12 @@ mod tests { settings.set_snapshot_path("../testdata/output/"); settings.set_description(fs::read_to_string(path).unwrap()); settings.add_filter( - r"/.*/typechecker/test_data/inputs/symbol_table", - "[REDACTED]", + r"/.*/typechecker", + "[TYPECHECKER]", + ); + settings.add_filter( + r"/.*/typeshed", + "[TYPESHED]", ); settings.add_filter( r"module_name: .*.typechecker.test_data.inputs.symbol_table..*.py", diff --git a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@any.py.snap b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@any.py.snap index 2b46b7fb..f7fd846f 100644 --- a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@any.py.snap +++ b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@any.py.snap @@ -7,11 +7,11 @@ input_file: typechecker/src/type_check/test_data/inputs/any.py [ ( "(line: 1, character: 11):(line: 1, character: 12)", - Any, + Unknown, ), ( "(line: 1, character: 11):(line: 1, character: 16)", - Unknown, + Int, ), ( "(line: 1, character: 15):(line: 1, character: 16)", diff --git a/typechecker/src/type_check/type_evaluator.rs b/typechecker/src/type_check/type_evaluator.rs index fa02d4fa..2c5a872e 100755 --- a/typechecker/src/type_check/type_evaluator.rs +++ b/typechecker/src/type_check/type_evaluator.rs @@ -244,9 +244,9 @@ impl TypeEvaluator { } } Expression::Subscript(s) => { - let type_parameters = vec![self.get_type_from_annotation(&s.slice)]; // This is a generic type let typ = self.get_class_declaration(*s.value.clone(), None); + let type_parameters = vec![self.get_type_from_annotation(&s.slice)]; PythonType::Class(ClassType { details: typ, type_parameters, @@ -264,7 +264,6 @@ impl TypeEvaluator { _ => todo!(), } } - _ => PythonType::Unknown, }; @@ -703,11 +702,15 @@ impl TypeEvaluator { false } - fn get_class_declaration(&self, value: Expression, symbbol_table: Option) -> symbol_table::Class { + fn get_class_declaration( + &self, + value: Expression, + symbbol_table: Option, + ) -> symbol_table::Class { let symbol_table = match symbbol_table { Some(s) => s, None => self.symbol_table.clone(), - }; + }; match value { Expression::Constant(_) => todo!(), Expression::List(_) => todo!(), @@ -721,17 +724,25 @@ impl TypeEvaluator { // if it's not a builtin we want to get the class declaration form symbol table // and find where this class is defined } else { - let container_decl = - match symbol_table.lookup_in_scope(LookupSymbolRequest { - name: n.id.clone(), - position: Some(n.node.start), - }) { - Some(decl) => decl.last_declaration(), - None => panic!("Type {} has no declaration", n.id), - }; + let container_decl = match symbol_table.lookup_in_scope(LookupSymbolRequest { + name: n.id.clone(), + position: Some(n.node.start), + }) { + Some(decl) => decl.last_declaration(), + None => panic!("Type {} has no declaration", n.id), + }; match container_decl { Some(Declaration::Class(c)) => c.clone(), - Some(Declaration::Alias(a)) => self.resolve_alias(a), + Some(Declaration::Alias(a)) => { + if let Some(decl) = self.resolve_alias(a) { + match decl.last_declaration() { + Some(Declaration::Class(c)) => c.clone(), + _ => panic!("Alias {:?} is not a class", a), + } + } else { + panic!("Alias {:?} not found", a) + } + } _ => panic!("Type {} not found {:?}", n.id, container_decl), } } @@ -764,49 +775,19 @@ impl TypeEvaluator { // It searches through imported symbol tables for the module alias imports // and resolves the alias to the class declaration // TODO: refactor to resolve all aliases and not only classes - fn resolve_alias(&self, a: &symbol_table::Alias) -> symbol_table::Class { + fn resolve_alias(&self, a: &symbol_table::Alias) -> Option<&symbol_table::SymbolTableNode> { let class_name = match a.symbol_name { Some(ref name) => name.clone(), None => panic!("Alias {:?} has no symbol name", a.import_node), }; let imported_symbol_tables = self.imported_symbol_tables.clone(); - log::debug!( - "Searching for alias {}", - class_name, - ); + log::debug!("Searching for alias {}", class_name,); for symbol_table in imported_symbol_tables { - // TODO: only check the imports from typing for now. - if symbol_table.module_name - != ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi" - { - continue; - } if let Some(decl) = symbol_table.lookup_in_scope(LookupSymbolRequest { name: class_name.clone(), position: None, }) { - match decl.last_declaration() { - Some(c) => { - log::debug!("Found declaration: {:?}", c); - match c { - Declaration::Class(c) => return c.clone(), - Declaration::Alias(a) => return self.resolve_alias(a), - Declaration::Variable(v) => { - let type_annotation = v.type_annotation.clone(); - log::debug!("Type annotation for class is {:?} now searching for {:?} in symbol table", type_annotation, type_annotation.clone()); - match type_annotation { - Some(t) => { - log::debug!("Type annotation for class is {:?} now searching for {:?} in symbol table", t, t.clone()); - return self.get_class_declaration(t, Some(symbol_table)); - } - None => continue, - } - } - _ => panic!("Cannot resolve alias to class"), - } - } - _ => panic!("Alias has no declaration"), - } + return Some(decl); } } panic!("Alias {} not found", class_name); @@ -1085,26 +1066,98 @@ impl TraversalVisitorImmutGeneric for TypeEvaluator { } } -/// visits the ast and calls get_type on each expression and saves that type in -/// the types hashmap the key is the position of the expression in the source: -/// (line, start, end) -struct TypeEvalVisitor { +#[cfg(test)] +mod tests { + use insta::glob; + + use crate::build::BuildManager; + use crate::build_source::BuildSource; + use crate::settings::Settings; + + use super::*; + use std::fs; + use std::path::PathBuf; + + fn snapshot_type_eval(source: &str) -> String { + use enderpy_python_parser::Parser; + + let mut parser = Parser::new(source.to_string(), "".into()); + let ast_module = parser.parse(); + let build_source = BuildSource { + path: PathBuf::from("test-file"), + source: source.to_string(), + module: "test".to_string(), + followed: false, + }; + + // we use the manager to also import the python typeshed into moduels + // This can be refactored but for now it's fine + let mut manager = BuildManager::new(vec![build_source], Settings::test_settings()); + manager.build(); + + let mut all_symbol_tables = Vec::new(); + for module in manager.modules.values() { + all_symbol_tables.push(module.get_symbol_table()); + } + + let module = manager.get_state("test-file".into()).unwrap(); + let symbol_table = module.get_symbol_table(); + + let type_eval = TypeEvaluator { + symbol_table, + imported_symbol_tables: all_symbol_tables, + }; + + let mut type_eval_visitor = DumpTypes::new(module.file.clone(), type_eval); + type_eval_visitor.visit_module(); + + let result = type_eval_visitor.types; + + // sort result by key + let mut result_sorted = result.clone().into_iter().collect::>(); + result_sorted.sort_by(|a, b| a.0.cmp(&b.0)); + + format!("{:#?}", result_sorted) + } + + #[test] + fn test_type_evaluator() { + glob!("test_data/inputs/", "*.py", |path| { + let contents = fs::read_to_string(path).unwrap(); + let result = snapshot_type_eval(&contents); + let _ = env_logger::builder() + .filter_level(log::LevelFilter::Debug) + .is_test(true) + .try_init(); + + let mut settings = insta::Settings::clone_current(); + settings.set_snapshot_path("./test_data/output/"); + settings.set_description(fs::read_to_string(path).unwrap()); + settings.bind(|| { + insta::assert_snapshot!(result); + }); + }) + } +} + +/// visits the ast and calls get_type on each expression and saves that type in the types hashmap +/// the key is the position of the expression in the source: (line, start, end) +struct DumpTypes { pub type_eval: TypeEvaluator, pub types: HashMap, pub state: State, } -impl TypeEvalVisitor { - pub fn new(enderpy_file: EnderpyFile) -> Self { +impl DumpTypes { + pub fn new(enderpy_file: EnderpyFile, type_eval: TypeEvaluator) -> Self { let mut state = State::new(enderpy_file); + // TODO: this line runs on every test and it needs to add all of stdlib to symbol table. + // This is not efficient and needs to be refactored state.populate_symbol_table(); let symbol_table = state.get_symbol_table(); Self { types: HashMap::new(), - type_eval: TypeEvaluator { - symbol_table, - imported_symbol_tables: vec![], - }, + type_eval, state, } } @@ -1140,7 +1193,7 @@ impl TypeEvalVisitor { } /// Traverse the ast and call call save_type on each expression -impl TraversalVisitor for TypeEvalVisitor { +impl TraversalVisitor for DumpTypes { fn visit_stmt(&mut self, s: &ast::Statement) { // map all statements and call visit match s { @@ -1234,70 +1287,3 @@ impl TraversalVisitor for TypeEvalVisitor { } } } - -#[cfg(test)] -mod tests { - use std::{fs, path::PathBuf}; - - use insta::glob; - - use super::*; - use crate::build_source::BuildSource; - - fn snapshot_type_eval(source: &str) -> String { - use enderpy_python_parser::Parser; - - let mut parser = Parser::new(source.to_string(), "".into()); - let ast_module = parser.parse(); - - let enderpy_file = EnderpyFile::from( - ast_module, - Box::new(BuildSource { - path: PathBuf::from(""), - source: source.to_string(), - module: "test".to_string(), - followed: false, - }), - vec![], - ); - - let mut module = State::new(enderpy_file); - module.populate_symbol_table(); - let symbol_table = module.get_symbol_table(); - - let type_eval = TypeEvaluator { - symbol_table, - imported_symbol_tables: vec![], - }; - - let mut type_eval_visitor = TypeEvalVisitor::new(module.file); - type_eval_visitor.visit_module(); - - let result = type_eval_visitor.types; - - // sort result by key - let mut result_sorted = result.clone().into_iter().collect::>(); - result_sorted.sort_by(|a, b| a.0.cmp(&b.0)); - - format!("{:#?}", result_sorted) - } - - #[test] - fn test_type_evaluator() { - glob!("test_data/inputs/", "*.py", |path| { - let contents = fs::read_to_string(path).unwrap(); - let result = snapshot_type_eval(&contents); - let _ = env_logger::builder() - .filter_level(log::LevelFilter::Debug) - .is_test(true) - .try_init(); - - let mut settings = insta::Settings::clone_current(); - settings.set_snapshot_path("./test_data/output/"); - settings.set_description(fs::read_to_string(path).unwrap()); - settings.bind(|| { - insta::assert_snapshot!(result); - }); - }) - } -} diff --git a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@class_definition.py.snap b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@class_definition.py.snap index 6c90df28..9df594d9 100644 --- a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@class_definition.py.snap +++ b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@class_definition.py.snap @@ -6,34 +6,6 @@ input_file: typechecker/test_data/inputs/symbol_table/class_definition.py --- ------------------- global scope: -Symbols: in global (id: [REDACTED]) -c -- Declarations: ---: Class { - name: "c", - declaration_path: DeclarationPath { - module_name: [REDACTED]", - node: Node { - start: 0, - end: 80, - }, - }, - methods: [ - "__init__", - ], - attributes: { - "c": Name( - Name { - node: Node { - start: 78, - end: 79, - }, - id: "b", - }, - ), - }, -} - all scopes: Symbols: in __init__ (id: [REDACTED]) a @@ -254,5 +226,33 @@ __init__ raise_statements: [], } +Symbols: in global (id: [REDACTED]) +c +- Declarations: +--: Class { + name: "c", + declaration_path: DeclarationPath { + module_name: [REDACTED]", + node: Node { + start: 0, + end: 80, + }, + }, + methods: [ + "__init__", + ], + attributes: { + "c": Name( + Name { + node: Node { + start: 78, + end: 79, + }, + id: "b", + }, + ), + }, +} + ------------------- diff --git a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@function_definition.py.snap b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@function_definition.py.snap index 134f0507..b53298ce 100644 --- a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@function_definition.py.snap +++ b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@function_definition.py.snap @@ -6,103 +6,6 @@ input_file: typechecker/test_data/inputs/symbol_table/function_definition.py --- ------------------- global scope: -Symbols: in global (id: [REDACTED]) -func -- Declarations: ---: Function { - declaration_path: DeclarationPath { - module_name: [REDACTED]", - node: Node { - start: 0, - end: 37, - }, - }, - function_node: FunctionDef { - node: Node { - start: 0, - end: 37, - }, - name: "func", - args: Arguments { - node: Node { - start: 9, - end: 29, - }, - posonlyargs: [ - Arg { - node: Node { - start: 9, - end: 10, - }, - arg: "a", - annotation: None, - }, - Arg { - node: Node { - start: 12, - end: 13, - }, - arg: "b", - annotation: None, - }, - ], - args: [ - Arg { - node: Node { - start: 19, - end: 24, - }, - arg: "c", - annotation: None, - }, - ], - vararg: None, - kwonlyargs: [], - kw_defaults: [], - kwarg: Some( - Arg { - node: Node { - start: 28, - end: 29, - }, - arg: "e", - annotation: None, - }, - ), - defaults: [ - Constant( - Constant { - node: Node { - start: 23, - end: 24, - }, - value: 2, - }, - ), - ], - }, - body: [ - Pass( - Pass { - node: Node { - start: 32, - end: 36, - }, - }, - ), - ], - decorator_list: [], - returns: None, - type_comment: None, - type_params: [], - }, - is_method: false, - is_generator: false, - return_statements: [], - yeild_statements: [], - raise_statements: [], -} - all scopes: Symbols: in func (id: [REDACTED]) a @@ -200,5 +103,102 @@ e default_value: None, } +Symbols: in global (id: [REDACTED]) +func +- Declarations: +--: Function { + declaration_path: DeclarationPath { + module_name: [REDACTED]", + node: Node { + start: 0, + end: 37, + }, + }, + function_node: FunctionDef { + node: Node { + start: 0, + end: 37, + }, + name: "func", + args: Arguments { + node: Node { + start: 9, + end: 29, + }, + posonlyargs: [ + Arg { + node: Node { + start: 9, + end: 10, + }, + arg: "a", + annotation: None, + }, + Arg { + node: Node { + start: 12, + end: 13, + }, + arg: "b", + annotation: None, + }, + ], + args: [ + Arg { + node: Node { + start: 19, + end: 24, + }, + arg: "c", + annotation: None, + }, + ], + vararg: None, + kwonlyargs: [], + kw_defaults: [], + kwarg: Some( + Arg { + node: Node { + start: 28, + end: 29, + }, + arg: "e", + annotation: None, + }, + ), + defaults: [ + Constant( + Constant { + node: Node { + start: 23, + end: 24, + }, + value: 2, + }, + ), + ], + }, + body: [ + Pass( + Pass { + node: Node { + start: 32, + end: 36, + }, + }, + ), + ], + decorator_list: [], + returns: None, + type_comment: None, + type_params: [], + }, + is_method: false, + is_generator: false, + return_statements: [], + yeild_statements: [], + raise_statements: [], +} + ------------------- diff --git a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@imports.py.snap b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@imports.py.snap index deb5116e..61966151 100644 --- a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@imports.py.snap +++ b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@imports.py.snap @@ -6,6 +6,7 @@ input_file: typechecker/test_data/inputs/symbol_table/imports.py --- ------------------- global scope: +all scopes: Symbols: in global (id: [REDACTED]) * - Declarations: @@ -50,10 +51,10 @@ Symbols: in global (id: [REDACTED]) is_stub_package: false, import_type: Local, resolved_paths: [ - "[REDACTED]/variables.py", + "[TYPECHECKER]/test_data/inputs/symbol_table/variables.py", ], search_path: Some( - "[REDACTED]", + "[TYPECHECKER]/test_data/inputs/symbol_table", ), is_stub_file: false, is_native_lib: false, @@ -105,28 +106,69 @@ Symbols: in global (id: [REDACTED]) ), import_result: ImportResult { is_relative: false, - is_import_found: false, + is_import_found: true, is_partly_resolved: false, is_namespace_package: false, - is_init_file_present: false, + is_init_file_present: true, is_stub_package: false, - import_type: Local, - resolved_paths: [], - search_path: None, - is_stub_file: false, + import_type: BuiltIn, + resolved_paths: [ + "[TYPESHED]/stdlib/os/__init__.pyi", + ], + search_path: Some( + "[TYPESHED]/stdlib", + ), + is_stub_file: true, is_native_lib: false, - is_stdlib_typeshed_file: false, + is_stdlib_typeshed_file: true, is_third_party_typeshed_file: false, is_local_typings_file: false, implicit_imports: ImplicitImports( - {}, + { + "path": ImplicitImport { + is_stub_file: true, + is_native_lib: false, + path: "[TYPESHED]/stdlib/os/path.pyi", + py_typed: None, + }, + }, ), filtered_implicit_imports: ImplicitImports( {}, ), - non_stub_import_result: None, + non_stub_import_result: Some( + ImportResult { + is_relative: false, + is_import_found: false, + is_partly_resolved: false, + is_namespace_package: false, + is_init_file_present: false, + is_stub_package: false, + import_type: Local, + resolved_paths: [], + search_path: Some( + "", + ), + is_stub_file: false, + is_native_lib: false, + is_stdlib_typeshed_file: false, + is_third_party_typeshed_file: false, + is_local_typings_file: false, + implicit_imports: ImplicitImports( + {}, + ), + filtered_implicit_imports: ImplicitImports( + {}, + ), + non_stub_import_result: None, + py_typed_info: None, + package_directory: None, + }, + ), py_typed_info: None, - package_directory: None, + package_directory: Some( + "[TYPESHED]/stdlib/os", + ), }, } a @@ -172,10 +214,10 @@ a is_stub_package: false, import_type: Local, resolved_paths: [ - "[REDACTED]/variables.py", + "[TYPECHECKER]/test_data/inputs/symbol_table/variables.py", ], search_path: Some( - "[REDACTED]", + "[TYPECHECKER]/test_data/inputs/symbol_table", ), is_stub_file: false, is_native_lib: false, @@ -232,10 +274,10 @@ import_test is_stub_package: false, import_type: Local, resolved_paths: [ - "[REDACTED]/import_test/__init__.py", + "[TYPECHECKER]/test_data/inputs/symbol_table/import_test/__init__.py", ], search_path: Some( - "[REDACTED]", + "[TYPECHECKER]/test_data/inputs/symbol_table", ), is_stub_file: false, is_native_lib: false, @@ -251,7 +293,7 @@ import_test non_stub_import_result: None, py_typed_info: None, package_directory: Some( - "[REDACTED]/import_test", + "[TYPECHECKER]/test_data/inputs/symbol_table/import_test", ), }, } @@ -291,17 +333,22 @@ join ), import_result: ImportResult { is_relative: false, - is_import_found: false, + is_import_found: true, is_partly_resolved: false, is_namespace_package: false, - is_init_file_present: false, + is_init_file_present: true, is_stub_package: false, - import_type: Local, - resolved_paths: [], - search_path: None, - is_stub_file: false, + import_type: BuiltIn, + resolved_paths: [ + "[TYPESHED]/stdlib/os/__init__.pyi", + "[TYPESHED]/stdlib/os/path.pyi", + ], + search_path: Some( + "[TYPESHED]/stdlib", + ), + is_stub_file: true, is_native_lib: false, - is_stdlib_typeshed_file: false, + is_stdlib_typeshed_file: true, is_third_party_typeshed_file: false, is_local_typings_file: false, implicit_imports: ImplicitImports( @@ -310,9 +357,39 @@ join filtered_implicit_imports: ImplicitImports( {}, ), - non_stub_import_result: None, + non_stub_import_result: Some( + ImportResult { + is_relative: false, + is_import_found: false, + is_partly_resolved: false, + is_namespace_package: false, + is_init_file_present: false, + is_stub_package: false, + import_type: Local, + resolved_paths: [], + search_path: Some( + "", + ), + is_stub_file: false, + is_native_lib: false, + is_stdlib_typeshed_file: false, + is_third_party_typeshed_file: false, + is_local_typings_file: false, + implicit_imports: ImplicitImports( + {}, + ), + filtered_implicit_imports: ImplicitImports( + {}, + ), + non_stub_import_result: None, + py_typed_info: None, + package_directory: None, + }, + ), py_typed_info: None, - package_directory: None, + package_directory: Some( + "[TYPESHED]/stdlib/os", + ), }, } os.path @@ -347,17 +424,22 @@ os.path symbol_name: None, import_result: ImportResult { is_relative: false, - is_import_found: false, + is_import_found: true, is_partly_resolved: false, is_namespace_package: false, - is_init_file_present: false, + is_init_file_present: true, is_stub_package: false, - import_type: Local, - resolved_paths: [], - search_path: None, - is_stub_file: false, + import_type: BuiltIn, + resolved_paths: [ + "[TYPESHED]/stdlib/os/__init__.pyi", + "[TYPESHED]/stdlib/os/path.pyi", + ], + search_path: Some( + "[TYPESHED]/stdlib", + ), + is_stub_file: true, is_native_lib: false, - is_stdlib_typeshed_file: false, + is_stdlib_typeshed_file: true, is_third_party_typeshed_file: false, is_local_typings_file: false, implicit_imports: ImplicitImports( @@ -366,9 +448,39 @@ os.path filtered_implicit_imports: ImplicitImports( {}, ), - non_stub_import_result: None, + non_stub_import_result: Some( + ImportResult { + is_relative: false, + is_import_found: false, + is_partly_resolved: false, + is_namespace_package: false, + is_init_file_present: false, + is_stub_package: false, + import_type: Local, + resolved_paths: [], + search_path: Some( + "", + ), + is_stub_file: false, + is_native_lib: false, + is_stdlib_typeshed_file: false, + is_third_party_typeshed_file: false, + is_local_typings_file: false, + implicit_imports: ImplicitImports( + {}, + ), + filtered_implicit_imports: ImplicitImports( + {}, + ), + non_stub_import_result: None, + py_typed_info: None, + package_directory: None, + }, + ), py_typed_info: None, - package_directory: None, + package_directory: Some( + "[TYPESHED]/stdlib/os", + ), }, } variables @@ -410,10 +522,10 @@ variables is_stub_package: false, import_type: Local, resolved_paths: [ - "[REDACTED]/variables.py", + "[TYPECHECKER]/test_data/inputs/symbol_table/variables.py", ], search_path: Some( - "[REDACTED]", + "[TYPECHECKER]/test_data/inputs/symbol_table", ), is_stub_file: false, is_native_lib: false, @@ -432,6 +544,5 @@ variables }, } -all scopes: ------------------- diff --git a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@simple_var_assignment.py.snap b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@simple_var_assignment.py.snap index 008b1fe6..d1f03fd6 100644 --- a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@simple_var_assignment.py.snap +++ b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@simple_var_assignment.py.snap @@ -6,6 +6,7 @@ input_file: typechecker/test_data/inputs/symbol_table/simple_var_assignment.py --- ------------------- global scope: +all scopes: Symbols: in global (id: [REDACTED]) a - Declarations: @@ -146,6 +147,5 @@ f is_constant: false, } -all scopes: ------------------- diff --git a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@type_alias.py.snap b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@type_alias.py.snap index b2cb7c31..4b34b057 100644 --- a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@type_alias.py.snap +++ b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@type_alias.py.snap @@ -6,6 +6,7 @@ input_file: typechecker/test_data/inputs/symbol_table/type_alias.py --- ------------------- global scope: +all scopes: Symbols: in global (id: [REDACTED]) Alias1 - Declarations: @@ -120,6 +121,5 @@ AliasToAnotherAlias }, } -all scopes: ------------------- diff --git a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@variables.py.snap b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@variables.py.snap index 9cf95a03..27603206 100644 --- a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@variables.py.snap +++ b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@variables.py.snap @@ -6,6 +6,7 @@ input_file: typechecker/test_data/inputs/symbol_table/variables.py --- ------------------- global scope: +all scopes: Symbols: in global (id: [REDACTED]) a - Declarations: @@ -33,6 +34,5 @@ a is_constant: false, } -all scopes: ------------------- From 14eec5d3cca81f3acf307b724dc94b2a5d7fafcf Mon Sep 17 00:00:00 2001 From: Glyphack Date: Thu, 4 Jan 2024 23:55:49 +0100 Subject: [PATCH 06/13] Resolved alias --- parser/src/lexer/mod.rs | 2 +- parser/src/parser/ast.rs | 1 - typechecker/src/build.rs | 367 ++++++++++-------- typechecker/src/build_source.rs | 11 +- typechecker/src/nodes.rs | 47 ++- .../module_descriptor.rs | 2 +- typechecker/src/semantic_analyzer.rs | 36 +- typechecker/src/state.rs | 62 +-- typechecker/src/symbol_table.rs | 28 +- .../type_check/test_data/inputs/literal.py | 9 + ..._tests__type_evaluator@literal.py.snap.new | 47 +++ ...uator__tests__type_evaluator@union.py.snap | 47 ++- typechecker/src/type_check/type_evaluator.rs | 92 ++++- 13 files changed, 445 insertions(+), 306 deletions(-) mode change 100644 => 100755 typechecker/src/build.rs create mode 100644 typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap.new diff --git a/parser/src/lexer/mod.rs b/parser/src/lexer/mod.rs index 59407374..b5c88c8b 100644 --- a/parser/src/lexer/mod.rs +++ b/parser/src/lexer/mod.rs @@ -1206,7 +1206,7 @@ def", f = c - # Path: test_local.py + # Path: test_local.py ", ], ) diff --git a/parser/src/parser/ast.rs b/parser/src/parser/ast.rs index af8676ef..03edbcff 100644 --- a/parser/src/parser/ast.rs +++ b/parser/src/parser/ast.rs @@ -346,7 +346,6 @@ impl fmt::Debug for ConstantValue { let mut s = s.chars().skip(1).collect::(); s = s.chars().take(s.len() - 1).collect::(); - // drop first another char from beginning s = s.chars().skip(1).collect::(); return write!(f, "{:?}", s); diff --git a/typechecker/src/build.rs b/typechecker/src/build.rs old mode 100644 new mode 100755 index f111601b..17b0807b --- a/typechecker/src/build.rs +++ b/typechecker/src/build.rs @@ -1,16 +1,17 @@ use std::{collections::HashMap, path::PathBuf}; -use enderpy_python_parser::{error::ParsingError, Parser}; +use enderpy_python_parser::error::ParsingError; use env_logger::Builder; use log::info; use crate::{ build_source::BuildSource, diagnostic::Diagnostic, - nodes::EnderpyFile, + nodes::{EnderpyFile, ImportKinds}, ruff_python_import_resolver as ruff_python_resolver, ruff_python_import_resolver::{ - config::Config, execution_environment, module_descriptor::ImportModuleDescriptor, resolver, + config::Config, execution_environment, import_result::ImportResult, + module_descriptor::ImportModuleDescriptor, resolver, }, settings::Settings, state::State, @@ -65,66 +66,39 @@ impl BuildManager { } None } - pub fn parse(&self, build_source: &BuildSource) -> EnderpyFile { - let file_path = build_source.path.to_str().unwrap_or(""); - let mut parser = Parser::new(build_source.source.clone(), file_path.into()); - let tree = parser.parse(); - EnderpyFile::from(tree, Box::new(build_source.clone()), parser.errors) - } // Entry point to analyze the program pub fn build(&mut self) { self.populate_modules(); - self.pre_analysis(); - } - - // Performs pre-analysis on the source files - // Fills up the symbol table for each module - fn pre_analysis(&mut self) { - let execution_environment = &execution_environment::ExecutionEnvironment { - root: self.options.root.clone(), - python_version: ruff_python_resolver::python_version::PythonVersion::Py312, - python_platform: ruff_python_resolver::python_platform::PythonPlatform::Darwin, - // Adding a blank path to the extra paths is a hack to make the resolver work - extra_paths: vec![PathBuf::from("")], - }; - - let import_config = &Config { - typeshed_path: self.options.import_discovery.typeshed_path.clone(), - stub_path: None, - venv_path: Some(self.options.root.clone()), - venv: None, - }; - let host = &ruff_python_resolver::host::StaticHost::new(vec![]); - for state in self.modules.iter_mut() { - state - .1 - .resolve_file_imports(execution_environment, import_config, host); - state.1.populate_symbol_table(); - } } + // Resolves imports in all files and then populates the symbol table fn populate_modules(&mut self) { for build_source in self.build_sources.iter() { - let file = self.parse(build_source); - let state = State::new(file); - self.modules.insert(build_source.module.clone(), state); + let build_source: BuildSource = build_source.clone(); + let state = State::new(build_source.into()); + self.modules.insert(state.file.module_name(), state); } - let initial_files = self.modules.values().collect(); - let new_files = match self.options.follow_imports { - crate::settings::FollowImports::All => self.gather_files(initial_files, true), - crate::settings::FollowImports::Skip => self.gather_files(initial_files, false), + let (new_files, imports) = match self.options.follow_imports { + crate::settings::FollowImports::All => { + self.gather_files(self.build_sources.clone(), true) + } + crate::settings::FollowImports::Skip => { + self.gather_files(self.build_sources.clone(), false) + } }; - for file in new_files { - self.modules.insert(file.file.module_name().clone(), file); + for build_source in new_files { + let module = self.create_module(build_source); + self.modules.insert(module.file.module_name(), module); } - for module in self.modules.values() { + for module in self.modules.values_mut() { info!("file: {:#?}", module.file.module_name()); + module.populate_symbol_table(&imports); } } // Performs type checking passes over the code - // This step hapens after the binding phase + // This step happens after the binding phase pub fn type_check(&mut self) { self.build(); // TODO: This is a hack to get all the symbol tables so we can resolve imports @@ -188,7 +162,16 @@ impl BuildManager { } } - fn gather_files(&self, current_files: Vec<&State>, add_indirect_imports: bool) -> Vec { + // Given a list of files, this function will resolve the imports in the files + // and add them to the modules. + fn gather_files( + &self, + initial_files: Vec, + add_indirect_imports: bool, + ) -> ( + Vec, + HashMap, + ) { let execution_environment = &execution_environment::ExecutionEnvironment { root: self.options.root.clone(), python_version: ruff_python_resolver::python_version::PythonVersion::Py312, @@ -203,142 +186,196 @@ impl BuildManager { venv: None, }; + let mut files_to_resolve: Vec = vec![]; + files_to_resolve.extend(initial_files); + let mut import_results = HashMap::new(); + let mut imported_sources = Vec::new(); + log::debug!("import options: {:?}", execution_environment); - let mut new_imports = vec![]; - let mut discovered_files = vec![]; - for state in current_files { - let resolved_imports = - self.resolve_file_imports(state, execution_environment, import_config); + let mut new_imports: Vec = vec![]; + let host = &ruff_python_resolver::host::StaticHost::new(vec![]); + loop { + let enderpy_file = match files_to_resolve.pop() { + Some(source) => EnderpyFile::from(source), + None => break, + }; + let resolved_imports = self.resolve_file_imports( + enderpy_file, + execution_environment, + import_config, + host, + &import_results, + ); // check if the resolved_imports are not in the current files and add them to // the new imports - for (_, state) in resolved_imports { - if !self.modules.contains_key(&state.file.module_name()) { - new_imports.push(state); - } - } - } - - if !add_indirect_imports { - return discovered_files; - } - - discovered_files.extend(new_imports.clone()); - - while !new_imports.is_empty() { - let mut next_imports = vec![]; - for state in new_imports { - let resolved_imports = - self.resolve_file_imports(&state, execution_environment, import_config); - // check if the resolved_imports are not in the current files and add them to - // the new imports - for (_, state) in resolved_imports { - if !self.modules.contains_key(&state.file.module_name()) { - // check no discovered file with the same name exists - if !discovered_files - .iter() - .any(|x| x.file.module_name() == state.file.module_name()) + for (import_desc, resolved) in resolved_imports { + import_results.insert(import_desc, resolved.clone()); + if resolved.is_import_found { + for resolved_path in resolved.resolved_paths.iter() { + let source = match std::fs::read_to_string(resolved_path) { + Ok(source) => source, + Err(e) => { + log::warn!("cannot read file: {}", e); + continue; + } + }; + let build_source = match BuildSource::from_path(resolved_path.clone(), true) { - next_imports.push(state); - } + Ok(build_source) => { + imported_sources.push(build_source.clone()); + files_to_resolve.push(build_source); + } + Err(e) => { + log::warn!("cannot read file: {}", e); + continue; + } + }; } + + // TODO: not sure if this part is needed. Need to check when we have more tests + // on builder. This is supposed to add implicit imports to the build sources + // implicit import example: import foo.bar + // foo/bar/__init__.py + // In this case, we need to add foo/bar/__init__.py to the build sources + // for (name, implicit_import) in resolved.implicit_imports.iter() { + // if self + // .modules + // .contains_key(&get_module_name(&implicit_import.path)) + // { + // log::debug!( + // "implicit import already exists: {}", + // get_module_name(&implicit_import.path) + // ); + // continue; + // } + // let source = std::fs::read_to_string(implicit_import.path.clone()).unwrap(); + // let build_source = + // BuildSource::from_path(implicit_import.path.clone(), true).unwrap(); + // self.build_sources.push(build_source); + // self.add_to_modules(&build_source); + // match self.modules.get(&build_source.module) { + // Some(discovered_module) => { + // if !self.modules.contains_key(&build_source.module) { + // new_imports.push(discovered_module); + // } + // } + // None => { + // panic!("cannot find module: {}", build_source.module); + // } + // } + // } } } - discovered_files.extend(next_imports.clone()); - new_imports = next_imports; } - discovered_files + + // if !add_indirect_imports { + // return; + // } + // + // // Follow the import files and add them to the modules as well + // while !new_imports.is_empty() { + // let mut next_imports = vec![]; + // for state in new_imports { + // let resolved_imports = state.resolve_file_imports( + // execution_environment, + // import_config, + // host, + // &cached_imports, + // ); + // // check if the resolved_imports are not in the current files and add them to + // // the new imports + // for (_, resolved_import) in resolved_imports { + // if !resolved_import.is_import_found { + // continue; + // } + // for resolved_path in resolved_import.resolved_paths { + // if self.modules.contains_key(&get_module_name(&resolved_path)) { + // log::debug!("imported file already in modules: {:?}", resolved_path); + // continue; + // } + // let build_source = match BuildSource::from_path(resolved_path, true) { + // Ok(build_source) => build_source, + // Err(e) => { + // log::warn!("cannot read file: {}", e); + // continue; + // } + // }; + // match self.modules.get(&build_source.module) { + // Some(state) => { + // if !self.modules.contains_key(&build_source.module) { + // next_imports.push(state); + // } + // } + // None => { + // panic!("cannot find module: {}", build_source.module); + // } + // } + // } + // } + // } + // new_imports = next_imports; + // } + (imported_sources, import_results) + } + + // TODO: refactor to implement From/to trait + fn create_module(&self, build_source: BuildSource) -> State { + let state = State::new(EnderpyFile::from(build_source)); + state } - // Resolves imports in a file and return the resolved paths - // TODO: This function is doing duplicate work because we resolve the imports in - // the State module as well. We should refactor this and possibly only do it - // in the State module fn resolve_file_imports( &self, - state: &State, - execution_environment: &execution_environment::ExecutionEnvironment, - import_config: &Config, - ) -> HashMap { - let host = &ruff_python_resolver::host::StaticHost::new(vec![]); - let mut resolved_paths = HashMap::new(); - let mut resolved_imports = vec![]; - for import in state.file.imports.iter() { + file: EnderpyFile, + execution_environment: &ruff_python_resolver::execution_environment::ExecutionEnvironment, + import_config: &ruff_python_resolver::config::Config, + host: &ruff_python_resolver::host::StaticHost, + cached_imports: &HashMap, + ) -> HashMap { + log::debug!("resolving imports for file: {}", file.module_name()); + let mut imports = HashMap::new(); + for import in file.imports.iter() { let import_descriptions = match import { - crate::nodes::ImportKinds::Import(i) => i + ImportKinds::Import(i) => i .names .iter() - .map( - |x| ruff_python_resolver::module_descriptor::ImportModuleDescriptor { - leading_dots: 0, - name_parts: x - .name - .chars() - .skip_while(|c| *c == '.') - .collect::() - .split('.') - .map(std::string::ToString::to_string) - .collect(), - imported_symbols: vec![], - }, - ) + .map(|x| { + ruff_python_resolver::module_descriptor::ImportModuleDescriptor::from(x) + }) .collect::>(), - crate::nodes::ImportKinds::ImportFrom(i) => { - vec![ - ruff_python_resolver::module_descriptor::ImportModuleDescriptor { - leading_dots: i.level, - name_parts: i - .module - .chars() - .skip_while(|c| *c == '.') - .collect::() - .split('.') - .map(std::string::ToString::to_string) - .collect(), - imported_symbols: i.names.iter().map(|x| x.name.clone()).collect(), - }, - ] + ImportKinds::ImportFrom(i) => { + vec![ruff_python_resolver::module_descriptor::ImportModuleDescriptor::from(i)] } }; + log::debug!("import descriptions: {:?}", import_descriptions); for import_desc in import_descriptions { - let mut resolved = resolver::resolve_import( - state.file.path().as_path(), - execution_environment, - &import_desc, - import_config, - host, - ); - if resolved.is_import_found { - for resolved_path in resolved.resolved_paths.iter() { - let source = match std::fs::read_to_string(resolved_path) { - Ok(source) => source, - Err(e) => { - log::warn!("cannot read file: {}", e); - continue; - } - }; - let build_source = - BuildSource::from_path(resolved_path.clone(), true).unwrap(); - resolved_imports.push(build_source); - } - - for (name, implicit_import) in resolved.implicit_imports.iter() { - let source = std::fs::read_to_string(implicit_import.path.clone()).unwrap(); - let build_source = - BuildSource::from_path(implicit_import.path.clone(), true).unwrap(); - resolved_imports.push(build_source); - } + let resolved = match cached_imports.contains_key(&import_desc) { + true => continue, + false => resolver::resolve_import( + file.path().as_path(), + execution_environment, + &import_desc, + import_config, + host, + ), + }; + + if !resolved.is_import_found { + let error = format!("cannot import name '{}'", import_desc.name()); + log::warn!("{}", error); + continue; } + log::debug!( + "resolved import: {} -> {:?}", + import_desc.name(), + resolved.resolved_paths + ); + imports.insert(import_desc, resolved.clone()); } } - for resolved_import in resolved_imports { - let file = self.parse(&resolved_import); - let state = State::new(file); - resolved_paths.insert(state.file.module_name().clone(), state); - } - - resolved_paths + return imports.clone(); } } @@ -444,14 +481,8 @@ mod tests { let mut settings = insta::Settings::clone_current(); settings.set_snapshot_path("../testdata/output/"); settings.set_description(fs::read_to_string(path).unwrap()); - settings.add_filter( - r"/.*/typechecker", - "[TYPECHECKER]", - ); - settings.add_filter( - r"/.*/typeshed", - "[TYPESHED]", - ); + settings.add_filter(r"/.*/typechecker", "[TYPECHECKER]"); + settings.add_filter(r"/.*/typeshed", "[TYPESHED]"); settings.add_filter( r"module_name: .*.typechecker.test_data.inputs.symbol_table..*.py", "module_name: [REDACTED]", diff --git a/typechecker/src/build_source.rs b/typechecker/src/build_source.rs index 8239e32b..005f492b 100644 --- a/typechecker/src/build_source.rs +++ b/typechecker/src/build_source.rs @@ -28,6 +28,15 @@ impl BuildSource { } } -fn get_module_name(path: &Path) -> String { +pub fn get_module_name(path: &Path) -> String { path.to_str().unwrap_or_default().replace(['/', '\\'], ".") } + +// impl Into for BuildSource { +// fn into(self) -> EnderpyFile { +// let file_path = self.path.to_str().unwrap_or("could not get path"); +// let mut parser = Parser::new(self.source.clone(), file_path.into()); +// let tree = parser.parse(); +// EnderpyFile::from(tree, Box::new(self.clone()), parser.errors) +// } +// } diff --git a/typechecker/src/nodes.rs b/typechecker/src/nodes.rs index c0f70cf5..e45d4e9b 100755 --- a/typechecker/src/nodes.rs +++ b/typechecker/src/nodes.rs @@ -8,8 +8,9 @@ use std::path::PathBuf; use enderpy_python_parser as parser; -use enderpy_python_parser::ast::{Import, ImportFrom, Module, Statement}; +use enderpy_python_parser::ast::{Import, ImportFrom, Statement}; use parser::error::ParsingError; +use parser::Parser; use crate::{ast_visitor::TraversalVisitor, build_source::BuildSource, diagnostic::Position}; @@ -28,29 +29,12 @@ pub struct EnderpyFile { // All high level statements inside the file pub body: Vec, - pub build_source: Box, + pub build_source: BuildSource, // Parser Errors pub errors: Vec, } impl EnderpyFile { - pub fn from(ast: Module, build_source: Box, errors: Vec) -> Self { - let mut file = Self { - defs: vec![], - imports: vec![], - body: vec![], - build_source, - errors, - }; - - for stmt in &ast.body { - file.visit_stmt(stmt); - file.body.push(stmt.clone()); - } - - file - } - pub fn module_name(&self) -> String { self.build_source.module.clone() } @@ -82,6 +66,31 @@ impl EnderpyFile { } } +impl From for EnderpyFile { + fn from(build_source: BuildSource) -> Self { + let mut parser = Parser::new( + build_source.source.clone(), + build_source.path.as_path().to_str().unwrap().to_owned(), + ); + let tree = parser.parse(); + + let mut file = EnderpyFile { + defs: vec![], + imports: vec![], + body: vec![], + build_source, + errors: parser.errors, + }; + + for stmt in &tree.body { + file.visit_stmt(stmt); + file.body.push(stmt.clone()); + } + + file + } +} + impl TraversalVisitor for EnderpyFile { fn visit_stmt(&mut self, s: &Statement) { // map all statements and call visit diff --git a/typechecker/src/ruff_python_import_resolver/module_descriptor.rs b/typechecker/src/ruff_python_import_resolver/module_descriptor.rs index 4e315fc3..c397cafa 100644 --- a/typechecker/src/ruff_python_import_resolver/module_descriptor.rs +++ b/typechecker/src/ruff_python_import_resolver/module_descriptor.rs @@ -1,6 +1,6 @@ use enderpy_python_parser::ast::{Alias, ImportFrom}; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ImportModuleDescriptor { pub leading_dots: usize, pub name_parts: Vec, diff --git a/typechecker/src/semantic_analyzer.rs b/typechecker/src/semantic_analyzer.rs index 208721c4..3cdc2acb 100644 --- a/typechecker/src/semantic_analyzer.rs +++ b/typechecker/src/semantic_analyzer.rs @@ -11,7 +11,7 @@ use crate::{ import_result::ImportResult, module_descriptor::ImportModuleDescriptor, }, symbol_table::{ - Alias, Class, Declaration, DeclarationPath, Function, Paramter, SymbolScope, SymbolTable, + Alias, Class, Declaration, DeclarationPath, Function, Parameter, SymbolScope, SymbolTable, SymbolTableNode, SymbolTableScope, SymbolTableType, TypeAlias, Variable, }, }; @@ -27,7 +27,7 @@ pub struct SemanticAnalyzer { /// if we have a file with the following imports this is how we use the map /// import os -> imports.get("os") /// from os import path -> imports.get("os") - pub imports: HashMap, + pub imports: HashMap, // TODO: Replace errors with another type errors: Vec, @@ -36,9 +36,9 @@ pub struct SemanticAnalyzer { #[allow(unused)] impl SemanticAnalyzer { - pub fn new(file: EnderpyFile, imports: HashMap) -> Self { - let globals = SymbolTable::global(file.module_name()); + pub fn new(file: EnderpyFile, imports: HashMap) -> Self { log::debug!("Creating semantic analyzer for {}", file.module_name()); + let globals = SymbolTable::global(file.clone()); SemanticAnalyzer { globals, file, @@ -93,7 +93,7 @@ impl SemanticAnalyzer { } } Expression::Attribute(_) => {} - // TODO: Add oher expressions that can be assigned + // TODO: Add other expressions that can be assigned _ => {} } } @@ -112,7 +112,7 @@ impl SemanticAnalyzer { self.create_symbol( pos_only.arg.clone(), - Declaration::Parameter(Paramter { + Declaration::Parameter(Parameter { declaration_path, parameter_node: pos_only.clone(), type_annotation: pos_only.annotation.clone(), @@ -137,7 +137,7 @@ impl SemanticAnalyzer { self.create_symbol( arg.arg.clone(), - Declaration::Parameter(Paramter { + Declaration::Parameter(Parameter { declaration_path, parameter_node: arg.clone(), type_annotation: arg.annotation.clone(), @@ -153,7 +153,7 @@ impl SemanticAnalyzer { }; self.create_symbol( arg.arg.clone(), - Declaration::Parameter(Paramter { + Declaration::Parameter(Parameter { declaration_path, parameter_node: arg.clone(), type_annotation: arg.annotation.clone(), @@ -169,7 +169,7 @@ impl SemanticAnalyzer { }; self.create_symbol( arg.arg.clone(), - Declaration::Parameter(Paramter { + Declaration::Parameter(Parameter { declaration_path, parameter_node: arg.clone(), type_annotation: arg.annotation.clone(), @@ -185,7 +185,7 @@ impl SemanticAnalyzer { }; self.create_symbol( arg.arg.clone(), - Declaration::Parameter(Paramter { + Declaration::Parameter(Parameter { declaration_path, parameter_node: arg.clone(), type_annotation: arg.annotation.clone(), @@ -264,10 +264,7 @@ impl TraversalVisitor for SemanticAnalyzer { fn visit_import(&mut self, i: &parser::ast::Import) { for alias in &i.names { - let import_result = match self - .imports - .get(&ImportModuleDescriptor::from(alias).name()) - { + let import_result = match self.imports.get(&ImportModuleDescriptor::from(alias)) { Some(result) => result.clone(), None => ImportResult::not_found(), }; @@ -296,11 +293,10 @@ impl TraversalVisitor for SemanticAnalyzer { node: alias.node, }; // TODO: Report unresolved import if import_result is None - let module_import_result = - match self.imports.get(&ImportModuleDescriptor::from(_i).name()) { - Some(result) => result.clone(), - None => ImportResult::not_found(), - }; + let module_import_result = match self.imports.get(&ImportModuleDescriptor::from(_i)) { + Some(result) => result.clone(), + None => ImportResult::not_found(), + }; let declaration = Declaration::Alias(Alias { declaration_path, import_from_node: Some(_i.clone()), @@ -453,7 +449,7 @@ impl TraversalVisitor for SemanticAnalyzer { is_method: self.is_inside_class(), is_generator: !yeild_statements.is_empty(), return_statements, - yeild_statements, + yield_statements: yeild_statements, raise_statements, }); self.create_symbol(f.name.clone(), function_declaration); diff --git a/typechecker/src/state.rs b/typechecker/src/state.rs index 600f71c5..9c57e8aa 100644 --- a/typechecker/src/state.rs +++ b/typechecker/src/state.rs @@ -4,9 +4,8 @@ use crate::{ ast_visitor::TraversalVisitor, diagnostic::Diagnostic, nodes::EnderpyFile, - ruff_python_import_resolver as ruff_python_resolver, ruff_python_import_resolver::{ - import_result::ImportResult, module_descriptor::ImportModuleDescriptor, resolver, + import_result::ImportResult, module_descriptor::ImportModuleDescriptor, }, semantic_analyzer::SemanticAnalyzer, symbol_table::SymbolTable, @@ -17,23 +16,23 @@ pub struct State { pub file: EnderpyFile, symbol_table: SymbolTable, pub diagnostics: Vec, - // Map of import names to the result of the import - pub imports: HashMap, } impl State { pub fn new(file: EnderpyFile) -> Self { - let module_name = file.module_name(); + let symbol_table = SymbolTable::global(file.clone()); Self { file, - symbol_table: SymbolTable::global(module_name), + symbol_table, diagnostics: Vec::new(), - imports: HashMap::new(), } } /// entry point to fill up the symbol table from the global definitions - pub fn populate_symbol_table(&mut self) { - let mut sem_anal = SemanticAnalyzer::new(self.file.clone(), self.imports.clone()); + pub fn populate_symbol_table( + &mut self, + imports: &HashMap, + ) { + let mut sem_anal = SemanticAnalyzer::new(self.file.clone(), imports.clone()); for stmt in &self.file.body { sem_anal.visit_stmt(stmt) } @@ -45,49 +44,4 @@ impl State { pub fn get_symbol_table(&self) -> SymbolTable { self.symbol_table.clone() } - - pub fn resolve_file_imports( - &mut self, - execution_environment: &ruff_python_resolver::execution_environment::ExecutionEnvironment, - import_config: &ruff_python_resolver::config::Config, - host: &ruff_python_resolver::host::StaticHost, - ) { - log::debug!("resolving imports for file: {}", self.file.module_name()); - for import in self.file.imports.iter() { - let import_descriptions = match import { - crate::nodes::ImportKinds::Import(i) => i - .names - .iter() - .map(|x| { - ruff_python_resolver::module_descriptor::ImportModuleDescriptor::from(x) - }) - .collect::>(), - crate::nodes::ImportKinds::ImportFrom(i) => { - vec![ruff_python_resolver::module_descriptor::ImportModuleDescriptor::from(i)] - } - }; - log::debug!("import descriptions: {:?}", import_descriptions); - - for import_desc in import_descriptions { - let resolved = resolver::resolve_import( - self.file.path().as_path(), - execution_environment, - &import_desc, - import_config, - host, - ); - if !resolved.is_import_found { - let error = format!("cannot import name '{}'", import_desc.name()); - log::warn!("{}", error); - continue; - } - log::debug!( - "resolved import: {} -> {:?}", - import_desc.name(), - resolved.resolved_paths - ); - self.imports.insert(import_desc.name(), resolved); - } - } - } } diff --git a/typechecker/src/symbol_table.rs b/typechecker/src/symbol_table.rs index da28ea89..d422b490 100644 --- a/typechecker/src/symbol_table.rs +++ b/typechecker/src/symbol_table.rs @@ -1,8 +1,11 @@ -use std::{collections::HashMap, fmt::Display}; +use std::{collections::HashMap, fmt::Display, path::PathBuf}; use enderpy_python_parser::ast::{self, Node}; -use crate::{ruff_python_import_resolver::import_result::ImportResult, type_check::builtins}; +use crate::{ + nodes::EnderpyFile, ruff_python_import_resolver::import_result::ImportResult, + type_check::builtins, +}; #[derive(Debug, Clone)] pub struct SymbolTable { @@ -18,6 +21,7 @@ pub struct SymbolTable { /// Name of the module that this symbol table is for pub module_name: String, + pub file_path: PathBuf, } #[derive(Debug, Clone)] @@ -80,7 +84,7 @@ pub enum Declaration { // Alias is used for imports Alias(Alias), - Parameter(Paramter), + Parameter(Parameter), // TypeParameterDeclaration represents a type parameter in a generic class or function. // It models type parameters declared on classes and functions like T in List[T]. TypeParameter(TypeParameter), @@ -126,7 +130,7 @@ pub struct Function { /// return statements that are reachable in the top level function body pub return_statements: Vec, /// yield statements that are reachable in the top level function body - pub yeild_statements: Vec, + pub yield_statements: Vec, /// raise statements that are reachable in the top level function body pub raise_statements: Vec, } @@ -154,14 +158,14 @@ pub struct Class { // Method names, can be used to look up the function in the symbol table // of the class pub methods: Vec, - // instance attibutes that are defined in the __init__ method + // instance attributes that are defined in the __init__ method // if the attribute is referencing another symbol we need to look up that symbol in the // __init__ method pub attributes: HashMap, } #[derive(Debug, Clone)] -pub struct Paramter { +pub struct Parameter { pub declaration_path: DeclarationPath, pub parameter_node: ast::Arg, pub type_annotation: Option, @@ -208,7 +212,7 @@ pub enum SymbolScope { } impl SymbolTable { - pub fn global(module_name: String) -> Self { + pub fn global(enderpy_file: EnderpyFile) -> Self { let mut builtin_scope = SymbolTableScope { id: get_id(), symbol_table_type: SymbolTableType::BUILTIN, @@ -296,7 +300,8 @@ impl SymbolTable { scopes: vec![builtin_scope, global_scope], all_scopes: vec![], _locals: HashMap::new(), - module_name, + module_name: enderpy_file.module_name(), + file_path: enderpy_file.path(), } } @@ -313,13 +318,15 @@ impl SymbolTable { return &self.current_scope().symbol_table_type; } - /// Retuns scopes until the given position + /// Returns scopes until the given position /// the scopes are sorted by start position descending pub fn innermost_scope(&self, pos: usize) -> Option<&SymbolTableScope> { + log::debug!("looking for innermost scope for pos: {}", pos); + log::debug!("all scopes: {:#?}", self.all_scopes); return self .all_scopes .iter() - .filter(|scope| scope.start_pos < pos) + .filter(|scope| scope.start_pos <= pos) .last(); } @@ -327,6 +334,7 @@ impl SymbolTable { /// search for symbol in that scope /// if not found search in parent scope /// continue until found or no parent scope + /// TODO: This function does not work on the literal test pub fn lookup_in_scope(&self, lookup_request: LookupSymbolRequest) -> Option<&SymbolTableNode> { match lookup_request.position { Some(pos) => { diff --git a/typechecker/src/type_check/test_data/inputs/literal.py b/typechecker/src/type_check/test_data/inputs/literal.py index 268743a0..ffc28c17 100644 --- a/typechecker/src/type_check/test_data/inputs/literal.py +++ b/typechecker/src/type_check/test_data/inputs/literal.py @@ -2,3 +2,12 @@ a: Literal["foo"] = "foo" + + +class Foo: + def __init__(self, name: Literal["foo"]) -> None: + self.name = name + + +def func(literal_name: Literal["foo"]) -> None: + print(literal_name) diff --git a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap.new b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap.new new file mode 100644 index 00000000..f6809203 --- /dev/null +++ b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap.new @@ -0,0 +1,47 @@ +--- +source: typechecker/src/type_check/type_evaluator.rs +assertion_line: 1184 +description: "from typing import Literal\n\n\na: Literal[\"foo\"] = \"foo\"\n\n\nclass Foo:\n def __init__(self, name: Literal[\"foo\"]) -> None:\n self.name = name\n\n\ndef func(literal_name: Literal[\"foo\"]) -> None:\n print(literal_name)\n" +expression: result +input_file: typechecker/src/type_check/test_data/inputs/literal.py +--- +[ + ( + "(line: 12, character: 10):(line: 12, character: 22)", + Unknown, + ), + ( + "(line: 12, character: 4):(line: 12, character: 9)", + Unknown, + ), + ( + "(line: 3, character: 20):(line: 3, character: 25)", + Str, + ), + ( + "(line: 3, character: 3):(line: 3, character: 17)", + Class( + ClassType { + details: Class { + name: "Literal", + declaration_path: DeclarationPath { + module_name: ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi", + node: Node { + start: 5140, + end: 5161, + }, + }, + methods: [], + attributes: {}, + }, + type_parameters: [ + Unknown, + ], + }, + ), + ), + ( + "(line: 8, character: 20):(line: 8, character: 24)", + Unknown, + ), +] diff --git a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@union.py.snap b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@union.py.snap index bd40ae9c..a1cc4169 100644 --- a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@union.py.snap +++ b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@union.py.snap @@ -7,12 +7,24 @@ input_file: typechecker/src/type_check/test_data/inputs/union.py [ ( "(line: 10, character: 3):(line: 10, character: 24)", - MultiValue( - [ - Str, - Int, - None, - ], + Class( + ClassType { + details: Class { + name: "Union", + declaration_path: DeclarationPath { + module_name: ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi", + node: Node { + start: 4773, + end: 4792, + }, + }, + methods: [], + attributes: {}, + }, + type_parameters: [ + Unknown, + ], + }, ), ), ( @@ -42,11 +54,24 @@ input_file: typechecker/src/type_check/test_data/inputs/union.py ), ( "(line: 6, character: 3):(line: 6, character: 18)", - MultiValue( - [ - Str, - Int, - ], + Class( + ClassType { + details: Class { + name: "Union", + declaration_path: DeclarationPath { + module_name: ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi", + node: Node { + start: 4773, + end: 4792, + }, + }, + methods: [], + attributes: {}, + }, + type_parameters: [ + Unknown, + ], + }, ), ), ( diff --git a/typechecker/src/type_check/type_evaluator.rs b/typechecker/src/type_check/type_evaluator.rs index 2c5a872e..db9c4eb7 100755 --- a/typechecker/src/type_check/type_evaluator.rs +++ b/typechecker/src/type_check/type_evaluator.rs @@ -19,7 +19,10 @@ use crate::{ ast_visitor_generic::TraversalVisitorImmutGeneric, nodes::EnderpyFile, state::State, - symbol_table::{self, Declaration, LookupSymbolRequest, SymbolTable, SymbolTableNode}, + symbol_table::{ + self, Class, Declaration, DeclarationPath, LookupSymbolRequest, SymbolTable, + SymbolTableNode, + }, type_check::types::ClassType, }; @@ -197,7 +200,7 @@ impl TypeEvaluator { // ast::Expression::Name(n) => { // comp_targets.insert(n.id, self.get_type(&gens.iter)); // } - // _ => panic!("comperhension target must be a name, or does it?"), + // _ => panic!("comprehension target must be a name, or does it?"), // } // } @@ -223,7 +226,7 @@ impl TypeEvaluator { } // This function tries to find the python type from an annotation expression - // If the annotation is invalid it returns uknown type + // If the annotation is invalid it returns unknown type pub fn get_type_from_annotation(&self, type_annotation: &ast::Expression) -> PythonType { log::debug!("Getting type from annotation: {:?}", type_annotation); let expr_type = match type_annotation { @@ -258,8 +261,8 @@ impl TypeEvaluator { // flatten the bit or expression if the left and right are also bit or // TODO handle when left and right are also binary operator // Like a | b | c | d - let union_paramters = self.flatten_bit_or(b); - self.handle_union_type(union_paramters) + let union_parameters = self.flatten_bit_or(b); + self.handle_union_type(union_parameters) } _ => todo!(), } @@ -287,7 +290,7 @@ impl TypeEvaluator { self.get_type_from_annotation(&type_annotation) } else { let inferred_return_type = self.infer_function_return_type(f); - log::debug!("infered_return_type: {:?}", inferred_return_type); + log::debug!("inferred_return_type: {:?}", inferred_return_type); inferred_return_type }; @@ -351,9 +354,9 @@ impl TypeEvaluator { if !f.is_abstract() && !f.raise_statements.is_empty() { return PythonType::Never; } - if !f.yeild_statements.is_empty() { + if !f.yield_statements.is_empty() { let mut yield_types = vec![]; - for yield_statement in &f.yeild_statements { + for yield_statement in &f.yield_statements { if let Some(value) = &yield_statement.value { yield_types.push(self.get_type(value).unwrap_or(PythonType::Unknown)); } @@ -505,8 +508,8 @@ impl TypeEvaluator { ast::ConstantValue::Str(s) => LiteralValue::Str(s), ast::ConstantValue::Bytes(b) => LiteralValue::Bytes(b), ast::ConstantValue::None => LiteralValue::None, - // Tuple is illegal if it has parantheses, otherwise it's allowed and the output - // a multiValued type Currently even mypy does not supoort + // Tuple is illegal if it has parentheses, otherwise it's allowed and the output + // a multiValued type Currently even mypy does not support // this, who am I to do it? https://mypy-play.net/?mypy=latest&python=3.10&gist=0df0421d5c85f3b75f65a51cae8616ce ast::ConstantValue::Tuple(t) => { if t.len() == 1 { @@ -723,6 +726,7 @@ impl TypeEvaluator { builtin_type // if it's not a builtin we want to get the class declaration form symbol table // and find where this class is defined + // TODO it's good to have a function that finds the initial declaration of a symbol } else { let container_decl = match symbol_table.lookup_in_scope(LookupSymbolRequest { name: n.id.clone(), @@ -731,12 +735,40 @@ impl TypeEvaluator { Some(decl) => decl.last_declaration(), None => panic!("Type {} has no declaration", n.id), }; + log::debug!("container_decl: {:?}", container_decl); match container_decl { Some(Declaration::Class(c)) => c.clone(), Some(Declaration::Alias(a)) => { - if let Some(decl) = self.resolve_alias(a) { + if let Some((found_in_symbol_table, decl)) = self.resolve_alias(a) { match decl.last_declaration() { Some(Declaration::Class(c)) => c.clone(), + // The container type here can be a variable, e.g. in + // typing.pyi there is Literal: _SpecialForm + // If it's a variable then try to find the class node + // of that variable + // TODO: need to clean up here to check special classes and + // creata a class with their methods on it. For example Literal + // and Union + Some(Declaration::Variable(v)) => { + if let Some(annotation) = &v.type_annotation { + let pointing_class = self.get_class_declaration( + annotation.clone(), + Some(found_in_symbol_table.clone()), + ); + if pointing_class.name == "_SpecialForm" { + return Class { + name: n.id, + declaration_path: v.declaration_path.clone(), + methods: vec![], + attributes: HashMap::new(), + }; + } else { + pointing_class + } + } else { + panic!("not implemented") + } + } _ => panic!("Alias {:?} is not a class", a), } } else { @@ -774,20 +806,35 @@ impl TypeEvaluator { // Follows Alias declaration and resolves it to a class declaration // It searches through imported symbol tables for the module alias imports // and resolves the alias to the class declaration - // TODO: refactor to resolve all aliases and not only classes - fn resolve_alias(&self, a: &symbol_table::Alias) -> Option<&symbol_table::SymbolTableNode> { + // TODO: refactor all liases and not only classes + fn resolve_alias( + &self, + a: &symbol_table::Alias, + ) -> Option<(symbol_table::SymbolTable, symbol_table::SymbolTableNode)> { let class_name = match a.symbol_name { Some(ref name) => name.clone(), None => panic!("Alias {:?} has no symbol name", a.import_node), }; - let imported_symbol_tables = self.imported_symbol_tables.clone(); - log::debug!("Searching for alias {}", class_name,); - for symbol_table in imported_symbol_tables { + // log::debug!("symbol tables: {:#?}", self.imported_symbol_tables); + for symbol_table in self.imported_symbol_tables.iter() { + if !a + .import_result + .resolved_paths + .contains(&symbol_table.file_path) + { + continue; + } + log::debug!( + "Searching for alias {}, in symbol table for file {}", + class_name, + symbol_table.module_name + ); if let Some(decl) = symbol_table.lookup_in_scope(LookupSymbolRequest { name: class_name.clone(), position: None, }) { - return Some(decl); + log::debug!("Found alias {:#?} in symbol table", decl); + return Some((symbol_table.clone(), decl.clone())); } } panic!("Alias {} not found", class_name); @@ -1090,7 +1137,7 @@ mod tests { followed: false, }; - // we use the manager to also import the python typeshed into moduels + // we use the manager to also import the python typeshed into modules // This can be refactored but for now it's fine let mut manager = BuildManager::new(vec![build_source], Settings::test_settings()); manager.build(); @@ -1153,7 +1200,7 @@ impl DumpTypes { let mut state = State::new(enderpy_file); // TODO: this line runs on every test and it needs to add all of stdlib to symbol table. // This is not efficient and needs to be refactored - state.populate_symbol_table(); + state.populate_symbol_table(&HashMap::new()); let symbol_table = state.get_symbol_table(); Self { types: HashMap::new(), @@ -1277,7 +1324,12 @@ impl TraversalVisitor for DumpTypes { ast::Expression::Attribute(a) => self.visit_attribute(a), ast::Expression::Subscript(s) => self.visit_subscript(s), ast::Expression::Slice(s) => self.visit_slice(s), - ast::Expression::Call(c) => self.visit_call(c), + ast::Expression::Call(c) => { + self.save_type(&c.func); + for arg in &c.args { + self.save_type(arg); + } + } ast::Expression::Await(a) => self.visit_await(a), ast::Expression::Compare(c) => self.visit_compare(c), ast::Expression::Lambda(l) => self.visit_lambda(l), From edf48b8e20d3c17ae4571a77ebd7369e0cb6c05a Mon Sep 17 00:00:00 2001 From: Glyphack Date: Thu, 4 Jan 2024 23:57:31 +0100 Subject: [PATCH 07/13] update snaps --- ...n_parser__lexer__tests__indentation-4.snap | 16 +++---- ...tor__tests__type_evaluator@literal.py.snap | 36 +++++++++++--- ..._tests__type_evaluator@literal.py.snap.new | 47 ------------------- ...sts__symbol_table@class_definition.py.snap | 4 +- ...__symbol_table@function_definition.py.snap | 10 ++-- 5 files changed, 45 insertions(+), 68 deletions(-) delete mode 100644 typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap.new diff --git a/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__indentation-4.snap b/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__indentation-4.snap index ede2a236..5ac25b8f 100644 --- a/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__indentation-4.snap +++ b/parser/src/lexer/snapshots/enderpy_python_parser__lexer__tests__indentation-4.snap @@ -1,6 +1,6 @@ --- -source: parser/src/lexer/lexer.rs -description: "if a:\n\n f = c\n\n # Path: test_local.py\t\n" +source: parser/src/lexer/mod.rs +description: "if a:\n\n f = c\n\n # Path: test_local.py\n" --- [ Token { @@ -80,23 +80,23 @@ description: "if a:\n\n f = c\n\n # Path: test_local.py\t\n" Token { kind: Comment, value: Str( - "# Path: test_local.py\t", + "# Path: test_local.py", ), start: 22, - end: 44, + end: 43, }, Token { kind: NewLine, value: None, - start: 44, - end: 45, + start: 43, + end: 44, }, Token { kind: Dedent, value: Indent( 1, ), - start: 45, - end: 45, + start: 44, + end: 44, }, ] diff --git a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap index dba9e98b..14c0c451 100644 --- a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap +++ b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap @@ -1,22 +1,46 @@ --- source: typechecker/src/type_check/type_evaluator.rs -description: "from typing import Literal\n\n\na: Literal[\"foo\"] = \"foo\"\n" +description: "from typing import Literal\n\n\na: Literal[\"foo\"] = \"foo\"\n\n\nclass Foo:\n def __init__(self, name: Literal[\"foo\"]) -> None:\n self.name = name\n\n\ndef func(literal_name: Literal[\"foo\"]) -> None:\n print(literal_name)\n" expression: result input_file: typechecker/src/type_check/test_data/inputs/literal.py --- [ + ( + "(line: 12, character: 10):(line: 12, character: 22)", + Unknown, + ), + ( + "(line: 12, character: 4):(line: 12, character: 9)", + Unknown, + ), ( "(line: 3, character: 20):(line: 3, character: 25)", Str, ), ( "(line: 3, character: 3):(line: 3, character: 17)", - KnownValue( - KnownValue { - literal_value: Str( - "foo", - ), + Class( + ClassType { + details: Class { + name: "Literal", + declaration_path: DeclarationPath { + module_name: ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi", + node: Node { + start: 5140, + end: 5161, + }, + }, + methods: [], + attributes: {}, + }, + type_parameters: [ + Unknown, + ], }, ), ), + ( + "(line: 8, character: 20):(line: 8, character: 24)", + Unknown, + ), ] diff --git a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap.new b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap.new deleted file mode 100644 index f6809203..00000000 --- a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap.new +++ /dev/null @@ -1,47 +0,0 @@ ---- -source: typechecker/src/type_check/type_evaluator.rs -assertion_line: 1184 -description: "from typing import Literal\n\n\na: Literal[\"foo\"] = \"foo\"\n\n\nclass Foo:\n def __init__(self, name: Literal[\"foo\"]) -> None:\n self.name = name\n\n\ndef func(literal_name: Literal[\"foo\"]) -> None:\n print(literal_name)\n" -expression: result -input_file: typechecker/src/type_check/test_data/inputs/literal.py ---- -[ - ( - "(line: 12, character: 10):(line: 12, character: 22)", - Unknown, - ), - ( - "(line: 12, character: 4):(line: 12, character: 9)", - Unknown, - ), - ( - "(line: 3, character: 20):(line: 3, character: 25)", - Str, - ), - ( - "(line: 3, character: 3):(line: 3, character: 17)", - Class( - ClassType { - details: Class { - name: "Literal", - declaration_path: DeclarationPath { - module_name: ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi", - node: Node { - start: 5140, - end: 5161, - }, - }, - methods: [], - attributes: {}, - }, - type_parameters: [ - Unknown, - ], - }, - ), - ), - ( - "(line: 8, character: 20):(line: 8, character: 24)", - Unknown, - ), -] diff --git a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@class_definition.py.snap b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@class_definition.py.snap index 9df594d9..58feefe4 100644 --- a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@class_definition.py.snap +++ b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@class_definition.py.snap @@ -60,7 +60,7 @@ b } self - Declarations: ---: Paramter { +--: Parameter { declaration_path: DeclarationPath { module_name: [REDACTED]", node: Node { @@ -222,7 +222,7 @@ __init__ is_method: true, is_generator: false, return_statements: [], - yeild_statements: [], + yield_statements: [], raise_statements: [], } diff --git a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@function_definition.py.snap b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@function_definition.py.snap index b53298ce..f6d4fa53 100644 --- a/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@function_definition.py.snap +++ b/typechecker/testdata/output/enderpy_python_type_checker__build__tests__symbol_table@function_definition.py.snap @@ -10,7 +10,7 @@ all scopes: Symbols: in func (id: [REDACTED]) a - Declarations: ---: Paramter { +--: Parameter { declaration_path: DeclarationPath { module_name: [REDACTED]", node: Node { @@ -31,7 +31,7 @@ a } b - Declarations: ---: Paramter { +--: Parameter { declaration_path: DeclarationPath { module_name: [REDACTED]", node: Node { @@ -52,7 +52,7 @@ b } c - Declarations: ---: Paramter { +--: Parameter { declaration_path: DeclarationPath { module_name: [REDACTED]", node: Node { @@ -83,7 +83,7 @@ c } e - Declarations: ---: Paramter { +--: Parameter { declaration_path: DeclarationPath { module_name: [REDACTED]", node: Node { @@ -196,7 +196,7 @@ func is_method: false, is_generator: false, return_statements: [], - yeild_statements: [], + yield_statements: [], raise_statements: [], } From a4a5aa9cbceda2e98585708113f7c904c799b4d8 Mon Sep 17 00:00:00 2001 From: Glyphack Date: Fri, 5 Jan 2024 00:03:22 +0100 Subject: [PATCH 08/13] Get submodules --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f2b57a0..df87570f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,9 @@ jobs: steps: - uses: Swatinem/rust-cache@v2 - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + submodules: true - name: setup toolchain uses: hecrj/setup-rust-action@v1 with: @@ -33,4 +35,4 @@ jobs: - name: rustfmt run: cargo +nightly fmt --all -- --check - name: clippy - run: cargo +nightly clippy --all --all-features --tests -- -D warnings \ No newline at end of file + run: cargo +nightly clippy --all --all-features --tests -- -D warnings From c94845aab7608b7545bf011e2a069f31f57ee62a Mon Sep 17 00:00:00 2001 From: Glyphack Date: Fri, 5 Jan 2024 00:08:55 +0100 Subject: [PATCH 09/13] redact local path --- ...eck__type_evaluator__tests__type_evaluator@literal.py.snap | 2 +- ...check__type_evaluator__tests__type_evaluator@union.py.snap | 4 ++-- typechecker/src/type_check/type_evaluator.rs | 2 ++ 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap index 14c0c451..6977b6b7 100644 --- a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap +++ b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@literal.py.snap @@ -24,7 +24,7 @@ input_file: typechecker/src/type_check/test_data/inputs/literal.py details: Class { name: "Literal", declaration_path: DeclarationPath { - module_name: ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi", + module_name: [TYPESHED].stdlib.typing.pyi", node: Node { start: 5140, end: 5161, diff --git a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@union.py.snap b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@union.py.snap index a1cc4169..c5c99159 100644 --- a/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@union.py.snap +++ b/typechecker/src/type_check/test_data/output/enderpy_python_type_checker__type_check__type_evaluator__tests__type_evaluator@union.py.snap @@ -12,7 +12,7 @@ input_file: typechecker/src/type_check/test_data/inputs/union.py details: Class { name: "Union", declaration_path: DeclarationPath { - module_name: ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi", + module_name: [TYPESHED].stdlib.typing.pyi", node: Node { start: 4773, end: 4792, @@ -59,7 +59,7 @@ input_file: typechecker/src/type_check/test_data/inputs/union.py details: Class { name: "Union", declaration_path: DeclarationPath { - module_name: ".Users.glyphack.Programming.enderpy.typeshed.stdlib.typing.pyi", + module_name: [TYPESHED].stdlib.typing.pyi", node: Node { start: 4773, end: 4792, diff --git a/typechecker/src/type_check/type_evaluator.rs b/typechecker/src/type_check/type_evaluator.rs index db9c4eb7..3d2e8240 100755 --- a/typechecker/src/type_check/type_evaluator.rs +++ b/typechecker/src/type_check/type_evaluator.rs @@ -1177,7 +1177,9 @@ mod tests { .is_test(true) .try_init(); + // TODO move this redaction setting to a central place let mut settings = insta::Settings::clone_current(); + settings.add_filter(r"module_name: .*.typeshed.", "module_name: [TYPESHED]."); settings.set_snapshot_path("./test_data/output/"); settings.set_description(fs::read_to_string(path).unwrap()); settings.bind(|| { From 184c3f59a180c5051f6bea166cb5d72460c65d4d Mon Sep 17 00:00:00 2001 From: Glyphack Date: Fri, 5 Jan 2024 00:14:23 +0100 Subject: [PATCH 10/13] cargo fmt --- parser/src/parser/parser.rs | 6 +-- typechecker/src/build.rs | 50 +++++++++++--------- typechecker/src/nodes.rs | 3 +- typechecker/src/state.rs | 3 +- typechecker/src/type_check/type_evaluator.rs | 25 +++++----- 5 files changed, 46 insertions(+), 41 deletions(-) diff --git a/parser/src/parser/parser.rs b/parser/src/parser/parser.rs index 3c15f7a7..c0b4a46b 100644 --- a/parser/src/parser/parser.rs +++ b/parser/src/parser/parser.rs @@ -2113,7 +2113,7 @@ impl Parser { return Ok(Expression::Tuple(Box::new(Tuple { node: self.finish_node(node), elements: vec![], - }))) + }))); } if self.eat(Kind::Lambda) { let params_list = self.parse_parameters(true).expect("lambda params"); @@ -2962,8 +2962,8 @@ impl Parser { // after seeing vararg the must_have_default is reset // until we see a default value again must_have_default = false; - // In this case this is not a vararg but only marks the end of positional arguments - // e.g. def foo(a, b, *, c, d) + // In this case this is not a vararg but only marks the end of positional + // arguments e.g. def foo(a, b, *, c, d) if self.eat(Kind::Comma) { continue; } diff --git a/typechecker/src/build.rs b/typechecker/src/build.rs index 17b0807b..a0457122 100755 --- a/typechecker/src/build.rs +++ b/typechecker/src/build.rs @@ -232,36 +232,41 @@ impl BuildManager { }; } - // TODO: not sure if this part is needed. Need to check when we have more tests - // on builder. This is supposed to add implicit imports to the build sources + // TODO: not sure if this part is needed. Need to check when + // we have more tests on builder. This + // is supposed to add implicit imports to the build sources // implicit import example: import foo.bar // foo/bar/__init__.py - // In this case, we need to add foo/bar/__init__.py to the build sources - // for (name, implicit_import) in resolved.implicit_imports.iter() { + // In this case, we need to add foo/bar/__init__.py to the + // build sources for (name, + // implicit_import) in resolved.implicit_imports.iter() { // if self // .modules - // .contains_key(&get_module_name(&implicit_import.path)) - // { + // .contains_key(&get_module_name(&implicit_import. + // path)) { // log::debug!( // "implicit import already exists: {}", // get_module_name(&implicit_import.path) // ); // continue; // } - // let source = std::fs::read_to_string(implicit_import.path.clone()).unwrap(); - // let build_source = - // BuildSource::from_path(implicit_import.path.clone(), true).unwrap(); + // let source = + // std::fs::read_to_string(implicit_import.path.clone()). + // unwrap(); let build_source = + // BuildSource::from_path(implicit_import.path. + // clone(), true).unwrap(); // self.build_sources.push(build_source); // self.add_to_modules(&build_source); // match self.modules.get(&build_source.module) { // Some(discovered_module) => { - // if !self.modules.contains_key(&build_source.module) { + // if + // !self.modules.contains_key(&build_source.module) { // new_imports.push(discovered_module); // } // } // None => { - // panic!("cannot find module: {}", build_source.module); - // } + // panic!("cannot find module: {}", + // build_source.module); } // } // } } @@ -282,20 +287,21 @@ impl BuildManager { // host, // &cached_imports, // ); - // // check if the resolved_imports are not in the current files and add them to - // // the new imports + // // check if the resolved_imports are not in the current files and add + // them to // the new imports // for (_, resolved_import) in resolved_imports { // if !resolved_import.is_import_found { // continue; // } // for resolved_path in resolved_import.resolved_paths { - // if self.modules.contains_key(&get_module_name(&resolved_path)) { - // log::debug!("imported file already in modules: {:?}", resolved_path); - // continue; + // if + // self.modules.contains_key(&get_module_name(&resolved_path)) { + // log::debug!("imported file already in modules: {:?}", + // resolved_path); continue; // } - // let build_source = match BuildSource::from_path(resolved_path, true) { - // Ok(build_source) => build_source, - // Err(e) => { + // let build_source = match + // BuildSource::from_path(resolved_path, true) { + // Ok(build_source) => build_source, Err(e) => { // log::warn!("cannot read file: {}", e); // continue; // } @@ -307,8 +313,8 @@ impl BuildManager { // } // } // None => { - // panic!("cannot find module: {}", build_source.module); - // } + // panic!("cannot find module: {}", + // build_source.module); } // } // } // } diff --git a/typechecker/src/nodes.rs b/typechecker/src/nodes.rs index e45d4e9b..278f40d5 100755 --- a/typechecker/src/nodes.rs +++ b/typechecker/src/nodes.rs @@ -9,8 +9,7 @@ use std::path::PathBuf; use enderpy_python_parser as parser; use enderpy_python_parser::ast::{Import, ImportFrom, Statement}; -use parser::error::ParsingError; -use parser::Parser; +use parser::{error::ParsingError, Parser}; use crate::{ast_visitor::TraversalVisitor, build_source::BuildSource, diagnostic::Position}; diff --git a/typechecker/src/state.rs b/typechecker/src/state.rs index 9c57e8aa..833d10d7 100644 --- a/typechecker/src/state.rs +++ b/typechecker/src/state.rs @@ -36,7 +36,8 @@ impl State { for stmt in &self.file.body { sem_anal.visit_stmt(stmt) } - // TODO: Hacky way to add the global scope to all scopes in symbol table after finishing + // TODO: Hacky way to add the global scope to all scopes in symbol table after + // finishing sem_anal.globals.exit_scope(); self.symbol_table = sem_anal.globals; } diff --git a/typechecker/src/type_check/type_evaluator.rs b/typechecker/src/type_check/type_evaluator.rs index 3d2e8240..3159cf59 100755 --- a/typechecker/src/type_check/type_evaluator.rs +++ b/typechecker/src/type_check/type_evaluator.rs @@ -724,9 +724,10 @@ impl TypeEvaluator { Expression::Name(n) => { if let Some(builtin_type) = self.get_builtin_type(n.id.as_str()) { builtin_type - // if it's not a builtin we want to get the class declaration form symbol table - // and find where this class is defined - // TODO it's good to have a function that finds the initial declaration of a symbol + // if it's not a builtin we want to get the class declaration + // form symbol table and find where this class + // is defined TODO it's good to have a function + // that finds the initial declaration of a symbol } else { let container_decl = match symbol_table.lookup_in_scope(LookupSymbolRequest { name: n.id.clone(), @@ -1115,15 +1116,12 @@ impl TraversalVisitorImmutGeneric for TypeEvaluator { #[cfg(test)] mod tests { - use insta::glob; + use std::{fs, path::PathBuf}; - use crate::build::BuildManager; - use crate::build_source::BuildSource; - use crate::settings::Settings; + use insta::glob; use super::*; - use std::fs; - use std::path::PathBuf; + use crate::{build::BuildManager, build_source::BuildSource, settings::Settings}; fn snapshot_type_eval(source: &str) -> String { use enderpy_python_parser::Parser; @@ -1189,8 +1187,9 @@ mod tests { } } -/// visits the ast and calls get_type on each expression and saves that type in the types hashmap -/// the key is the position of the expression in the source: (line, start, end) +/// visits the ast and calls get_type on each expression and saves that type in +/// the types hashmap the key is the position of the expression in the source: +/// (line, start, end) struct DumpTypes { pub type_eval: TypeEvaluator, pub types: HashMap, @@ -1200,8 +1199,8 @@ struct DumpTypes { impl DumpTypes { pub fn new(enderpy_file: EnderpyFile, type_eval: TypeEvaluator) -> Self { let mut state = State::new(enderpy_file); - // TODO: this line runs on every test and it needs to add all of stdlib to symbol table. - // This is not efficient and needs to be refactored + // TODO: this line runs on every test and it needs to add all of stdlib to + // symbol table. This is not efficient and needs to be refactored state.populate_symbol_table(&HashMap::new()); let symbol_table = state.get_symbol_table(); Self { From 1a898ee5d4fef9a6bdf1bd7878f3f96f7fa0a79a Mon Sep 17 00:00:00 2001 From: Glyphack Date: Fri, 5 Jan 2024 00:15:34 +0100 Subject: [PATCH 11/13] cargo clippy --- typechecker/src/build.rs | 8 +- typechecker/src/settings.rs | 2 +- typechecker/src/type_check/type_evaluator.rs | 152 +++++++++---------- 3 files changed, 81 insertions(+), 81 deletions(-) diff --git a/typechecker/src/build.rs b/typechecker/src/build.rs index a0457122..9fd8a06a 100755 --- a/typechecker/src/build.rs +++ b/typechecker/src/build.rs @@ -219,7 +219,7 @@ impl BuildManager { continue; } }; - let build_source = match BuildSource::from_path(resolved_path.clone(), true) + match BuildSource::from_path(resolved_path.clone(), true) { Ok(build_source) => { imported_sources.push(build_source.clone()); @@ -326,8 +326,8 @@ impl BuildManager { // TODO: refactor to implement From/to trait fn create_module(&self, build_source: BuildSource) -> State { - let state = State::new(EnderpyFile::from(build_source)); - state + + State::new(EnderpyFile::from(build_source)) } fn resolve_file_imports( @@ -381,7 +381,7 @@ impl BuildManager { } } - return imports.clone(); + imports.clone() } } diff --git a/typechecker/src/settings.rs b/typechecker/src/settings.rs index 1b32c4f5..7133959d 100644 --- a/typechecker/src/settings.rs +++ b/typechecker/src/settings.rs @@ -48,7 +48,7 @@ impl Settings { follow_imports: FollowImports::All, import_discovery: ImportDiscovery { python_executable: None, - typeshed_path: Some(PathBuf::from(file_dir.parent().unwrap().join("typeshed"))), + typeshed_path: Some(file_dir.parent().unwrap().join("typeshed")), }, } } diff --git a/typechecker/src/type_check/type_evaluator.rs b/typechecker/src/type_check/type_evaluator.rs index 3159cf59..a2f85155 100755 --- a/typechecker/src/type_check/type_evaluator.rs +++ b/typechecker/src/type_check/type_evaluator.rs @@ -20,7 +20,7 @@ use crate::{ nodes::EnderpyFile, state::State, symbol_table::{ - self, Class, Declaration, DeclarationPath, LookupSymbolRequest, SymbolTable, + self, Class, Declaration, LookupSymbolRequest, SymbolTable, SymbolTableNode, }, type_check::types::ClassType, @@ -757,12 +757,12 @@ impl TypeEvaluator { Some(found_in_symbol_table.clone()), ); if pointing_class.name == "_SpecialForm" { - return Class { + Class { name: n.id, declaration_path: v.declaration_path.clone(), methods: vec![], attributes: HashMap::new(), - }; + } } else { pointing_class } @@ -1114,79 +1114,6 @@ impl TraversalVisitorImmutGeneric for TypeEvaluator { } } -#[cfg(test)] -mod tests { - use std::{fs, path::PathBuf}; - - use insta::glob; - - use super::*; - use crate::{build::BuildManager, build_source::BuildSource, settings::Settings}; - - fn snapshot_type_eval(source: &str) -> String { - use enderpy_python_parser::Parser; - - let mut parser = Parser::new(source.to_string(), "".into()); - let ast_module = parser.parse(); - let build_source = BuildSource { - path: PathBuf::from("test-file"), - source: source.to_string(), - module: "test".to_string(), - followed: false, - }; - - // we use the manager to also import the python typeshed into modules - // This can be refactored but for now it's fine - let mut manager = BuildManager::new(vec![build_source], Settings::test_settings()); - manager.build(); - - let mut all_symbol_tables = Vec::new(); - for module in manager.modules.values() { - all_symbol_tables.push(module.get_symbol_table()); - } - - let module = manager.get_state("test-file".into()).unwrap(); - let symbol_table = module.get_symbol_table(); - - let type_eval = TypeEvaluator { - symbol_table, - imported_symbol_tables: all_symbol_tables, - }; - - let mut type_eval_visitor = DumpTypes::new(module.file.clone(), type_eval); - type_eval_visitor.visit_module(); - - let result = type_eval_visitor.types; - - // sort result by key - let mut result_sorted = result.clone().into_iter().collect::>(); - result_sorted.sort_by(|a, b| a.0.cmp(&b.0)); - - format!("{:#?}", result_sorted) - } - - #[test] - fn test_type_evaluator() { - glob!("test_data/inputs/", "*.py", |path| { - let contents = fs::read_to_string(path).unwrap(); - let result = snapshot_type_eval(&contents); - let _ = env_logger::builder() - .filter_level(log::LevelFilter::Debug) - .is_test(true) - .try_init(); - - // TODO move this redaction setting to a central place - let mut settings = insta::Settings::clone_current(); - settings.add_filter(r"module_name: .*.typeshed.", "module_name: [TYPESHED]."); - settings.set_snapshot_path("./test_data/output/"); - settings.set_description(fs::read_to_string(path).unwrap()); - settings.bind(|| { - insta::assert_snapshot!(result); - }); - }) - } -} - /// visits the ast and calls get_type on each expression and saves that type in /// the types hashmap the key is the position of the expression in the source: /// (line, start, end) @@ -1340,3 +1267,76 @@ impl TraversalVisitor for DumpTypes { } } } + +#[cfg(test)] +mod tests { + use std::{fs, path::PathBuf}; + + use insta::glob; + + use super::*; + use crate::{build::BuildManager, build_source::BuildSource, settings::Settings}; + + fn snapshot_type_eval(source: &str) -> String { + use enderpy_python_parser::Parser; + + let mut parser = Parser::new(source.to_string(), "".into()); + let ast_module = parser.parse(); + let build_source = BuildSource { + path: PathBuf::from("test-file"), + source: source.to_string(), + module: "test".to_string(), + followed: false, + }; + + // we use the manager to also import the python typeshed into modules + // This can be refactored but for now it's fine + let mut manager = BuildManager::new(vec![build_source], Settings::test_settings()); + manager.build(); + + let mut all_symbol_tables = Vec::new(); + for module in manager.modules.values() { + all_symbol_tables.push(module.get_symbol_table()); + } + + let module = manager.get_state("test-file".into()).unwrap(); + let symbol_table = module.get_symbol_table(); + + let type_eval = TypeEvaluator { + symbol_table, + imported_symbol_tables: all_symbol_tables, + }; + + let mut type_eval_visitor = DumpTypes::new(module.file.clone(), type_eval); + type_eval_visitor.visit_module(); + + let result = type_eval_visitor.types; + + // sort result by key + let mut result_sorted = result.clone().into_iter().collect::>(); + result_sorted.sort_by(|a, b| a.0.cmp(&b.0)); + + format!("{:#?}", result_sorted) + } + + #[test] + fn test_type_evaluator() { + glob!("test_data/inputs/", "*.py", |path| { + let contents = fs::read_to_string(path).unwrap(); + let result = snapshot_type_eval(&contents); + let _ = env_logger::builder() + .filter_level(log::LevelFilter::Debug) + .is_test(true) + .try_init(); + + // TODO move this redaction setting to a central place + let mut settings = insta::Settings::clone_current(); + settings.add_filter(r"module_name: .*.typeshed.", "module_name: [TYPESHED]."); + settings.set_snapshot_path("./test_data/output/"); + settings.set_description(fs::read_to_string(path).unwrap()); + settings.bind(|| { + insta::assert_snapshot!(result); + }); + }) + } +} From 136f7dc68a2d1408c99266ecf0cd1e1b6cb13a9e Mon Sep 17 00:00:00 2001 From: Glyphack Date: Fri, 5 Jan 2024 00:19:18 +0100 Subject: [PATCH 12/13] cargo fmt --- typechecker/src/build.rs | 6 ++---- typechecker/src/type_check/type_evaluator.rs | 5 +---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/typechecker/src/build.rs b/typechecker/src/build.rs index 9fd8a06a..a6ead952 100755 --- a/typechecker/src/build.rs +++ b/typechecker/src/build.rs @@ -219,8 +219,7 @@ impl BuildManager { continue; } }; - match BuildSource::from_path(resolved_path.clone(), true) - { + match BuildSource::from_path(resolved_path.clone(), true) { Ok(build_source) => { imported_sources.push(build_source.clone()); files_to_resolve.push(build_source); @@ -300,7 +299,7 @@ impl BuildManager { // resolved_path); continue; // } // let build_source = match - // BuildSource::from_path(resolved_path, true) { + // BuildSource::from_path(resolved_path, true) { // Ok(build_source) => build_source, Err(e) => { // log::warn!("cannot read file: {}", e); // continue; @@ -326,7 +325,6 @@ impl BuildManager { // TODO: refactor to implement From/to trait fn create_module(&self, build_source: BuildSource) -> State { - State::new(EnderpyFile::from(build_source)) } diff --git a/typechecker/src/type_check/type_evaluator.rs b/typechecker/src/type_check/type_evaluator.rs index a2f85155..0d64cb35 100755 --- a/typechecker/src/type_check/type_evaluator.rs +++ b/typechecker/src/type_check/type_evaluator.rs @@ -19,10 +19,7 @@ use crate::{ ast_visitor_generic::TraversalVisitorImmutGeneric, nodes::EnderpyFile, state::State, - symbol_table::{ - self, Class, Declaration, LookupSymbolRequest, SymbolTable, - SymbolTableNode, - }, + symbol_table::{self, Class, Declaration, LookupSymbolRequest, SymbolTable, SymbolTableNode}, type_check::types::ClassType, }; From b7494b43b0105d4f7262e16a391d387603801ead Mon Sep 17 00:00:00 2001 From: Glyphack Date: Fri, 5 Jan 2024 00:23:29 +0100 Subject: [PATCH 13/13] cargo fmt --- typechecker/src/build.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/typechecker/src/build.rs b/typechecker/src/build.rs index a6ead952..eb39b1eb 100755 --- a/typechecker/src/build.rs +++ b/typechecker/src/build.rs @@ -194,11 +194,8 @@ impl BuildManager { log::debug!("import options: {:?}", execution_environment); let mut new_imports: Vec = vec![]; let host = &ruff_python_resolver::host::StaticHost::new(vec![]); - loop { - let enderpy_file = match files_to_resolve.pop() { - Some(source) => EnderpyFile::from(source), - None => break, - }; + while let Some(source) = files_to_resolve.pop() { + let enderpy_file = EnderpyFile::from(source); let resolved_imports = self.resolve_file_imports( enderpy_file, execution_environment,