From 2ea5ad3c642260c036841ff0a7513d4bf1f960ed Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Tue, 1 Oct 2024 19:04:04 +0200 Subject: [PATCH 1/4] Include string with "syntax not supported" error --- Sources/Environment.swift | 2 +- Sources/Error.swift | 4 ++-- Sources/Runtime.swift | 20 ++++++++++---------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Sources/Environment.swift b/Sources/Environment.swift index 503ad68..505c3ee 100644 --- a/Sources/Environment.swift +++ b/Sources/Environment.swift @@ -100,7 +100,7 @@ class Environment { args[0] is UndefinedValue }, "equalto": { _ in - throw JinjaError.syntaxNotSupported + throw JinjaError.syntaxNotSupported("equalto") }, ] diff --git a/Sources/Error.swift b/Sources/Error.swift index 4b820f3..cd03914 100644 --- a/Sources/Error.swift +++ b/Sources/Error.swift @@ -12,7 +12,7 @@ enum JinjaError: Error, LocalizedError { case parser(String) case runtime(String) case todo(String) - case syntaxNotSupported + case syntaxNotSupported(String) var errorDescription: String? { switch self { @@ -20,7 +20,7 @@ enum JinjaError: Error, LocalizedError { case .parser(let message): return "Parser error: \(message)" case .runtime(let message): return "Runtime error: \(message)" case .todo(let message): return "Todo error: \(message)" - case .syntaxNotSupported: return "Syntax not supported" + case .syntaxNotSupported(let string): return "Syntax not supported: \(string)" } } } diff --git a/Sources/Runtime.swift b/Sources/Runtime.swift index 66b17e8..bc01043 100644 --- a/Sources/Runtime.swift +++ b/Sources/Runtime.swift @@ -296,7 +296,7 @@ struct Interpreter { throw JinjaError.runtime("Cannot unpack non-iterable type: \(type(of:current))") } default: - throw JinjaError.syntaxNotSupported + throw JinjaError.syntaxNotSupported(String(describing: node.loopvar)) } let evaluated = try self.evaluateBlock(statements: node.body, environment: scope) @@ -353,10 +353,10 @@ struct Interpreter { } else if let left = left as? NumericValue, let right = right as? NumericValue { switch node.operation.value { - case "+": throw JinjaError.syntaxNotSupported - case "-": throw JinjaError.syntaxNotSupported - case "*": throw JinjaError.syntaxNotSupported - case "/": throw JinjaError.syntaxNotSupported + case "+": throw JinjaError.syntaxNotSupported("+") + case "-": throw JinjaError.syntaxNotSupported("-") + case "*": throw JinjaError.syntaxNotSupported("*") + case "/": throw JinjaError.syntaxNotSupported("/") case "%": switch left.value { case is Int: @@ -364,10 +364,10 @@ struct Interpreter { default: throw JinjaError.runtime("Unknown value type:\(type(of: left.value))") } - case "<": throw JinjaError.syntaxNotSupported - case ">": throw JinjaError.syntaxNotSupported - case ">=": throw JinjaError.syntaxNotSupported - case "<=": throw JinjaError.syntaxNotSupported + case "<": throw JinjaError.syntaxNotSupported("<") + case ">": throw JinjaError.syntaxNotSupported(">") + case ">=": throw JinjaError.syntaxNotSupported(">=") + case "<=": throw JinjaError.syntaxNotSupported("<=") default: throw JinjaError.runtime("Unknown operation type:\(node.operation.value)") } @@ -380,7 +380,7 @@ struct Interpreter { } } else if right is ArrayValue { - throw JinjaError.syntaxNotSupported + throw JinjaError.syntaxNotSupported("right is ArrayValue") } if left is StringValue || right is StringValue { From 45c7ed8bc40fe6a1b181716e8b8e12b26bd2e1db Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Tue, 1 Oct 2024 19:06:55 +0200 Subject: [PATCH 2/4] Handle `none` --- Sources/Ast.swift | 4 ++++ Sources/Environment.swift | 2 ++ Sources/Lexer.swift | 4 ++++ Sources/Parser.swift | 19 ++++++++++------- Sources/Runtime.swift | 4 +++- Sources/Template.swift | 1 + Tests/LexerTests.swift | 44 ++++++++++++++++++++++++++++++++++++++- 7 files changed, 68 insertions(+), 10 deletions(-) diff --git a/Sources/Ast.swift b/Sources/Ast.swift index 3789a2c..7460284 100644 --- a/Sources/Ast.swift +++ b/Sources/Ast.swift @@ -120,3 +120,7 @@ struct KeywordArgumentExpression: Expression { var key: Identifier var value: any Expression } + +struct NullLiteral: Literal { + var value: Any? = nil +} diff --git a/Sources/Environment.swift b/Sources/Environment.swift index 505c3ee..6394a5b 100644 --- a/Sources/Environment.swift +++ b/Sources/Environment.swift @@ -165,6 +165,8 @@ class Environment { } return ObjectValue(value: object) + case is NullValue: + return NullValue() default: throw JinjaError.runtime("Cannot convert to runtime value: \(input) type:\(type(of: input))") } diff --git a/Sources/Lexer.swift b/Sources/Lexer.swift index a49dd82..c96373a 100644 --- a/Sources/Lexer.swift +++ b/Sources/Lexer.swift @@ -12,6 +12,7 @@ enum TokenType: String { case numericLiteral = "NumericLiteral" case booleanLiteral = "BooleanLiteral" + case nullLiteral = "NullLiteral" case stringLiteral = "StringLiteral" case identifier = "Identifier" case equals = "Equals" @@ -69,8 +70,10 @@ let keywords: [String: TokenType] = [ "and": .and, "or": .or, "not": .not, + // Literals "true": .booleanLiteral, "false": .booleanLiteral, + "none": .nullLiteral, ] func isWord(char: String) -> Bool { @@ -226,6 +229,7 @@ func tokenize(_ source: String, options: PreprocessOptions = PreprocessOptions() case .identifier, .numericLiteral, .booleanLiteral, + .nullLiteral, .stringLiteral, .closeParen, .closeSquareBracket: diff --git a/Sources/Parser.swift b/Sources/Parser.swift index 4fca749..31c71f9 100644 --- a/Sources/Parser.swift +++ b/Sources/Parser.swift @@ -187,21 +187,18 @@ func parse(tokens: [Token]) throws -> Program { while typeof(.is) { current += 1 let negate = typeof(.not) - if negate { current += 1 } - var filter = try parsePrimaryExpression() - - if let boolLiteralFlter = filter as? BoolLiteral { - filter = Identifier(value: String(boolLiteralFlter.value)) + if let boolLiteralFilter = filter as? BoolLiteral { + filter = Identifier(value: String(boolLiteralFilter.value)) + } else if filter is NullLiteral { + filter = Identifier(value: "none") } - if let test = filter as? Identifier { operand = TestExpression(operand: operand as! Expression, negate: negate, test: test) - } - else { + } else { throw JinjaError.syntax("Expected identifier for the test") } } @@ -373,6 +370,9 @@ func parse(tokens: [Token]) throws -> Program { case .booleanLiteral: current += 1 return BoolLiteral(value: token.value == "true") + case .nullLiteral: + current += 1 + return NullLiteral() case .identifier: current += 1 return Identifier(value: token.value) @@ -415,6 +415,9 @@ func parse(tokens: [Token]) throws -> Program { current += 1 return ObjectLiteral(value: values) + case .nullLiteral: + current += 1 + return NullLiteral() default: throw JinjaError.syntax("Unexpected token: \(token.type)") } diff --git a/Sources/Runtime.swift b/Sources/Runtime.swift index bc01043..6a4f54f 100644 --- a/Sources/Runtime.swift +++ b/Sources/Runtime.swift @@ -693,8 +693,10 @@ struct Interpreter { return BooleanValue(value: statement.value) case let statement as FilterExpression: return try self.evaluateFilterExpression(node: statement, environment: environment) + case is NullLiteral: + return NullValue() default: - throw JinjaError.runtime("Unknown node type: \(type(of:statement))") + throw JinjaError.runtime("Unknown node type: \(type(of:statement)), statement: \(String(describing: statement))") } } else { diff --git a/Sources/Template.swift b/Sources/Template.swift index ea1605b..8f78efe 100644 --- a/Sources/Template.swift +++ b/Sources/Template.swift @@ -20,6 +20,7 @@ public struct Template { try env.set(name: "false", value: false) try env.set(name: "true", value: true) + try env.set(name: "none", value: NullValue()) try env.set( name: "raise_exception", value: { (args: String) throws in diff --git a/Tests/LexerTests.swift b/Tests/LexerTests.swift index 6640c68..8b22f9c 100644 --- a/Tests/LexerTests.swift +++ b/Tests/LexerTests.swift @@ -145,6 +145,9 @@ final class LexerTests: XCTestCase { "UNDEFINED_VARIABLES": "{{ undefined_variable }}", "UNDEFINED_ACCESS": "{{ object.undefined_attribute }}", + // Null + "NULL_VARIABLE": "{% if not null_val is defined %}{% set null_val = none %}{% endif %}{% if null_val is not none %}{{ 'fail' }}{% else %}{{ 'pass' }}{% endif %}", + // Ternary operator "TERNARY_OPERATOR": "|{{ 'a' if true else 'b' }}|{{ 'a' if false else 'b' }}|{{ 'a' if 1 + 1 == 2 else 'b' }}|{{ 'a' if 1 + 1 == 3 or 1 * 2 == 3 else 'b' }}|", @@ -2032,7 +2035,7 @@ final class LexerTests: XCTestCase { Token(value: "unknown", type: .stringLiteral), Token(value: ")", type: .closeParen), Token(value: "is", type: .is), - Token(value: "none", type: .identifier), + Token(value: "none", type: .nullLiteral), Token(value: "}}", type: .closeExpression), Token(value: "|", type: .text), Token(value: "{{", type: .openExpression), @@ -2177,6 +2180,45 @@ final class LexerTests: XCTestCase { Token(value: "}}", type: .closeExpression), ], + // Null + "NULL_VARIABLE": [ + Token(value: "{%", type: .openStatement), + Token(value: "if", type: .if), + Token(value: "not", type: .not), + Token(value: "null_val", type: .identifier), + Token(value: "is", type: .is), + Token(value: "defined", type: .identifier), + Token(value: "%}", type: .closeStatement), + Token(value: "{%", type: .openStatement), + Token(value: "set", type: .set), + Token(value: "null_val", type: .identifier), + Token(value: "=", type: .equals), + Token(value: "none", type: .nullLiteral), + Token(value: "%}", type: .closeStatement), + Token(value: "{%", type: .openStatement), + Token(value: "endif", type: .endIf), + Token(value: "%}", type: .closeStatement), + Token(value: "{%", type: .openStatement), + Token(value: "if", type: .if), + Token(value: "null_val", type: .identifier), + Token(value: "is", type: .is), + Token(value: "not", type: .not), + Token(value: "none", type: .nullLiteral), + Token(value: "%}", type: .closeStatement), + Token(value: "{{", type: .openExpression), + Token(value: "fail", type: .stringLiteral), + Token(value: "}}", type: .closeExpression), + Token(value: "{%", type: .openStatement), + Token(value: "else", type: .else), + Token(value: "%}", type: .closeStatement), + Token(value: "{{", type: .openExpression), + Token(value: "pass", type: .stringLiteral), + Token(value: "}}", type: .closeExpression), + Token(value: "{%", type: .openStatement), + Token(value: "endif", type: .endIf), + Token(value: "%}", type: .closeStatement), + ], + // Ternary operator "TERNARY_OPERATOR": [ Token(value: "|", type: .text), From 4afffc97603ddd6fc45b53aa3441bbe44c178c03 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Tue, 1 Oct 2024 19:10:02 +0200 Subject: [PATCH 3/4] Handle left StringValue with right ObjectValue --- Sources/Runtime.swift | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Sources/Runtime.swift b/Sources/Runtime.swift index 6a4f54f..6eceb7f 100644 --- a/Sources/Runtime.swift +++ b/Sources/Runtime.swift @@ -428,7 +428,20 @@ struct Interpreter { } if left is StringValue, right is ObjectValue { - throw JinjaError.syntaxNotSupported + switch node.operation.value { + case "in": + if let leftString = (left as? StringValue)?.value, + let rightObject = right as? ObjectValue { + return BooleanValue(value: rightObject.value.keys.contains(leftString)) + } + case "not in": + if let leftString = (left as? StringValue)?.value, + let rightObject = right as? ObjectValue { + return BooleanValue(value: !rightObject.value.keys.contains(leftString)) + } + default: + throw JinjaError.runtime("Unsupported operation '\(node.operation.value)' between StringValue and ObjectValue") + } } throw JinjaError.syntax( @@ -664,6 +677,17 @@ struct Interpreter { throw JinjaError.runtime("Unknown filter: \(node.filter)") } + func evaluateTestExpression(node: TestExpression, environment: Environment) throws -> any RuntimeValue { + let operand = try self.evaluate(statement: node.operand, environment: environment) + + if let testFunction = environment.tests[node.test.value] { + let result = try testFunction(operand) + return BooleanValue(value: node.negate ? !result : result) + } else { + throw JinjaError.runtime("Unknown test: \(node.test.value)") + } + } + func evaluate(statement: Statement?, environment: Environment) throws -> any RuntimeValue { if let statement { switch statement { @@ -693,6 +717,8 @@ struct Interpreter { return BooleanValue(value: statement.value) case let statement as FilterExpression: return try self.evaluateFilterExpression(node: statement, environment: environment) + case let statement as TestExpression: + return try self.evaluateTestExpression(node: statement, environment: environment) case is NullLiteral: return NullValue() default: From 2c91972bcced70c91086975d7c79a4c8418a9352 Mon Sep 17 00:00:00 2001 From: Anthony DePasquale Date: Thu, 3 Oct 2024 20:30:12 +0200 Subject: [PATCH 4/4] Remove duplicate `.nullLiteral` case from `parsePrimaryExpression` --- Sources/Parser.swift | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Sources/Parser.swift b/Sources/Parser.swift index 31c71f9..36d4b60 100644 --- a/Sources/Parser.swift +++ b/Sources/Parser.swift @@ -389,13 +389,11 @@ func parse(tokens: [Token]) throws -> Program { var values: [Expression] = [] while !typeof(.closeSquareBracket) { try values.append(parseExpression() as! Expression) - if typeof(.comma) { current += 1 } } current += 1 - return ArrayLiteral(value: values) case .openCurlyBracket: current += 1 @@ -404,20 +402,13 @@ func parse(tokens: [Token]) throws -> Program { let key = try parseExpression() try expect(type: .colon, error: "Expected colon between key and value in object literal") let value = try parseExpression() - values.append((key as! Expression, value as! Expression)) - if typeof(.comma) { current += 1 } } - current += 1 - return ObjectLiteral(value: values) - case .nullLiteral: - current += 1 - return NullLiteral() default: throw JinjaError.syntax("Unexpected token: \(token.type)") }