diff --git a/src/main/java/com/amazon/ion/impl/macro/Environment.kt b/src/main/java/com/amazon/ion/impl/macro/Environment.kt new file mode 100644 index 000000000..dfbd05f75 --- /dev/null +++ b/src/main/java/com/amazon/ion/impl/macro/Environment.kt @@ -0,0 +1,30 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.impl.macro + +/** + * An `Environment` contains variable bindings for a given macro evaluation. + * + * The [arguments] is a list of expressions for the arguments that were passed to the current macro. + * It may also contain other expressions if the current macro invocation is part of a larger evaluation. + * + * The [argumentIndices] is a mapping from parameter index to the start of the corresponding expression in [arguments]. + * + * The [parentEnvironment] is an environment to use if any of the expressions in this environment + * contains a variable that references something from an outer macro invocation. + */ +data class Environment private constructor( + // Any variables found here have to be looked up in [parentEnvironment] + val arguments: List, + // TODO: Replace with IntArray + val argumentIndices: List, + val parentEnvironment: Environment?, +) { + fun createChild(arguments: List, argumentIndices: List) = Environment(arguments, argumentIndices, this) + companion object { + @JvmStatic + val EMPTY = Environment(emptyList(), emptyList(), null) + @JvmStatic + fun create(arguments: List, argumentIndices: List) = Environment(arguments, argumentIndices, null) + } +} diff --git a/src/main/java/com/amazon/ion/impl/macro/Macro.kt b/src/main/java/com/amazon/ion/impl/macro/Macro.kt index ab64aac2f..6870130dc 100644 --- a/src/main/java/com/amazon/ion/impl/macro/Macro.kt +++ b/src/main/java/com/amazon/ion/impl/macro/Macro.kt @@ -3,6 +3,7 @@ package com.amazon.ion.impl.macro import com.amazon.ion.impl.* +import com.amazon.ion.impl.macro.Macro.Parameter.Companion.zeroToManyTagged /** * A [Macro] is either a [SystemMacro] or a [TemplateMacro]. @@ -12,6 +13,11 @@ sealed interface Macro { data class Parameter(val variableName: String, val type: ParameterEncoding, val cardinality: ParameterCardinality) { override fun toString() = "$type::$variableName${cardinality.sigil}" + + companion object { + @JvmStatic + fun zeroToManyTagged(name: String) = Parameter(name, Macro.ParameterEncoding.Tagged, Macro.ParameterCardinality.ZeroOrMore) + } } // TODO: See if we can DRY up ParameterEncoding and PrimitiveType @@ -101,8 +107,8 @@ data class TemplateMacro(override val signature: List, val body * Macros that are built in, rather than being defined by a template. */ enum class SystemMacro(override val signature: List) : Macro { - Values(listOf(Macro.Parameter("values", Macro.ParameterEncoding.Tagged, Macro.ParameterCardinality.ZeroOrMore))), - MakeString(listOf(Macro.Parameter("text", Macro.ParameterEncoding.Tagged, Macro.ParameterCardinality.ZeroOrMore))), + Values(listOf(zeroToManyTagged("values"))), + MakeString(listOf(zeroToManyTagged("text"))), // TODO: Other system macros ; } diff --git a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt index 54b75fb0c..448d0d209 100644 --- a/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt +++ b/src/main/java/com/amazon/ion/impl/macro/MacroEvaluator.kt @@ -1,6 +1,9 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 package com.amazon.ion.impl.macro import com.amazon.ion.* +import com.amazon.ion.impl.* import com.amazon.ion.impl.macro.Expression.* /** @@ -19,11 +22,129 @@ class MacroEvaluator( // TODO: Add expansion limit ) { + /** + * Implementations must update [ExpansionInfo.i] in order for [ExpansionInfo.hasNext] to work properly. + */ + private fun interface Expander { + fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression + } + + private object SimpleExpander : Expander { + override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { + return expansionInfo.nextSourceExpression() + } + } + + private object MakeStringExpander : Expander { + override fun nextExpression(expansionInfo: ExpansionInfo, macroEvaluator: MacroEvaluator): Expression { + // Tell the macro evaluator to treat this as a values expansion... + macroEvaluator.expansionStack.peek().expansionKind = ExpansionKind.Values + val minDepth = macroEvaluator.expansionStack.size() + // ...But capture the output and turn it into a String + val sb = StringBuilder() + while (true) { + when (val expr: DataModelExpression? = macroEvaluator.expandNext(minDepth)) { + is StringValue -> sb.append(expr.value) + is SymbolValue -> sb.append(expr.value.assumeText()) + is NullValue -> {} + null -> break + is DataModelValue -> throw IonException("Invalid argument type for 'make_string': ${expr.type}") + is FieldName -> TODO("Unreachable. We shouldn't be able to get here without first encountering a StructValue.") + } + } + return StringValue(value = sb.toString()) + } + } + + private enum class ExpansionKind(val expander: Expander) { + Container(SimpleExpander), + TemplateBody(SimpleExpander), + Values(SimpleExpander), + MakeString(MakeStringExpander), + ; + + companion object { + @JvmStatic + fun forSystemMacro(macro: SystemMacro): ExpansionKind { + return when (macro) { + SystemMacro.Values -> Values + SystemMacro.MakeString -> MakeString + } + } + } + } + + private inner class ExpansionInfo : Iterator { + /** The [ExpansionKind]. */ + @JvmField var expansionKind: ExpansionKind = ExpansionKind.Values + /** + * The evaluation [Environment]—i.e. variable bindings. + */ + @JvmField var environment: Environment? = null + /** + * The [Expression]s being expanded. This MUST be the original list, not a sublist because + * (a) we don't want to be allocating new sublists all the time, and (b) the + * start and end indices of the expressions may be incorrect if a sublist is taken. + */ + @JvmField var expressions: List? = null + // /** Start of [expressions] that are applicable for this [ExpansionInfo] */ + // TODO: Do we actually need this for anything other than debugging? + // @JvmField var startInclusive: Int = 0 + /** End of [expressions] that are applicable for this [ExpansionInfo] */ + @JvmField var endExclusive: Int = 0 + /** Current position within [expressions] of this expansion */ + @JvmField var i: Int = 0 + + /** Checks if this expansion can produce any more expressions */ + override fun hasNext(): Boolean = i < endExclusive + + /** Returns the next expression from this expansion */ + override fun next(): Expression { + return expansionKind.expander.nextExpression(this, this@MacroEvaluator) + } + + /** + * Returns the next expression from the input expressions ([expressions]) of this Expansion. + * This is intended for use in [Expander] implementations. + */ + fun nextSourceExpression(): Expression { + val next = expressions!![i] + i++ + if (next is HasStartAndEnd) i = next.endExclusive + return next + } + + override fun toString() = """ + |ExpansionInfo( + | expansionKind: $expansionKind, + | environment: $environment, + | expressions: [ + | ${expressions!!.joinToString(",\n| ") { it.toString() } } + | ], + | endExclusive: $endExclusive, + | i: $i, + |) + """.trimMargin() + } + + private val expansionStack = _Private_RecyclingStack(8) { ExpansionInfo() } + + private var currentExpr: DataModelExpression? = null + /** * Initialize the macro evaluator with an E-Expression. */ fun initExpansion(encodingExpressions: List) { - TODO() + val eExp = encodingExpressions[0] + eExp as? EExpression ?: throw IllegalStateException("Attempted to initialize macro evaluator for an expression that is not an e-expression: $eExp") + + pushMacro( + eExp.address, + eExp.startInclusive, + eExp.endExclusive, + Environment.EMPTY, + encodingExpressions, + ) } /** @@ -31,14 +152,76 @@ class MacroEvaluator( * Returns null if at the end of a container or at the end of the expansion. */ fun expandNext(): DataModelExpression? { - TODO() + return expandNext(-1) + } + + /** + * Evaluate the macro expansion until the next [DataModelExpression] can be returned. + * Returns null if at the end of a container or at the end of the expansion. + * + * Treats [minDepth] as the minimum expansion depth allowed—i.e. it will not step out any further than + * [minDepth]. This is used for built-in macros when they need to delegate something to the macro evaluator + * but don't want the macro evaluator to step out beyond the invoking built-in macro. + */ + private fun expandNext(minDepth: Int): DataModelExpression? { + + /* ==== Evaluation Algorithm ==== + 01 | Check the top expansion in the expansion stack + 02 | If there is none, return null (macro expansion is over) + 03 | If there is one, but it has no more expressions... + 04 | If the expansion kind is a data-model container type, return null (user needs to step out) + 05 | If the expansion kind is not a data-model container type, automatically step out + 06 | If there is one, and it has more expressions... + 07 | If it is a scalar, return that + 08 | If it is a container, return that (user needs to step in) + 09 | If it is a variable, using parent Environment, push variable ExpansionInfo onto the stack and goto 1 + 10 | If it is an expression group, using current Environment, push expression group ExpansionInfo onto the stack and goto 1 + 11 | If it is a macro invocation, create updated Environment, push ExpansionInfo onto stack, and goto 1 + 12 | If it is an e-expression, using empty Environment, push ExpansionInfo onto stack and goto 1 + */ + + currentExpr = null + while (!expansionStack.isEmpty) { + if (!expansionStack.peek().hasNext()) { + if (expansionStack.peek().expansionKind == ExpansionKind.Container) { + // End of container. User needs to step out. + // TODO: Do we need something to distinguish End-Of-Expansion from End-Of-Container? + return null + } else { + // End of a macro invocation or something else that is not part of the data model, + // so we seamlessly close this out and continue with the parent expansion. + if (expansionStack.size() > minDepth) { + expansionStack.pop() + continue + } else { + // End of expansion for something internal. + return null + } + } + } + when (val currentExpr = expansionStack.peek().next()) { + Placeholder -> TODO("unreachable") + is MacroInvocation -> pushTdlMacroExpansion(currentExpr) + is EExpression -> pushEExpressionExpansion(currentExpr) + is VariableRef -> pushVariableExpansion(currentExpr) + is ExpressionGroup -> pushExpressionGroup(currentExpr) + is DataModelExpression -> { + this.currentExpr = currentExpr + break + } + } + } + return currentExpr } /** * Steps out of the current [DataModelContainer]. */ fun stepOut() { - TODO() + // step out of anything we find until we have stepped out of a container. + while (expansionStack.pop()?.expansionKind != ExpansionKind.Container) { + if (expansionStack.isEmpty) throw IonException("Nothing to step out of.") + } } /** @@ -46,6 +229,165 @@ class MacroEvaluator( * Throws [IonException] if not positioned on a container. */ fun stepIn() { - TODO() + val expression = requireNotNull(currentExpr) { "Not positioned on a value" } + expression as? DataModelContainer ?: throw IonException("Not positioned on a container.") + val currentExpansion = expansionStack.peek() + pushExpansion(ExpansionKind.Container, expression.startInclusive, expression.endExclusive, currentExpansion.environment!!, currentExpansion.expressions!!) + } + + /** + * Push a variable onto the expansion stack. + * + * Variables are a little bit different from other expansions. There is only one (top) expression + * in a variable expansion. It can be another variable, a value, a macro invocation, or an expression group. + * Furthermore, the current environment becomes the "source expressions" for the expansion, and the + * parent of the current environment becomes the environment in which the variable is expanded (thus + * maintaining the proper scope of variables). + */ + private fun pushVariableExpansion(expression: VariableRef) { + val currentEnvironment = expansionStack.peek().environment ?: Environment.EMPTY + val argumentExpressionIndex = currentEnvironment.argumentIndices[expression.signatureIndex] + + // Argument was elided; don't push anything so that we skip the empty expansion + if (argumentExpressionIndex < 0) return + + pushExpansion( + expansionKind = ExpansionKind.Values, + argsStartInclusive = argumentExpressionIndex, + // There can only be one expression for an argument. It's either a value, macro, or expression group. + argsEndExclusive = argumentExpressionIndex + 1, + environment = currentEnvironment.parentEnvironment ?: Environment.EMPTY, + expressions = currentEnvironment.arguments + ) + } + + private fun pushExpressionGroup(expr: ExpressionGroup) { + val currentExpansion = expansionStack.peek() + pushExpansion(ExpansionKind.Values, expr.startInclusive, expr.endExclusive, currentExpansion.environment!!, currentExpansion.expressions!!) + } + + /** + * Push a macro from a TDL macro invocation, found in the current expansion, to the expansion stack + */ + private fun pushTdlMacroExpansion(expression: MacroInvocation) { + val currentExpansion = expansionStack.peek() + pushMacro( + address = expression.address, + argsStartInclusive = expression.startInclusive, + argsEndExclusive = expression.endExclusive, + currentExpansion.environment!!, + encodingExpressions = currentExpansion.expressions!!, + ) + } + + /** + * Push a macro from the e-expression [expression] onto the expansionStack, handling concerns such as + * looking up the macro reference, setting up the environment, etc. + */ + private fun pushEExpressionExpansion(expression: EExpression) { + val currentExpansion = expansionStack.peek() + pushMacro( + address = expression.address, + argsStartInclusive = expression.startInclusive, + argsEndExclusive = expression.endExclusive, + environment = Environment.EMPTY, + encodingExpressions = currentExpansion.expressions!!, + ) + } + + /** + * Pushes a macro invocation to the expansionStack + */ + private fun pushMacro( + address: MacroRef, + argsStartInclusive: Int, + argsEndExclusive: Int, + environment: Environment, + encodingExpressions: List, + ) { + val macro = encodingContext.macroTable[address] ?: throw IonException("No such macro: $address") + + val argIndices = calculateArgumentIndices(macro, encodingExpressions, argsStartInclusive, argsEndExclusive) + + when (macro) { + is TemplateMacro -> pushExpansion( + ExpansionKind.TemplateBody, + argsStartInclusive = 0, + argsEndExclusive = macro.body.size, + expressions = macro.body, + environment = environment.createChild(encodingExpressions, argIndices) + ) + // TODO: Values and MakeString have the same code in their blocks. As we get further along, see + // if this is generally applicable for all system macros. + is SystemMacro -> { + val kind = ExpansionKind.forSystemMacro(macro) + pushExpansion(kind, argsStartInclusive, argsEndExclusive, environment, encodingExpressions,) + } + } + } + + /** + * Pushes an expansion to the expansion stack. + */ + private fun pushExpansion( + expansionKind: ExpansionKind, + argsStartInclusive: Int, + argsEndExclusive: Int, + environment: Environment, + expressions: List, + ) { + expansionStack.push { + it.expansionKind = expansionKind + it.environment = environment + it.expressions = expressions + it.i = argsStartInclusive + it.endExclusive = argsEndExclusive + } + } + + /** + * Given a [Macro] (or more specifically, its signature), calculates the position of each of its arguments + * in [encodingExpressions]. The result is a list that can be used to map from a parameter's + * signature index to the encoding expression index. Any trailing, optional arguments that are + * elided have a value of -1. + * + * This function also validates that the correct number of parameters are present. If there are + * too many parameters or too few parameters, this will throw [IonException]. + */ + private fun calculateArgumentIndices( + macro: Macro, + encodingExpressions: List, + argsStartInclusive: Int, + argsEndExclusive: Int + ): List { + // TODO: For TDL macro invocations, see if we can calculate this during the "compile" step. + var numArgs = 0 + val argsIndices = IntArray(macro.signature.size) + var currentArgIndex = argsStartInclusive + for (p in macro.signature) { + if (currentArgIndex >= argsEndExclusive) { + if (!p.cardinality.canBeVoid) throw IonException("No value provided for parameter ${p.variableName}") + // Elided rest parameter. + argsIndices[numArgs] = -1 + } else { + argsIndices[numArgs] = currentArgIndex + currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { + is HasStartAndEnd -> expr.endExclusive + else -> currentArgIndex + 1 + } + } + numArgs++ + } + while (currentArgIndex < argsEndExclusive) { + currentArgIndex = when (val expr = encodingExpressions[currentArgIndex]) { + is HasStartAndEnd -> expr.endExclusive + else -> currentArgIndex + 1 + } + numArgs++ + } + if (numArgs > macro.signature.size) { + throw IonException("Too many arguments. Expected ${macro.signature.size}, but found $numArgs") + } + return argsIndices.toList() } } diff --git a/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt new file mode 100644 index 000000000..22155128a --- /dev/null +++ b/src/test/java/com/amazon/ion/impl/macro/MacroEvaluatorTest.kt @@ -0,0 +1,616 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package com.amazon.ion.impl.macro + +import com.amazon.ion.FakeSymbolToken +import com.amazon.ion.IonType +import com.amazon.ion.impl.macro.Expression.* +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class MacroEvaluatorTest { + + // Helper object with macro table entries that can be used to make the tests more concise. + private object Macros { + val MAKE_STRING = "make_string" to SystemMacro.MakeString + val VALUES = "values" to SystemMacro.Values + val IDENTITY = "identity" to template("x!", listOf(VariableRef(0))) + + val PI = "pi" to template("", listOf(FloatValue(emptyList(), 3.14159))) + + val FOO_STRUCT = "foo_struct" to template( + "x*", + listOf( + StructValue(emptyList(), 0, 3, mapOf("foo" to listOf(2))), + FieldName(FakeSymbolToken("foo", -1)), + VariableRef(0), + ) + ) + } + + @Test + fun `a trivial constant macro evaluation`() { + // Given: + // (macro pi () 3.14159) + // When: + // (:pi) + // Then: + // 3.14159 + + val evaluator = evaluatorWithMacroTable(Macros.PI) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("pi"), 0, 1) + ) + ) + assertEquals(FloatValue(emptyList(), 3.14159), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `a nested constant macro evaluation`() { + // Given: + // (macro pi () 3.14159) + // (macro special_number () (pi)) + // When: + // (:special_number) + // Then: + // 3.14159 + + val evaluator = evaluatorWithMacroTable( + Macros.PI, + "special_number" to template( + "", + listOf( + MacroInvocation(MacroRef.ByName("pi"), 0, 1) + ) + ), + ) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("special_number"), 0, 1) + ) + ) + assertEquals(FloatValue(emptyList(), 3.14159), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `constant macro with empty list`() { + // Given: + // (macro foo () []) + // When: + // (:foo) + // Then: + // [] + + val evaluator = evaluatorWithMacroTable( + "foo" to template( + "", + listOf( + ListValue(emptyList(), 0, 1) + ) + ) + ) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("foo"), 0, 1) + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `constant macro with single element list`() { + // Given: + // (macro foo () ["a"]) + // When: + // (:foo) + // Then: + // ["a"] + + val evaluator = evaluatorWithMacroTable( + "foo" to template( + "", + listOf( + ListValue(emptyList(), 0, 2), + StringValue(value = "a"), + ) + ) + ) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("foo"), 0, 1) + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + assertEquals(StringValue(value = "a"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `constant macro with multi element list`() { + // Given: + // (macro ABCs () ["a", "b", "c"]) + // When: + // (:ABCs) + // Then: + // [ "a", "b", "c" ] + + val evaluator = evaluatorWithMacroTable( + "ABCs" to template( + "", + listOf( + ListValue(emptyList(), 0, 4), + StringValue(value = "a"), + StringValue(value = "b"), + StringValue(value = "c"), + ) + ) + ) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("ABCs"), 0, 1) + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + assertEquals(StringValue(value = "a"), evaluator.expandNext()) + assertEquals(StringValue(value = "b"), evaluator.expandNext()) + assertEquals(StringValue(value = "c"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `it should be possible to step out of a container before the end is reached`() { + // Given: + // (macro ABCs () ["a", "b", "c"]) + // When: + // (:ABCs) + // Then: + // [ "a", "b", "c" ] + + val evaluator = evaluatorWithMacroTable( + "ABCs" to template( + "", + listOf( + ListValue(emptyList(), 0, 4), + StringValue(value = "a"), + StringValue(value = "b"), + StringValue(value = "c"), + ) + ) + ) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("ABCs"), 0, 1) + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + assertEquals(StringValue(value = "a"), evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `a trivial variable substitution`() { + // Given: + // (macro identity (x!) x) + // When: + // (:identity true) + // Then: + // true + + val evaluator = evaluatorWithMacroTable(Macros.IDENTITY) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("identity"), 0, 2), + BoolValue(emptyList(), true) + ) + ) + + assertEquals(BoolValue(emptyList(), true), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `a trivial variable substitution with empty list`() { + // Given: + // (macro identity (x!) x) + // When: + // (:identity []) + // Then: + // [] + + val evaluator = evaluatorWithMacroTable(Macros.IDENTITY) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("identity"), 0, 2), + ListValue(emptyList(), 1, 2) + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `a trivial variable substitution with single element list`() { + // Given: + // (macro identity (x!) x) + // When: + // (:identity ["a"]) + // Then: + // ["a"] + + val evaluator = evaluatorWithMacroTable(Macros.IDENTITY) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("identity"), 0, 3), + ListValue(emptyList(), 1, 3), + StringValue(value = "a"), + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + assertEquals(StringValue(value = "a"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `a variable that gets used twice`() { + // Given: + // (macro double_identity (x!) [x, x]) + // When: + // (:double_identity "a") + // Then: + // ["a", "a"] + + val evaluator = evaluatorWithMacroTable( + "double_identity" to template( + "x!", + listOf( + ListValue(emptyList(), 0, 3), + VariableRef(0), + VariableRef(0), + ) + ) + ) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("double_identity"), 0, 2), + StringValue(value = "a"), + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + assertEquals(StringValue(value = "a"), evaluator.expandNext()) + assertEquals(StringValue(value = "a"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `invoke values with scalars`() { + // Given: + // When: + // (:values 1 "a") + // Then: + // 1 "a" + + val evaluator = evaluatorWithMacroTable(Macros.VALUES) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("values"), 0, 4), + ExpressionGroup(1, 4), + LongIntValue(emptyList(), 1), + StringValue(emptyList(), "a") + ) + ) + + assertEquals(LongIntValue(emptyList(), 1), evaluator.expandNext()) + assertEquals(StringValue(emptyList(), "a"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `a trivial nested variable substitution`() { + // Given: + // (macro identity (x!) x) + // (macro nested_identity (x!) (identity x)) + // When: + // (:nested_identity true) + // Then: + // true + + val evaluator = evaluatorWithMacroTable( + Macros.IDENTITY, + "nested_identity" to template( + "x!", + listOf( + MacroInvocation(MacroRef.ByName("identity"), 0, 2), + VariableRef(0) + ) + ), + ) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("nested_identity"), 0, 2), + BoolValue(emptyList(), true) + ) + ) + + assertEquals(BoolValue(emptyList(), true), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `a trivial void variable substitution`() { + // Given: + // (macro voidable_identity (x?) x) + // When: + // (:voidable_identity (:)) + // Then: + // + + val evaluator = evaluatorWithMacroTable( + "voidable_identity" to template( + "x?", + listOf( + VariableRef(0) + ) + ) + ) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("voidable_identity"), 0, 1), + ) + ) + + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `simple make_string`() { + // Given: + // When: + // (:make_string "a" "b" "c") + // Then: + // "abc" + + val evaluator = evaluatorWithMacroTable(Macros.MAKE_STRING) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("make_string"), 0, 5), + ExpressionGroup(1, 5), + StringValue(emptyList(), "a"), + StringValue(emptyList(), "b"), + StringValue(emptyList(), "c"), + ) + ) + + assertEquals(StringValue(emptyList(), "abc"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `nested make_string`() { + // Given: + // When: + // (:make_string "a" (:make_string "b" "c" "d")) + // Then: + // "abcd" + + val evaluator = evaluatorWithMacroTable(Macros.MAKE_STRING) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("make_string"), 0, 8), + ExpressionGroup(1, 8), + StringValue(emptyList(), "a"), + EExpression(MacroRef.ByName("make_string"), 3, 8), + ExpressionGroup(4, 8), + StringValue(emptyList(), "b"), + StringValue(emptyList(), "c"), + StringValue(emptyList(), "d"), + ) + ) + + assertEquals(StringValue(emptyList(), "abcd"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `macro with a variable substitution in struct field position`() { + // Given: + // (macro foo_struct (x*) {foo: x}) + // When: + // (:foo_struct bar) + // Then: + // {foo: bar} + + val evaluator = evaluatorWithMacroTable(Macros.FOO_STRUCT) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("foo_struct"), 0, 2), + StringValue(value = "bar") + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + assertEquals(FieldName(FakeSymbolToken("foo", -1)), evaluator.expandNext()) + assertEquals(StringValue(value = "bar"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `macro with a variable substitution in struct field position with multiple arguments`() { + // Given: + // (macro foo_struct (x*) {foo: x}) + // When: + // (:foo_struct (: bar baz)) + // Then: + // {foo: bar, foo: baz} + + val evaluator = evaluatorWithMacroTable(Macros.FOO_STRUCT) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("foo_struct"), 0, 4), + ExpressionGroup(1, 4), + StringValue(value = "bar"), + StringValue(value = "baz") + ) + ) + + assertIsInstance(evaluator.expandNext()) + evaluator.stepIn() + // Yes, the field name should be here only once. The Ion reader that wraps the evaluator + // is responsible for carrying the field name over to any values that follow. + assertEquals(FieldName(FakeSymbolToken("foo", -1)), evaluator.expandNext()) + assertEquals(StringValue(value = "bar"), evaluator.expandNext()) + assertEquals(StringValue(value = "baz"), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `macro with a variable substitution in struct field position with void argument`() { + // Given: + // (macro foo_struct (x*) {foo: x}) + // When: + // (:foo_struct (:)) + // Then: + // {} + + val evaluator = evaluatorWithMacroTable(Macros.FOO_STRUCT) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("foo_struct"), 0, 1), + ExpressionGroup(1, 1), + ) + ) + + assertEquals(IonType.STRUCT, (evaluator.expandNext() as? DataModelValue)?.type) + evaluator.stepIn() + // Yes, the field name should be here. The Ion reader that wraps the evaluator + // is responsible for discarding the field name if no values follow. + assertEquals(FieldName(FakeSymbolToken("foo", -1)), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + evaluator.stepOut() + assertEquals(null, evaluator.expandNext()) + } + + @Test + fun `e-expression with another e-expression as one of the arguments`() { + // Given: + // (macro pi () 3.14159) + // (macro identity (x) x) + // When: + // (:identity (:pi)) + // Then: + // 3.14159 + + val evaluator = evaluatorWithMacroTable(Macros.IDENTITY, Macros.PI) + + evaluator.initExpansion( + listOf( + EExpression(MacroRef.ByName("identity"), 0, 2), + EExpression(MacroRef.ByName("pi"), 1, 2), + ) + ) + + assertEquals(FloatValue(emptyList(), 3.14159), evaluator.expandNext()) + assertEquals(null, evaluator.expandNext()) + } + + companion object { + + /** Helper function to create template macros */ + fun template(parameters: String, body: List) = TemplateMacro(signature(parameters), body) + + /** Helper function to build a MacroEvaluator set up with a specific macro table */ + private fun evaluatorWithMacroTable(vararg idsToMacros: Pair): MacroEvaluator { + return MacroEvaluator( + EncodingContext( + idsToMacros.associate { (k, v) -> + when (k) { + is Number -> MacroRef.ById(k.toLong()) + is String -> MacroRef.ByName(k) + else -> throw IllegalArgumentException("Unsupported macro id $k") + } to v + } + ) + ) + } + + /** Helper function to turn a string into a signature. */ + private fun signature(text: String): List { + if (text.isBlank()) return emptyList() + return text.split(Regex(" +")).map { + val cardinality = Macro.ParameterCardinality.fromSigil("${it.last()}") + if (cardinality == null) { + Macro.Parameter(it, Macro.ParameterEncoding.Tagged, Macro.ParameterCardinality.ExactlyOne) + } else { + Macro.Parameter(it.dropLast(1), Macro.ParameterEncoding.Tagged, cardinality) + } + } + } + + private inline fun assertIsInstance(value: Any?) { + if (value !is T) { + val message = if (value == null) { + "Expected instance of ${T::class.qualifiedName}; was null" + } else if (null is T) { + "Expected instance of ${T::class.qualifiedName}?; was instance of ${value::class.qualifiedName}" + } else { + "Expected instance of ${T::class.qualifiedName}; was instance of ${value::class.qualifiedName}" + } + Assertions.fail(message) + } + } + } +}