Improve conformance test runner

popematt · Nov 15, 2024 · 625e1a8 · 625e1a8
1 parent 7cd94d4
commit 625e1a8
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 30 deletions.
diff --git a/src/test/java/com/amazon/ion/conformance/Config.kt b/src/test/java/com/amazon/ion/conformance/Config.kt
@@ -14,7 +14,7 @@ data class Config(
     /** Use for a skip list, or for running only one or two tests. Return true to run the test. */
     val testFilter: (File, String) -> Boolean = { _, _ -> true },
     /** Named set of reader builders (i.e. different reader configurations) to use for all tests. */
-    val readerBuilders: Map<String, IonReaderBuilder>,
+    val readerBuilder: IonReaderBuilder,
 ) {
     fun newCaseBuilder(file: File) = ConformanceTestBuilder(this, file)
 }
diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestBuilder.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestBuilder.kt
@@ -74,22 +74,18 @@ data class ConformanceTestBuilder(
     fun plus(name: String, newFragments: List<SeqElement>): ConformanceTestBuilder = copy(nameParts = nameParts + name, fragments = fragments + newFragments)
 
     fun build(executable: TestCaseSupport.() -> Unit): DynamicNode {
-        return config.readerBuilders.map { (readerName, readerBuilder) ->
-            val testName = "$fullName using $readerName"
-            val testCaseSupport = TestCaseSupport(this, readerBuilder)
-            dynamicTest(testName) {
-                if (!config.testFilter(file, testName)) throw TestAbortedException(testName)
-                debug { "Begin Test using $readerName" }
-                try {
-                    executable(testCaseSupport)
-                } catch (e: NotImplementedError) {
-                    if (config.failUnimplemented) throw e
-                    debug { "Ignored because ${e.message}" }
-                    throw TestAbortedException("$e")
-                }
+        val readerBuilder = config.readerBuilder
+        val testName = fullName
+        val testCaseSupport = TestCaseSupport(this, readerBuilder)
+        return dynamicTest(testName) {
+            if (!config.testFilter(file, testName)) throw TestAbortedException(testName)
+            try {
+                executable(testCaseSupport)
+            } catch (e: NotImplementedError) {
+                if (config.failUnimplemented) throw e
+                debug { "Ignored because ${e.message}" }
+                throw TestAbortedException("$e")
             }
-        }.let {
-            dynamicContainer(containerName, it)
         }
     }
 

diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestDslInterpreterTest.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestDslInterpreterTest.kt
@@ -18,7 +18,7 @@ object ConformanceTestDslInterpreterTest {
     private val CONFIG = Config(
         debugEnabled = true,
         failUnimplemented = false,
-        readerBuilders = mapOf("only reader" to IonReaderBuilder.standard()),
+        readerBuilder = IonReaderBuilder.standard(),
     )
 
     @JvmStatic

diff --git a/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt b/src/test/java/com/amazon/ion/conformance/ConformanceTestRunner.kt
@@ -7,15 +7,21 @@ import java.io.File
 import org.junit.jupiter.api.DynamicNode
 import org.junit.jupiter.api.TestFactory
 
-object ConformanceTestRunner {
-    val DEFAULT_READER_BUILDER_CONFIGURATIONS = mapOf(
-        "default reader" to IonReaderBuilder.standard()
-            .withCatalog(ION_CONFORMANCE_TEST_CATALOG),
-        "incremental reader" to IonReaderBuilder.standard()
-            .withCatalog(ION_CONFORMANCE_TEST_CATALOG)
-            .withIncrementalReadingEnabled(true),
-        // TODO: Other reader configurations
-    )
+object DefaultReaderConformanceTests : ConformanceTestRunner(
+    IonReaderBuilder.standard()
+        .withCatalog(ION_CONFORMANCE_TEST_CATALOG)
+)
+
+object IncrementalReaderConformanceTests : ConformanceTestRunner(
+    IonReaderBuilder.standard()
+        .withCatalog(ION_CONFORMANCE_TEST_CATALOG)
+        .withIncrementalReadingEnabled(true)
+)
+
+abstract class ConformanceTestRunner(
+    readerBuilder: IonReaderBuilder,
+    additionalSkipFilter: (File, String) -> Boolean = { _, _ -> true }
+) {
 
     private val DEFAULT_SKIP_FILTER: (File, String) -> Boolean = { file, completeTestName ->
         // `completeTestName` is the complete name of the test — that is all the test descriptions in a particular
@@ -48,6 +54,15 @@ object ConformanceTestRunner {
                 "If max_id not non-negative int, lack of exact-match must raise an error" in completeTestName -> false
                 else -> true
             }
+            "conformance/system_macros/" in file.absolutePath -> when {
+                // TODO: Implement parse_ion
+                file.endsWith("parse_ion.ion") -> false
+                // FIXME: Fails because the macro table isn't active yet
+                "in binary with a user macro address" in completeTestName -> false
+                // FIXME: com.amazon.ion.IonException: Invalid void argument for non-voidable parameter: value
+                "repeat can be invoked in binary with a system macro address" in completeTestName -> false
+                else -> true
+            }
             // Some of these are failing because
             //  - Ion Java doesn't support the Ion 1.1 system symbol table yet
             //  - The tokens `$ion_1_0` and `'$ion_1_0'` are never user values.
@@ -62,8 +77,8 @@ object ConformanceTestRunner {
     private val CONFIG = Config(
         debugEnabled = true,
         failUnimplemented = false,
-        readerBuilders = DEFAULT_READER_BUILDER_CONFIGURATIONS,
-        testFilter = DEFAULT_SKIP_FILTER,
+        readerBuilder = readerBuilder,
+        testFilter = { file, name -> DEFAULT_SKIP_FILTER(file, name) && additionalSkipFilter(file, name) },
     )
 
     @TestFactory

diff --git a/src/test/java/com/amazon/ion/conformance/fragments.kt b/src/test/java/com/amazon/ion/conformance/fragments.kt
@@ -37,7 +37,7 @@ fun AnyElement.isFragment(): Boolean {
 }
 
 // All known fragment keywords
-private val FRAGMENT_KEYWORDS = setOf("ivm", "text", "bytes", "toplevel", "encoding", "mactab")
+private val FRAGMENT_KEYWORDS = setOf("ivm", "text", "binary", "bytes", "toplevel", "encoding", "mactab")
 // Insert this between every fragment when transcoding to text
 val SERIALIZED_TEXT_FRAGMENT_SEPARATOR = "\n".toByteArray(Charsets.UTF_8)
 
@@ -112,7 +112,7 @@ fun TestCaseSupport.readFragments(fragments: List<SeqElement>): ByteArray {
     // TODO: Detect versions and switch accordingly.
     val encodeToBinary = 0 < fragments.count {
         debug { "Inspecting (${it.head} ...) at ${locationOf(it)}" }
-        it.head == "bytes"
+        it.head == "bytes" || it.head == "binary"
     }
 
     val encoding: Encoding = if (encodeToBinary) Binary10 else Text10
@@ -144,6 +144,7 @@ private fun TestCaseSupport.readFragment(fragment: SeqElement, encoding: Encodin
     return when (fragment.head) {
         "ivm" -> readIvmFragment(fragment, encoding)
         "text" -> readTextFragment(fragment, encoding)
+        "binary" -> readBytesFragment(fragment, encoding)
         "bytes" -> readBytesFragment(fragment, encoding)
         "toplevel" -> readTopLevelFragment(fragment, encoding)
         "mactab" -> TODO("mactab")