From 66ddddc93dc78c21cea184949d7516f942e3a2f8 Mon Sep 17 00:00:00 2001
From: zerbina <100542850+zerbina@users.noreply.github.com>
Date: Thu, 25 Jul 2024 16:23:06 +0000
Subject: [PATCH 1/2] tester: add support for multiple output fragments

* prepare the tester for supporting an arbitrary number of output
  fragments
* the runner signals the end of an output fragment via the `!BREAK!`
  string
* more fragments than what the specification requires is not an error
* the `output` specification specifies what the *last* output fragment
  has to be
* a file with the `.expected` extension and the same name as the test
  specifies and additional output fragment
---
 tools/tester.nim | 65 ++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 52 insertions(+), 13 deletions(-)

diff --git a/tools/tester.nim b/tools/tester.nim
index bbdb40e..8d8049e 100644
--- a/tools/tester.nim
+++ b/tools/tester.nim
@@ -21,11 +21,17 @@ import
   ]
 
 type
+  OutputSpec = object
+    fromFile: bool
+      ## whether the expected output is provided by a file
+    output: string
+      ## either a file path or the expected output
+
   TestSpec = object
     ## A test specification. Contains information about a test and what to
     ## expect from it.
-    expected: Option[string]
-      ## the output expected from the runner (if any)
+    expected: seq[OutputSpec]
+      ## the output(s) expected from the runner (if any)
     knownIssue: Option[string]
       ## whether the test is currently expected to fail due to a known issue
 
@@ -63,6 +69,38 @@ proc exec(cmd: string, args: openArray[string]): tuple[output: string,
   result.code = p.peekExitCode()
   p.close()
 
+proc content(s: OutputSpec): string =
+  ## Returns the string to compare the actual runner output against.
+  if s.fromFile:
+    let f = open(s.output, fmRead)
+    defer: f.close()
+    readAll(f)
+  else:
+    s.output
+
+proc compare(res: tuple[output: string, code: int], spec: TestSpec): TestResult =
+  ## Compares the runner output `res` against the `spec`.
+  if res.code == 0:
+    var i = 0
+    for got in split(res.output, "!BREAK!"):
+      if i < spec.expected.len:
+        let expect = content(spec.expected[i])
+        if got != expect:
+          return TestResult(kind: rkMismatch, got: got, expected: expect)
+
+      inc i
+
+    if i >= spec.expected.len:
+      # more output fragments being provided than there exist expectations for
+      # is fine
+      result = TestResult(kind: rkSuccess)
+    else:
+      # not enough output fragments
+      result = TestResult(kind: rkMismatch, got: "",
+                          expected: content(spec.expected[i]))
+  else:
+    result = TestResult(kind: rkError, output: res.output)
+
 var
   nimExe = findExe("nim")
   runner = ""
@@ -154,6 +192,13 @@ else:
     quit(1)
 
   var spec: TestSpec
+
+  block:
+    # check for and register files storing expected output
+    let expectedFile = changeFileExt(file, "expected")
+    if fileExists(expectedFile):
+      spec.expected.add OutputSpec(fromFile: true, output: expectedFile)
+
   # parse the specification, if any:
   if s.readLine() == "discard \"\"\"":
     var lines: string
@@ -176,7 +221,9 @@ else:
         of "knownIssue":
           spec.knownIssue = some evt.value
         of "output":
-          spec.expected = some strip(evt.value, leading=true, trailing=false)
+          spec.expected.add:
+            OutputSpec(fromFile: false,
+                       output: strip(evt.value, leading=true, trailing=false))
         else:
           echo "unknown key: ", evt.key
           quit(1)
@@ -192,16 +239,8 @@ else:
   else:
     s.close()
 
-  # execute the runner:
-  let execRes = exec(runner, [file])
-
-  var res = TestResult(kind: rkSuccess) # start with success
-  if execRes.code == 0:
-    if spec.expected.isSome and spec.expected.get != execRes.output:
-      res = TestResult(kind: rkMismatch, got: execRes.output,
-                       expected: spec.expected.get)
-  else:
-    res = TestResult(kind: rkError, output: execRes.output)
+  # execute the runner and check the output:
+  var res = compare(exec(runner, [file]), spec)
 
   # handle the "known issue" specification:
   if spec.knownIssue.isSome:

From 862269d0147cafa9dd2b2fafc6c316b238dbc94a Mon Sep 17 00:00:00 2001
From: zerbina <100542850+zerbina@users.noreply.github.com>
Date: Thu, 25 Jul 2024 16:23:07 +0000
Subject: [PATCH 2/2] tester: support specifying extra runner arguments

---
 tools/tester.nim | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tools/tester.nim b/tools/tester.nim
index 8d8049e..68757a8 100644
--- a/tools/tester.nim
+++ b/tools/tester.nim
@@ -30,6 +30,8 @@ type
   TestSpec = object
     ## A test specification. Contains information about a test and what to
     ## expect from it.
+    arguments: seq[string]
+      ## extra arguments to pass to the runner
     expected: seq[OutputSpec]
       ## the output(s) expected from the runner (if any)
     knownIssue: Option[string]
@@ -224,6 +226,8 @@ else:
           spec.expected.add:
             OutputSpec(fromFile: false,
                        output: strip(evt.value, leading=true, trailing=false))
+        of "arguments":
+          spec.arguments = split(evt.value, ' ')
         else:
           echo "unknown key: ", evt.key
           quit(1)
@@ -239,8 +243,11 @@ else:
   else:
     s.close()
 
+  var args = spec.arguments
+  args.add file
+
   # execute the runner and check the output:
-  var res = compare(exec(runner, [file]), spec)
+  var res = compare(exec(runner, args), spec)
 
   # handle the "known issue" specification:
   if spec.knownIssue.isSome: