[Runner Fixes]: Updated Docker Image to use Alpine & Fixed Stdout Reporting

Dockerfile

@@ -1,17 +1,15 @@
-FROM python:3.11.2-slim
+FROM python:3.11.5-alpine3.18
 
 COPY requirements.txt /requirements.txt
 
 RUN pip install -r /requirements.txt
 
-RUN apt-get update \
- && apt-get install curl -y \
- && apt-get remove curl -y \
- && apt-get autoremove -y \
- && rm -rf /var/lib/apt/lists/*
+RUN apk update && apk upgrade\
+ && apk --no-cache add curl bash\
+ && apk cache clean
 
 COPY . /opt/test-runner
 
 WORKDIR /opt/test-runner
 
-ENTRYPOINT [ "/opt/test-runner/bin/run.sh" ]
+ENTRYPOINT ["sh", "/opt/test-runner/bin/run.sh" ]

bin/run-in-docker.sh

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-set -e
+#!/usr/bin/env sh
 
 # Synopsis:
 # Test runner for run.sh in a docker container
@@ -19,6 +18,9 @@ set -e
 # Example:
 # ./run-in-docker.sh two-fer ./relative/path/to/two-fer/solution/folder/ ./relative/path/to/output/directory/
 
+# Stop executing when a command returns a non-zero return code
+set -e
+
 # If arguments not provided, print usage and exit
 if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
     echo "usage: run-in-docker.sh exercise-slug ./relative/path/to/solution/folder/ ./relative/path/to/output/directory/"

bin/run-tests-in-docker.sh

@@ -24,7 +24,7 @@ docker run \
     --network none \
     --read-only \
     --mount type=bind,src="${PWD}/test",dst=/opt/test-runner/test \
-    --mount type=volume,dst=/tmp \
+    --mount type=tmpfs,dst=/tmp \
     --workdir /opt/test-runner \
     --entrypoint pytest \
     exercism/python-test-runner -vv

bin/run.sh

@@ -1,4 +1,5 @@
-#! /bin/sh
+#! /usr/bin/env sh
+
 root="$( dirname "$( cd "$( dirname "$0" )" >/dev/null 2>&1 && pwd )" )"
 export PYTHONPATH="$root:$PYTHONPATH"
-python3 bin/run.py "$@"
+/usr/bin/env python3 bin/run.py "$@"

requirements.txt

@@ -1,4 +1,4 @@
 black<=22.3.0
 pytest~=7.2.2
-pytest-subtests~=0.10.0
+pytest-subtests~=0.11.0
 tomli>=1.1.0; python_full_version < '3.11.2'

runner/__init__.py

@@ -15,6 +15,7 @@
 from .sort import TestOrder
 
 
+
 class ResultsReporter:
     def __init__(self):
         self.results = Results()
@@ -37,7 +38,6 @@ def pytest_collection_modifyitems(self, session, config, items):
             for mark in item.iter_markers(name='task'):
                 self.tests[name] = Test(name=name, task_id=mark.kwargs['taskno'])
 
-
         def _sort_by_lineno(item):
             test_id = Hierarchy(item.nodeid)
             source = Path(item.fspath)
@@ -50,28 +50,60 @@ def pytest_runtest_logreport(self, report):
         Process a test setup / call / teardown report.
         """
 
-        name = report.head_line if report.head_line else ".".join(report.nodeid.split("::")[1:])
+        name = ".".join(report.nodeid.split("::")[1:])
+        if report.head_line:
+            name = report.head_line.split(" (")[0]
 
 
+        #Add variation name to test output.
         if name not in self.tests:
             self.tests[name] = Test(name)
 
-
         state = self.tests[name]
 
         # ignore successful setup and teardown stages
         if report.passed and report.when != "call":
             return
 
-        # Update tests that have already failed with capstdout and return.
+        #Update tests that have already failed with capstdout and return.
         if not state.is_passing():
-            if report.capstdout.rstrip('FFFFFFFF ').rstrip('uuuuu'):
-                state.output = report.capstdout.rstrip('FFFFFFFF ').rstrip('uuuuu')
+
+            #Check if a report is a concept exercise subtest parent.
+            if report.capstdout:
+
+                #split up the captured stdout by subtest result.
+                captures = [item for item in report.capstdout.split('\nu')]
+                if captures[0].startswith('u'):
+                    captures[0] = captures[0][1:]
+
+                parsed_captures = []
+
+                # Insert spacers for subtests and stdout entries in correct position.
+                for item in captures:
+                    empties = len(item) - len(item.lstrip('u'))
+                    if empties > 0:
+                        for number in range(1, empties+1):
+                            parsed_captures.append(' ')
+                        parsed_captures.append(item.lstrip('u'))
+                    else: parsed_captures.append(item)
+
+                # Generate variation numbers for each subtest output section.
+                variants = (f'[variation #{number}]: {item}' for
+                            item, number in zip(parsed_captures, range(1, len(parsed_captures)+1)))
+
+                # Go through the variations and match them to self.tests.
+                # Insert matched variation output into test output field.
+                for item in variants:
+                    for name in self.tests:
+                        if item.split(":")[0] in name and report.nodeid.split("::")[2] in name:
+                            self.tests[name].output = item.split("]: ")[1]
+            else:
+                state.output = report.capstdout
             return
 
-        # Record captured relevant stdout content for passed tests.
-        if report.capstdout:
-            state.output = report.capstdout
+        else:
+            if report.capstdout:
+                state.output = report.capstdout
 
         # Handle details of test failure
         if report.failed:
@@ -108,7 +140,6 @@ def pytest_runtest_logreport(self, report):
                 )
                 self.tests[parent_test_name].test_code = state.test_code
 
-
     def pytest_sessionfinish(self, session, exitstatus):
         """
         Processes the results into a report.
@@ -209,6 +240,7 @@ def run(slug: Slug, indir: Directory, outdir: Directory, args: List[str]) -> Non
 
     # dump the report
     out_file.write_text(reporter.results.as_json())
+
     # remove cache directories
     for cache_dir in ['.pytest_cache', '__pycache__']:
         dirpath = indir / cache_dir

runner/data.py

@@ -79,6 +79,7 @@ def output(self, captured: Output) -> None:
             return
 
         captured = captured.strip()
+
         truncate_msg = " [Output was truncated. Please limit to 500 chars]"
         if len(captured) > 500:
             captured = captured[: 500 - len(truncate_msg)] + truncate_msg
@@ -141,9 +142,7 @@ def error(self, message: Message = None) -> None:
     def _factory(items):
         result = {}
         for key, value in items:
-            if key == "_output" or key in {"message", "output", "subtest"} and value is None:
-                continue
-            elif key == "_output" or key in {"message", "output", "subtest"} and "\u001b[31mF\u001b[0m" in value:
+            if key == "_output" or key in {"message", "output", "subtest"} and value in (None, "", " "):
                 continue
 
             if isinstance(value, Status):