Skip to content

Commit

Permalink
Log input files and test case paths when ONNX tests fail. (#19)
Browse files Browse the repository at this point in the history
  • Loading branch information
ScottTodd authored Aug 29, 2024
1 parent 9f2ca6d commit 9e921d0
Showing 1 changed file with 62 additions and 20 deletions.
82 changes: 62 additions & 20 deletions onnx_ops/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -336,7 +336,12 @@ def test_compile(self):
self.compile_cmd, shell=True, capture_output=True, cwd=cwd
)
if proc.returncode != 0:
raise IreeCompileException(proc, cwd, self.compile_cmd)
raise IreeCompileException(
process=proc,
cwd=cwd,
input_mlir_name=self.spec.input_mlir_name,
compile_cmd=self.compile_cmd,
)

def test_run(self):
cwd = self.test_cwd
Expand All @@ -346,7 +351,13 @@ def test_run(self):
)
proc = subprocess.run(self.run_cmd, shell=True, capture_output=True, cwd=cwd)
if proc.returncode != 0:
raise IreeRunException(proc, cwd, self.compile_cmd, self.run_cmd)
raise IreeRunException(
process=proc,
cwd=cwd,
input_mlir_name=self.spec.input_mlir_name,
compile_cmd=self.compile_cmd,
run_cmd=self.run_cmd,
)

def repr_failure(self, excinfo):
"""Called when self.runtest() raises an exception."""
Expand Down Expand Up @@ -374,7 +385,11 @@ class IreeCompileException(Exception):
"""Compiler exception that preserves the command line and output."""

def __init__(
self, process: subprocess.CompletedProcess, cwd: str, compile_cmd: str
self,
process: subprocess.CompletedProcess,
cwd: Path,
input_mlir_name: Path,
compile_cmd: str,
):
try:
errs = process.stderr.decode("utf-8")
Expand All @@ -385,23 +400,37 @@ def __init__(
except:
outs = str(process.stdout)

super().__init__(
f"Error invoking iree-compile\n"
f"Error code: {process.returncode}\n"
f"Stderr diagnostics:\n{errs}\n\n"
f"Stdout diagnostics:\n{outs}\n\n"
f"Compiled with:\n"
f" cd {cwd} && {compile_cmd}\n\n"
test_github_url = (
"https://github.com/iree-org/iree-test-suites/blob/main/onnx_ops/"
+ cwd.relative_to(THIS_DIR).as_posix()
)

# TODO(scotttodd): handle large sources somehow (truncate, skip, link to source, etc.)
with open(cwd / input_mlir_name) as f:
input_mlir = f.read()

super().__init__(
f"Error invoking iree-compile\n"
f"Error code: {process.returncode}\n"
f"Stderr diagnostics:\n{errs}\n\n"
f"Stdout diagnostics:\n{outs}\n\n"
f"Test case source:\n"
f" {test_github_url}\n\n"
f"Input program:\n"
f"```\n{input_mlir}```\n\n"
f"Compiled with:\n"
f" cd {cwd} && {compile_cmd}\n\n"
)


class IreeRunException(Exception):
"""Runtime exception that preserves the command line and output."""

def __init__(
self,
process: subprocess.CompletedProcess,
cwd: str,
cwd: Path,
input_mlir_name: Path,
compile_cmd: str,
run_cmd: str,
):
Expand All @@ -414,17 +443,30 @@ def __init__(
except:
outs = str(process.stdout)

super().__init__(
f"Error invoking iree-run-module\n"
f"Error code: {process.returncode}\n"
f"Stderr diagnostics:\n{errs}\n"
f"Stdout diagnostics:\n{outs}\n"
f"Compiled with:\n"
f" cd {cwd} && {compile_cmd}\n\n"
f"Run with:\n"
f" cd {cwd} && {run_cmd}\n\n"
test_github_url = (
"https://github.com/iree-org/iree-test-suites/blob/main/onnx_ops/"
+ cwd.relative_to(THIS_DIR).as_posix()
)

# TODO(scotttodd): handle large sources somehow (truncate, skip, link to source, etc.)
with open(cwd / input_mlir_name) as f:
input_mlir = f.read()

super().__init__(
f"Error invoking iree-run-module\n"
f"Error code: {process.returncode}\n"
f"Stderr diagnostics:\n{errs}\n"
f"Stdout diagnostics:\n{outs}\n"
f"Test case source:\n"
f" {test_github_url}\n\n"
f"Input program:\n"
f"```\n{input_mlir}```\n\n"
f"Compiled with:\n"
f" cd {cwd} && {compile_cmd}\n\n"
f"Run with:\n"
f" cd {cwd} && {run_cmd}\n\n"
)


class IreeXFailCompileRunException(Exception):
pass

0 comments on commit 9e921d0

Please sign in to comment.