Automate updating expected failure lists in test suite configs. (iree…

…-org#17207)

For more information about this, see
nod-ai/SHARK-TestSuite#204.

* When pytest runs, it now produces structured logs using
`--report-log=logs.json` (via https://pypi.org/project/pytest-reportlog/
and some 'user_properties' written out for each test case in our
`conftest.py`)
* A new `update_config_xfails.py` script now parses through those logs
and uses the test outcomes to update the lists of expected compile and
run failures in the provided config.json file
* Some workflow logic connects both of these together and then uploads
the new config files (if the test run failed). These can be download
from the Summary page of a workflow run e.g.
https://github.com/iree-org/iree/actions/runs/8855202115?pr=17207

![image](https://github.com/iree-org/iree/assets/4010439/c457b64f-df02-4a3c-8cc6-a9b7fb323ad1)

.github/workflows/pkgci_regression_test_amdgpu_rocm.yml

@@ -60,7 +60,7 @@ jobs:
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
           repository: nod-ai/SHARK-TestSuite
-          ref: 320edabdfba7f5cf46e5cd88569fca52308d8988
+          ref: 816a8af832bd8518cf966e92cf5c2929d5c11a0f
           path: SHARK-TestSuite
           submodules: false
       - name: Installing external TestSuite Python requirements
@@ -74,7 +74,23 @@ jobs:
           pytest SHARK-TestSuite/iree_tests/onnx/ \
               -rpfE --timeout=30 --retries=2 --durations=20 \
               --config-files=build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json \
-              --no-skip-tests-missing-files
+              --no-skip-tests-missing-files \
+              --report-log=/tmp/iree_tests_onnx_gpu_rocm_rdna3_logs.json
+      - name: "Updating config file with latest XFAIL lists"
+        if: failure()
+        run: |
+          source ${VENV_DIR}/bin/activate
+          python SHARK-TestSuite/iree_tests/update_config_xfails.py \
+            --log-file=/tmp/iree_tests_onnx_gpu_rocm_rdna3_logs.json \
+            --config-file=build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json
+          cat build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json
+      - name: "Uploading new config file"
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: "onnx_gpu_rocm_rdna3.json"
+          path: "build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json"
+
   linux_x86_64_rocm_models:
     name: MI250 - Models
     runs-on: nodai-amdgpu-mi250-x86-64

.github/workflows/pkgci_regression_test_amdgpu_vulkan.yml

@@ -57,7 +57,7 @@ jobs:
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
           repository: nod-ai/SHARK-TestSuite
-          ref: 320edabdfba7f5cf46e5cd88569fca52308d8988
+          ref: 816a8af832bd8518cf966e92cf5c2929d5c11a0f
           path: SHARK-TestSuite
           submodules: false
       - name: Installing external TestSuite Python requirements
@@ -106,7 +106,7 @@ jobs:
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
           repository: nod-ai/SHARK-TestSuite
-          ref: 320edabdfba7f5cf46e5cd88569fca52308d8988
+          ref: 816a8af832bd8518cf966e92cf5c2929d5c11a0f
           path: SHARK-TestSuite
           submodules: false
           lfs: true
@@ -124,4 +124,19 @@ jobs:
           pytest SHARK-TestSuite/iree_tests/pytorch/models -k real_weights \
                -rpfxE --timeout=600 --durations=0 \
               --config-files=build_tools/pkgci/external_test_suite/pytorch_models_gpu_vulkan.json \
-              --no-skip-tests-missing-files
+              --no-skip-tests-missing-files \
+              --report-log=/tmp/iree_tests_pytorch_models_gpu_vulkan_logs.json
+      - name: "Updating config file with latest XFAIL lists"
+        if: failure()
+        run: |
+          source ${VENV_DIR}/bin/activate
+          python SHARK-TestSuite/iree_tests/update_config_xfails.py \
+            --log-file=/tmp/iree_tests_pytorch_models_gpu_vulkan_logs.json \
+            --config-file=build_tools/pkgci/external_test_suite/pytorch_models_gpu_vulkan.json
+          cat build_tools/pkgci/external_test_suite/pytorch_models_gpu_vulkan.json
+      - name: "Uploading new config file"
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: "pytorch_models_gpu_vulkan.json"
+          path: "build_tools/pkgci/external_test_suite/pytorch_models_gpu_vulkan.json"

.github/workflows/pkgci_regression_test_cpu.yml

@@ -57,7 +57,7 @@ jobs:
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
           repository: nod-ai/SHARK-TestSuite
-          ref: 320edabdfba7f5cf46e5cd88569fca52308d8988
+          ref: 816a8af832bd8518cf966e92cf5c2929d5c11a0f
           path: SHARK-TestSuite
           submodules: false
           lfs: true
@@ -71,7 +71,22 @@ jobs:
           pytest SHARK-TestSuite/iree_tests/onnx/ \
               -n auto -rpfE --timeout=30 --retries=2 --durations=20 \
               --config-files=build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json \
-              --no-skip-tests-missing-files
+              --no-skip-tests-missing-files \
+              --report-log=/tmp/iree_tests_onnx_cpu_llvm_sync_logs.json
+      - name: "Updating config file with latest XFAIL lists"
+        if: failure()
+        run: |
+          source ${VENV_DIR}/bin/activate
+          python SHARK-TestSuite/iree_tests/update_config_xfails.py \
+            --log-file=/tmp/iree_tests_onnx_cpu_llvm_sync_logs.json \
+            --config-file=build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json
+          cat build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json
+      - name: "Uploading new config file"
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: "onnx_cpu_llvm_sync.json"
+          path: "build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json"
 
   # Note: this is a persistent runner with a local cache. Downloading all input
   # files (50GB+) without a cache can take 20+ minutes.
@@ -107,7 +122,7 @@ jobs:
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
           repository: nod-ai/SHARK-TestSuite
-          ref: 320edabdfba7f5cf46e5cd88569fca52308d8988
+          ref: 816a8af832bd8518cf966e92cf5c2929d5c11a0f
           path: SHARK-TestSuite
           submodules: false
           lfs: true

.github/workflows/pkgci_regression_test_nvidiagpu_cuda.yml

@@ -57,7 +57,7 @@ jobs:
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
           repository: nod-ai/SHARK-TestSuite
-          ref: 320edabdfba7f5cf46e5cd88569fca52308d8988
+          ref: 816a8af832bd8518cf966e92cf5c2929d5c11a0f
           path: SHARK-TestSuite
           submodules: false
       - name: Installing external TestSuite Python requirements
@@ -70,4 +70,19 @@ jobs:
           pytest SHARK-TestSuite/iree_tests/onnx/ \
               -n 4 -rpfE --timeout=30 --retries=2 --durations=20 \
               --config-files=build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json \
-              --no-skip-tests-missing-files
+              --no-skip-tests-missing-files \
+              --report-log=/tmp/iree_tests_onnx_gpu_cuda_logs.json
+      - name: "Updating config file with latest XFAIL lists"
+        if: failure()
+        run: |
+          source ${VENV_DIR}/bin/activate
+          python SHARK-TestSuite/iree_tests/update_config_xfails.py \
+            --log-file=/tmp/iree_tests_onnx_gpu_cuda_logs.json \
+            --config-file=build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json
+          cat build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json
+      - name: "Uploading new config file"
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: "onnx_gpu_cuda.json"
+          path: "build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json"

.github/workflows/pkgci_regression_test_nvidiagpu_vulkan.yml

@@ -57,7 +57,7 @@ jobs:
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
           repository: nod-ai/SHARK-TestSuite
-          ref: 320edabdfba7f5cf46e5cd88569fca52308d8988
+          ref: 816a8af832bd8518cf966e92cf5c2929d5c11a0f
           path: SHARK-TestSuite
           submodules: false
       - name: Installing external TestSuite Python requirements
@@ -70,4 +70,19 @@ jobs:
           pytest SHARK-TestSuite/iree_tests/onnx/ \
               -n 4 -rpfE --timeout=30 --retries=2 --durations=20 \
               --config-files=build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json \
-              --no-skip-tests-missing-files
+              --no-skip-tests-missing-files \
+              --report-log=/tmp/iree_tests_onnx_gpu_vulkan_logs.json
+      - name: "Updating config file with latest XFAIL lists"
+        if: failure()
+        run: |
+          source ${VENV_DIR}/bin/activate
+          python SHARK-TestSuite/iree_tests/update_config_xfails.py \
+            --log-file=/tmp/iree_tests_onnx_gpu_vulkan_logs.json \
+            --config-file=build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json
+          cat build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json
+      - name: "Uploading new config file"
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: "onnx_gpu_vulkan.json"
+          path: "build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json"

build_tools/pkgci/external_test_suite/onnx_cpu_llvm_sync.json

@@ -1,15 +1,14 @@
 {
   "config_name": "cpu_llvm_sync",
-  "iree_compile_flags" : [
+  "iree_compile_flags": [
     "--iree-hal-target-backends=llvm-cpu"
   ],
   "iree_run_module_flags": [
     "--device=local-sync"
   ],
   "skip_compile_tests": [
-    // TODO(#16666): Fix these tests hanging at runtime (miscompiles?)
     "test_dequantizelinear",
-    "test_slice_default_axes",
+    "test_slice_default_axes"
   ],
   "skip_run_tests": [],
   "expected_compile_failures": [
@@ -601,7 +600,7 @@
     "test_unique_sorted_with_negative_axis",
     "test_unique_sorted_without_axis",
     "test_upsample_nearest",
-    "test_wrap_pad",
+    "test_wrap_pad"
   ],
   "expected_run_failures": [
     "test_add_uint8",
@@ -788,6 +787,6 @@
     "test_tril_square",
     "test_tril_square_neg",
     "test_tril_zero",
-    "test_where_long_example",
+    "test_where_long_example"
   ]
 }

build_tools/pkgci/external_test_suite/onnx_gpu_cuda.json

@@ -7,12 +7,10 @@
     "--device=cuda"
   ],
   "skip_compile_tests": [
-    // TODO(#16666): Fix these tests hanging at runtime (miscompiles?)
     "test_dequantizelinear",
     "test_slice_default_axes"
   ],
   "skip_run_tests": [
-    // Hanging on some machines?
     "test_gather_elements_negative_indices",
     "test_gridsample_zeros_padding",
     "test_scatter_elements_with_negative_indices"
@@ -718,22 +716,18 @@
     "test_max_int64",
     "test_max_int8",
     "test_max_uint16",
-    "test_max_uint16",
     "test_max_uint32",
     "test_max_uint64",
     "test_max_uint8",
-    "test_max_uint8",
     "test_min_float64",
     "test_min_int16",
     "test_min_int32",
     "test_min_int64",
     "test_min_int8",
     "test_min_uint16",
-    "test_min_uint16",
     "test_min_uint32",
     "test_min_uint64",
     "test_min_uint8",
-    "test_min_uint8",
     "test_mod_broadcast",
     "test_mod_int64_fmod",
     "test_mod_mixed_sign_float64",

build_tools/pkgci/external_test_suite/onnx_gpu_rocm_rdna3.json

@@ -8,9 +8,8 @@
     "--device=hip"
   ],
   "skip_compile_tests": [
-    // TODO(#16666): Fix these tests hanging at runtime (miscompiles?)
     "test_dequantizelinear",
-    "test_slice_default_axes",
+    "test_slice_default_axes"
   ],
   "skip_run_tests": [],
   "expected_compile_failures": [
@@ -599,7 +598,7 @@
     "test_unique_sorted_with_negative_axis",
     "test_unique_sorted_without_axis",
     "test_upsample_nearest",
-    "test_wrap_pad",
+    "test_wrap_pad"
   ],
   "expected_run_failures": [
     "test_add_uint8",
@@ -814,6 +813,6 @@
     "test_tril_square",
     "test_tril_square_neg",
     "test_tril_zero",
-    "test_where_long_example",
+    "test_where_long_example"
   ]
 }

build_tools/pkgci/external_test_suite/onnx_gpu_vulkan.json

@@ -1,15 +1,14 @@
 {
   "config_name": "gpu_vulkan",
-  "iree_compile_flags" : [
+  "iree_compile_flags": [
     "--iree-hal-target-backends=vulkan-spirv"
   ],
   "iree_run_module_flags": [
     "--device=vulkan"
   ],
   "skip_compile_tests": [
-    // TODO(#16666): Fix these tests hanging at runtime (miscompiles?)
     "test_dequantizelinear",
-    "test_slice_default_axes",
+    "test_slice_default_axes"
   ],
   "skip_run_tests": [],
   "expected_compile_failures": [
@@ -630,7 +629,7 @@
     "test_unique_sorted_with_negative_axis",
     "test_unique_sorted_without_axis",
     "test_upsample_nearest",
-    "test_wrap_pad",
+    "test_wrap_pad"
   ],
   "expected_run_failures": [
     "test_add_uint8",
@@ -743,20 +742,16 @@
     "test_max_int32",
     "test_max_int64",
     "test_max_int8",
-    "test_max_uint16",
     "test_max_uint32",
     "test_max_uint64",
-    "test_max_uint8",
     "test_min_float16",
     "test_min_float64",
     "test_min_int16",
     "test_min_int32",
     "test_min_int64",
     "test_min_int8",
-    "test_min_uint16",
     "test_min_uint32",
     "test_min_uint64",
-    "test_min_uint8",
     "test_mod_broadcast",
     "test_mod_mixed_sign_int16",
     "test_mod_mixed_sign_int32",
@@ -834,6 +829,6 @@
     "test_where_long_example",
     "test_xor_bcast3v1d",
     "test_xor_bcast4v2d",
-    "test_xor_bcast4v4d",
+    "test_xor_bcast4v4d"
   ]
 }