SYCL: Alternative solution to avoid runtime error of launching kernel…

… on xpu:1 when que… (#769) …rying SYCL kernel bundle ahead The kernel won't be built for devices except for the first device. Launching kernel on devices except for the first device will raise runtime error. Here is an alternative as a temporary solution to provide an extra hint to SYCL runtime. intel/llvm#15127 --------- Signed-off-by: Feng Yuan <[email protected]> Co-authored-by: Wang, Chuanqi <[email protected]>
intel · Sep 29, 2024 · 459f92c · 459f92c
1 parent 151a19e
commit 459f92c
Show file tree

Hide file tree

Showing 27 changed files with 58 additions and 29 deletions.
diff --git a/.github/workflows/_linux_ut.yml b/.github/workflows/_linux_ut.yml
@@ -22,7 +22,7 @@ on:
         required: true
         type: string
         default: ''
-        description: UT scope. `op_example,op_extended,op_ut,torch_xpu` Delimiter is comma
+        description: UT scope. `op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu` Delimiter is comma
       abi:
         required: false
         type: string
@@ -118,24 +118,38 @@ jobs:
           cd ..
           python pytorch/torch/utils/collect_env.py
       - name: Run XPU OP Examples
-        if: contains(inputs.ut, 'op_example') || github.event_name == 'schedule'
+        if: contains(inputs.ut, 'op_regression') || github.event_name == 'schedule'
         run: |
           cd ${{ github.workspace }}
           xpu-smi discovery
           source .github/scripts/env.sh
           source activate xpu_op_${ZE_AFFINITY_MASK}
           cd ${{ github.workspace }}
-          cd examples
+          cd test/regressions
           pip install pytest
-          timeout 8000 pytest -v 
+          timeout 8000 pytest -v
+      - name: Run XPU OP Regressions test on device 1
+        if: contains(inputs.ut, 'op_regression_dev1') || github.event_name == 'schedule'
+        run: |
+          cd ${{ github.workspace }}
+          xpu-smi discovery
+          source .github/scripts/env.sh
+          source activate xpu_op_${ZE_AFFINITY_MASK}
+          export ZE_AFFINITY_MASK_OLD=${ZE_AFFINITY_MASK}
+          unset ZE_AFFINITY_MASK
+          cd ${{ github.workspace }}
+          cd test/regressions
+          pip install pytest
+          timeout 8000 pytest -v test_operation_on_device_1.py
+          export ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK_OLD}
       - name: Run XPU OP Extended UT
         if: contains(inputs.ut, 'op_extended') || github.event_name == 'schedule'
         run: |
           source .github/scripts/env.sh
           source activate xpu_op_${ZE_AFFINITY_MASK}
           export PYTORCH_TEST_WITH_SLOW=1
           cd ../pytorch/third_party/torch-xpu-ops/test/xpu/extended/
-          timeout 10000 python run_test_with_skip.py 
+          timeout 10000 python run_test_with_skip.py
       - name: Run XPU OP UT
         if: contains(inputs.ut, 'op_ut') || github.event_name == 'schedule'
         run: |

diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml
@@ -22,7 +22,7 @@ on:
         required: false
         type: string
         default: 'torch_xpu'
-        description: UT scope. `op_example,op_extended,op_ut,torch_xpu`. Delimiter is comma
+        description: UT scope. `op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu`. Delimiter is comma
       triton:
         required: false
         type: string
@@ -69,20 +69,20 @@ jobs:
   Linux-Nightly-Ondemand-UT-Tests:
     if: github.event_name == 'schedule' || ${{ inputs.ut_suite }}
     uses: ./.github/workflows/_linux_ut.yml
-    with: 
+    with:
       keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }}
-      ut: ${{ github.event_name == 'schedule' && 'op_example,op_extended,op_ut,torch_xpu' || inputs.ut }}
+      ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu' || inputs.ut }}
       pytorch: ${{ github.event_name == 'schedule' && 'main' || inputs.pytorch }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}
       triton: ${{ github.event_name == 'schedule' && '' || inputs.triton }}
       runner: linux.idc.xpu
-  
+
   Linux-Weekly-UT-Tests-ABI-0:
     if: github.event_name == 'schedule' && github.event.schedule == '0 16 * * 5'
     uses: ./.github/workflows/_linux_ut.yml
-    with: 
+    with:
       abi: 0
-      ut: op_example,op_extended,op_ut,torch_xpu
+      ut: op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu
       runner: linux.idc.xpu
 
   Linux-Nightly-Ondemand-E2E-Tests:
@@ -93,7 +93,7 @@ jobs:
     env:
       pytorch: ${{ github.event_name == 'schedule' && 'main' || inputs.pytorch }}
       keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }}
-      ut: ${{ github.event_name == 'schedule' && 'op_example,op_extended,op_ut,torch_xpu' || inputs.ut }}
+      ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu' || inputs.ut }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}
     outputs:
       TORCH_BRANCH_ID: ${{ steps.pinned.outputs.TORCH_BRANCH_ID }}

diff --git a/.github/workflows/nightly_ondemand_rolling.yml b/.github/workflows/nightly_ondemand_rolling.yml
@@ -22,7 +22,7 @@ on:
         required: false
         type: string
         default: 'torch_xpu'
-        description: UT scope. `op_example,op_extended,op_ut,torch_xpu`. Delimiter is comma
+        description: UT scope. `op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu`. Delimiter is comma
       triton:
         required: false
         type: string
@@ -69,19 +69,19 @@ jobs:
   Linux-Nightly-Ondemand-UT-Tests-Rolling:
     if: github.event_name == 'schedule' || ${{ inputs.ut_suite }}
     uses: ./.github/workflows/_linux_ut.yml
-    with: 
+    with:
       keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }}
-      ut: ${{ github.event_name == 'schedule' && 'op_example,op_extended,op_ut,torch_xpu' || inputs.ut }}
+      ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu' || inputs.ut }}
       pytorch: ${{ github.event_name == 'schedule' && 'main' || inputs.pytorch }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}
       runner: pvc_rolling
-  
+
   Linux-Weekly-UT-Tests-ABI-0-Rolling:
     if: github.event_name == 'schedule' && github.event.schedule == '30 16 * * 5'
     uses: ./.github/workflows/_linux_ut.yml
-    with: 
+    with:
       abi: 0
-      ut: op_example,op_extended,op_ut,torch_xpu
+      ut: op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu
       runner: pvc_rolling
 
   Linux-Nightly-Ondemand-E2E-Tests-Rolling:
@@ -92,7 +92,7 @@ jobs:
     env:
       pytorch: ${{ github.event_name == 'schedule' && 'main' || inputs.pytorch }}
       keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }}
-      ut: ${{ github.event_name == 'schedule' && 'op_example,op_extended,op_ut,torch_xpu' || inputs.ut }}
+      ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu' || inputs.ut }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}
     outputs:
       TORCH_BRANCH_ID: ${{ steps.pinned.outputs.TORCH_BRANCH_ID }}

diff --git a/.github/workflows/nightly_ondemand_whl.yml b/.github/workflows/nightly_ondemand_whl.yml
@@ -17,7 +17,7 @@ on:
         required: false
         type: string
         default: 'torch_xpu'
-        description: UT scope. `op_example,op_extended,op_ut,torch_xpu`. Delimiter is comma
+        description: UT scope. `op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu`. Delimiter is comma
       suite:
         required: true
         type: string
@@ -59,8 +59,8 @@ jobs:
   Linux-Nightly-Ondemand-UT-WHL-Tests:
     if: github.event_name == 'schedule' || ${{ inputs.ut_suite }}
     uses: ./.github/workflows/_linux_ut.yml
-    with: 
-      ut: ${{ github.event_name == 'schedule' && 'op_example,op_extended,op_ut,torch_xpu' || inputs.ut }}
+    with:
+      ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu' || inputs.ut }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}
       pytorch: nightly_wheel
       runner: linux.idc.xpu
@@ -72,7 +72,7 @@ jobs:
     timeout-minutes: 3600
     env:
       pytorch: ${{ github.event_name == 'schedule' && 'nightly' || inputs.pytorch }}
-      ut: ${{ github.event_name == 'schedule' && 'op_example,op_extended,op_ut,torch_xpu' || inputs.ut }}
+      ut: ${{ github.event_name == 'schedule' && 'op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu' || inputs.ut }}
       python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }}
     outputs:
       TORCH_BRANCH_ID: ${{ steps.pinned.outputs.TORCH_BRANCH_ID }}

diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml
@@ -23,15 +23,15 @@ jobs:
     # Don't run on forked repos and draft PRs
     if: ${{ (github.repository_owner == 'intel') && (github.event.pull_request.draft == false) }}
     uses: ./.github/workflows/_linux_ut.yml
-    with: 
-      ut: op_example,op_extended,op_ut
+    with:
+      ut: op_regression,op_regression_dev1,op_extended,op_ut
       runner: linux.idc.xpu
 
   preci-ut-abi-0:
     # Don't run on forked repos and draft PRs
     if: ${{ (github.repository_owner == 'intel') && (github.event.pull_request.draft == false) }}
     uses: ./.github/workflows/_linux_ut.yml
-    with: 
+    with:
       abi: 0
       ut: op_extended
       runner: linux.idc.xpu

diff --git a/src/comm/DeviceProperties.h b/src/comm/DeviceProperties.h
@@ -16,8 +16,13 @@ static int64_t syclMaxWorkGroupSize(
   auto dev = q.get_device();
 
   auto kid = ::sycl::get_kernel_id<KernelClass>();
-  auto kbundle =
-      ::sycl::get_kernel_bundle<::sycl::bundle_state::executable>(ctx, {kid});
+  // The kernel won't be built for devices except for the first device.
+  // Launching kernel on devices except for the first device will raise
+  // runtime error. Here is an alternative as a temporary solution to
+  // provide an extra hint to SYCL runtime.
+  // https://github.com/intel/llvm/issues/15127
+  auto kbundle = ::sycl::get_kernel_bundle<::sycl::bundle_state::executable>(
+      ctx, {dev}, {kid});
 
   ::sycl::kernel k = kbundle.get_kernel(kid);
   return k.get_info<::sycl::info::kernel_device_specific::work_group_size>(dev);

diff --git a/examples/test_binary.py → test/regressions/test_binary.py b/examples/test_binary.py → test/regressions/test_binary.py
diff --git a/examples/test_cat.py → test/regressions/test_cat.py b/examples/test_cat.py → test/regressions/test_cat.py
diff --git a/examples/test_clamp_promotion.py → test/regressions/test_clamp_promotion.py b/examples/test_clamp_promotion.py → test/regressions/test_clamp_promotion.py
diff --git a/examples/test_compare.py → test/regressions/test_compare.py b/examples/test_compare.py → test/regressions/test_compare.py
diff --git a/examples/test_copy.py → test/regressions/test_copy.py b/examples/test_copy.py → test/regressions/test_copy.py
diff --git a/examples/test_div_mode.py → test/regressions/test_div_mode.py b/examples/test_div_mode.py → test/regressions/test_div_mode.py
diff --git a/examples/test_foreach_list.py → test/regressions/test_foreach_list.py b/examples/test_foreach_list.py → test/regressions/test_foreach_list.py
diff --git a/examples/test_foreach_scalar.py → test/regressions/test_foreach_scalar.py b/examples/test_foreach_scalar.py → test/regressions/test_foreach_scalar.py
diff --git a/examples/test_foreach_scalarlist.py → test/regressions/test_foreach_scalarlist.py b/examples/test_foreach_scalarlist.py → test/regressions/test_foreach_scalarlist.py
diff --git a/examples/test_index_and_index_put.py → test/regressions/test_index_and_index_put.py b/examples/test_index_and_index_put.py → test/regressions/test_index_and_index_put.py
diff --git a/examples/test_loops.py → test/regressions/test_loops.py b/examples/test_loops.py → test/regressions/test_loops.py
diff --git a/examples/test_nms.py → test/regressions/test_nms.py b/examples/test_nms.py → test/regressions/test_nms.py
diff --git a/test/regressions/test_operation_on_device_1.py b/test/regressions/test_operation_on_device_1.py
@@ -0,0 +1,10 @@
+import torch
+from torch.testing._internal.common_utils import TestCase
+
+class TestOperationOnDevice1(TestCase):
+    def test_sum_on_device1(self, dtype=torch.float):
+        if torch.xpu.device_count() >= 2:
+            a = torch.randn(2, 3, device=torch.device('xpu:1'))
+            res = a.sum()
+            ref = a.cpu().sum()
+            self.assertEqual(ref, res)
diff --git a/examples/test_rand.py → test/regressions/test_rand.py b/examples/test_rand.py → test/regressions/test_rand.py
diff --git a/examples/test_resize.py → test/regressions/test_resize.py b/examples/test_resize.py → test/regressions/test_resize.py
diff --git a/examples/test_sort.py → test/regressions/test_sort.py b/examples/test_sort.py → test/regressions/test_sort.py
diff --git a/examples/test_tensor_factory.py → test/regressions/test_tensor_factory.py b/examples/test_tensor_factory.py → test/regressions/test_tensor_factory.py
diff --git a/examples/test_torchvision_roi_align.py → ...regressions/test_torchvision_roi_align.py b/examples/test_torchvision_roi_align.py → ...regressions/test_torchvision_roi_align.py
diff --git a/examples/test_unary.py → test/regressions/test_unary.py b/examples/test_unary.py → test/regressions/test_unary.py
diff --git a/examples/test_upsample_nearest.py → test/regressions/test_upsample_nearest.py b/examples/test_upsample_nearest.py → test/regressions/test_upsample_nearest.py
diff --git a/test/xpu/xpu_test_utils.py b/test/xpu/xpu_test_utils.py
@@ -339,7 +339,7 @@
         }
     },
     "histogram": {
-        ("TestCommonXPU", "test_out_histogram_xpu_float32"):{
+        ("TestCommonXPU", "test_out"):{
             torch.float32: tol(atol=3e-5, rtol=5e-5),
         }
     }