Added pytorch build script migration rule file

clang/lib/DPCT/DPCT.cpp

@@ -1199,16 +1199,26 @@ int runDPCT(int argc, const char **argv) {
 
   if (MigrateBuildScriptOnly ||
       DpctGlobalInfo::getBuildScript() == BuildScriptKind::BS_Python) {
-    SmallString<128> PythonRuleFilePath(DpctInstallPath.getCanonicalPath());
-    llvm::sys::path::append(
-        PythonRuleFilePath,
-        Twine("extensions/python_rules/"
-              "python_build_script_migration_rule_ipex.yaml"));
-    if (llvm::sys::fs::exists(PythonRuleFilePath)) {
-      std::vector<clang::tooling::UnifiedPath> PythonRuleFiles{
-          PythonRuleFilePath};
-      importRules(PythonRuleFiles);
-      // generage helper functions file in the outroot dir here
+    // check if RuleFilePaths contains any user specified python migration rule
+    // file
+    bool pythonRuleFilePresent = std::any_of(
+        RuleFilePath.begin(), RuleFilePath.end(),
+        [](const clang::tooling::UnifiedPath &path) {
+          return path.getPath().contains("python_build_script_migration_rule");
+        });
+
+    if (!pythonRuleFilePresent) {
+      SmallString<128> PythonRuleFilePath(DpctInstallPath.getCanonicalPath());
+      llvm::sys::path::append(
+          PythonRuleFilePath,
+          Twine("extensions/python_rules/"
+                "python_build_script_migration_rule_pytorch.yaml"));
+      if (llvm::sys::fs::exists(PythonRuleFilePath)) {
+        std::vector<clang::tooling::UnifiedPath> PythonRuleFiles{
+            PythonRuleFilePath};
+        importRules(PythonRuleFiles);
+        // generage helper functions file in the outroot dir here
+      }
     }
   }
 

clang/test/dpct/python_migration/case_001/case_001_framework.cpp

@@ -4,23 +4,27 @@
 // RUN: cp %S/input.cmake ./input.cmake
 // RUN: cp %S/src/input.cu.txt ./input.cu
 // RUN: cp %S/src/compile_commands.json ./compile_commands.json
-// RUN: dpct -in-root ./ -out-root out --migrate-build-script-only
 
-// RUN: echo "begin" > %T/diff_1.txt
-// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff_1.txt
-// RUN: echo "end" >> %T/diff_1.txt
+// RUN: dpct -in-root ./ -out-root out --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected_pytorch.py %T/out/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
 // CHECK: begin
 // CHECK-NEXT: end
-
-// RUN: echo "begin" > %T/diff_2.txt
-// RUN: diff --strip-trailing-cr %S/expected.cmake %T/out/input.cmake >> %T/diff_2.txt
-// RUN: echo "end" >> %T/diff_2.txt
+// RUN: echo "begin" > %T/diff_cmake_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected.cmake %T/out/input.cmake >> %T/diff_cmake_pytorch.txt
+// RUN: echo "end" >> %T/diff_cmake_pytorch.txt
 // CHECK: begin
 // CHECK-NEXT: end
 
-// RUN: dpct -in-root ./ -out-root out --cuda-include-path="%cuda-path/include" --migrate-build-script=Python --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml -p ./
-// RUN: echo "begin" > %T/diff_3.txt
-// RUN: diff --strip-trailing-cr %S/src/expected.cpp.txt %T/out/input.dp.cpp >> %T/diff_3.txt
-// RUN: echo "end" >> %T/diff_3.txt
+// RUN: dpct -in-root ./ -out-root out_ipex --cuda-include-path="%cuda-path/include" --migrate-build-script=Python --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml -p ./
+// RUN: echo "begin" > %T/diff_ipex.txt
+// RUN: diff --strip-trailing-cr %S/expected_ipex.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
+// CHECK: begin
+// CHECK-NEXT: end
+// RUN: echo "begin" > %T/diff_cmake_ipex.txt
+// RUN: diff --strip-trailing-cr %S/src/expected.cpp.txt %T/out_ipex/input.dp.cpp >> %T/diff_cmake_ipex.txt
+// RUN: echo "end" >> %T/diff_cmake_ipex.txt
 // CHECK: begin
 // CHECK-NEXT: end

clang/test/dpct/python_migration/case_001/expected_pytorch.py

@@ -0,0 +1 @@
+import torch

clang/test/dpct/python_migration/case_002/case_002_import_torch_ext.cpp

@@ -1,10 +1,17 @@
 // RUN: rm -rf %T && mkdir -p %T
 // RUN: cd %T
 // RUN: cp %S/input.py ./input.py
-// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
-// RUN: echo "begin" > %T/diff.txt
-// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
-// RUN: echo "end" >> %T/diff.txt
 
+// RUN: dpct -in-root ./ -out-root out_pytorch ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected_pytorch.py %T/out_pytorch/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
+// CHECK: begin
+// CHECK-NEXT: end
+
+// RUN: dpct -in-root ./ -out-root out_ipex ./input.py --migrate-build-script-only --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml
+// RUN: echo "begin" > %T/diff_ipex.txt
+// RUN: diff --strip-trailing-cr %S/expected_ipex.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
 // CHECK: begin
 // CHECK-NEXT: end

clang/test/dpct/python_migration/case_002/expected_pytorch.py

@@ -0,0 +1,14 @@
+from setuptools import setup, Extension
+
+import torch
+from torch.utils import cpp_extension
+
+from torch.utils.cpp_extension import (
+    CppExtension,
+    CppExtension,
+    BuildExtension,
+    SYCL_HOME,
+)
+
+if SYCL_HOME:
+    var = cuda_specific_op()

clang/test/dpct/python_migration/case_003/case_003_torch_classes.cpp

@@ -1,10 +1,17 @@
 // RUN: rm -rf %T && mkdir -p %T
 // RUN: cd %T
 // RUN: cp %S/input.py ./input.py
-// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
-// RUN: echo "begin" > %T/diff.txt
-// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
-// RUN: echo "end" >> %T/diff.txt
 
+// RUN: dpct -in-root ./ -out-root out_pytorch ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected_pytorch.py %T/out_pytorch/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
+// CHECK: begin
+// CHECK-NEXT: end
+
+// RUN: dpct -in-root ./ -out-root out_ipex ./input.py --migrate-build-script-only --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml
+// RUN: echo "begin" > %T/diff_ipex.txt
+// RUN: diff --strip-trailing-cr %S/expected_ipex.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
 // CHECK: begin
 // CHECK-NEXT: end

clang/test/dpct/python_migration/case_003/expected_pytorch.py

@@ -0,0 +1,9 @@
+from torch.utils.cpp_extension import SYCL_HOME
+
+use_cuda = use_cuda and torch.xpu.is_available() and SYCL_HOME is not None
+
+from torch.utils.cpp_extension import (SYCL_HOME)
+
+extension = CppExtension if (use_cuda and SYCL_HOME) else CppExtension
+
+path += torch.utils.cpp_extension.SYCL_HOME

clang/test/dpct/python_migration/case_004/case_004_cuda_src_ext.cpp

@@ -1,10 +1,17 @@
 // RUN: rm -rf %T && mkdir -p %T
 // RUN: cd %T
 // RUN: cp %S/input.py ./input.py
-// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
-// RUN: echo "begin" > %T/diff.txt
-// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
-// RUN: echo "end" >> %T/diff.txt
 
+// RUN: dpct -in-root ./ -out-root out_pytorch ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/out_pytorch/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
+// CHECK: begin
+// CHECK-NEXT: end
+
+// RUN: dpct -in-root ./ -out-root out_ipex ./input.py --migrate-build-script-only --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml
+// RUN: echo "begin" > %T/diff_ipex.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
 // CHECK: begin
 // CHECK-NEXT: end

clang/test/dpct/python_migration/case_005/case_005_setup.cpp

@@ -1,10 +1,17 @@
 // RUN: rm -rf %T && mkdir -p %T
 // RUN: cd %T
 // RUN: cp %S/input.py ./input.py
-// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
-// RUN: echo "begin" > %T/diff.txt
-// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
-// RUN: echo "end" >> %T/diff.txt
 
+// RUN: dpct -in-root ./ -out-root out_pytorch ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected_pytorch.py %T/out_pytorch/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
+// CHECK: begin
+// CHECK-NEXT: end
+
+// RUN: dpct -in-root ./ -out-root out_ipex ./input.py --migrate-build-script-only --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected_ipex.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
 // CHECK: begin
 // CHECK-NEXT: end

clang/test/dpct/python_migration/case_005/expected_pytorch.py

@@ -0,0 +1,7 @@
+setup(
+    name='quant_cuda',
+    ext_modules=[cpp_extension.CppExtension(
+        'quant_cuda', ["quant_cuda.cpp", 'quant_cuda_kernel.dp.cpp']
+    , include_dirs=cpp_extension.include_paths('xpu'),)],
+    cmdclass={'build_ext': cpp_extension.BuildExtension}
+)

clang/test/dpct/python_migration/case_006/case_006_torch_cuda.cpp

@@ -1,10 +1,17 @@
 // RUN: rm -rf %T && mkdir -p %T
 // RUN: cd %T
 // RUN: cp %S/input.py ./input.py
-// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
-// RUN: echo "begin" > %T/diff.txt
-// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
-// RUN: echo "end" >> %T/diff.txt
 
+// RUN: dpct -in-root ./ -out-root out_pytorch ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/out_pytorch/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
+// CHECK: begin
+// CHECK-NEXT: end
+
+// RUN: dpct -in-root ./ -out-root out_ipex ./input.py --migrate-build-script-only --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
 // CHECK: begin
 // CHECK-NEXT: end

clang/test/dpct/python_migration/case_006/expected.py

@@ -11,4 +11,14 @@
 arch_list = ['']
 arch_list = ['']
 
+dev_name = torch.xpu.get_device_name()
+dev_name = xpu.get_device_name()
+d0_name = torch.xpu.get_device_name(devs[0])
+d0_name = xpu.get_device_name(devs[0])
+
+curr_st = torch.xpu.current_stream()
+curr_st = xpu.current_stream()
+d0_curr_st = torch.xpu.current_stream(devs[0])
+d0_curr_st = xpu.current_stream(devs[0])
+
 cuda_ver = torch.version.xpu

clang/test/dpct/python_migration/case_006/input.py

@@ -11,4 +11,14 @@
 arch_list = torch.cuda.get_arch_list()
 arch_list = cuda.get_arch_list()
 
+dev_name = torch.cuda.get_device_name()
+dev_name = cuda.get_device_name()
+d0_name = torch.cuda.get_device_name(devs[0])
+d0_name = cuda.get_device_name(devs[0])
+
+curr_st = torch.cuda.current_stream()
+curr_st = cuda.current_stream()
+d0_curr_st = torch.cuda.current_stream(devs[0])
+d0_curr_st = cuda.current_stream(devs[0])
+
 cuda_ver = torch.version.cuda

clang/test/dpct/python_migration/case_007/case_007_test_extension_name.cpp

@@ -1,12 +1,21 @@
 // RUN: rm -rf %T && mkdir -p %T
 // RUN: cd %T
-// RUN: mkdir -p out
+// RUN: mkdir -p out_pytorch
+// RUN: mkdir -p out_ipex
 // RUN: cp %S/input.py ./input.py
-// RUN: cp %S/MainSourceFiles.yaml ./out/MainSourceFiles.yaml
-// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
-// RUN: echo "begin" > %T/diff.txt
-// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
-// RUN: echo "end" >> %T/diff.txt
+// RUN: cp %S/MainSourceFiles.yaml ./out_pytorch/MainSourceFiles.yaml
+// RUN: cp %S/MainSourceFiles.yaml ./out_ipex/MainSourceFiles.yaml
 
+// RUN: dpct -in-root ./ -out-root out_pytorch ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/out_pytorch/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
+// CHECK: begin
+// CHECK-NEXT: end
+
+// RUN: dpct -in-root ./ -out-root out_ipex ./input.py --migrate-build-script-only --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
 // CHECK: begin
 // CHECK-NEXT: end

clang/test/dpct/python_migration/case_008/case_008_comments.cpp

@@ -1,10 +1,17 @@
 // RUN: rm -rf %T && mkdir -p %T
 // RUN: cd %T
 // RUN: cp %S/input.py ./input.py
-// RUN: dpct -in-root ./ -out-root out ./input.py --migrate-build-script-only
-// RUN: echo "begin" > %T/diff.txt
-// RUN: diff --strip-trailing-cr %S/expected.py %T/out/input.py >> %T/diff.txt
-// RUN: echo "end" >> %T/diff.txt
 
+// RUN: dpct -in-root ./ -out-root out_pytorch ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected_pytorch.py %T/out_pytorch/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
+// CHECK: begin
+// CHECK-NEXT: end
+
+// RUN: dpct -in-root ./ -out-root out_ipex ./input.py --migrate-build-script-only --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected_ipex.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
 // CHECK: begin
 // CHECK-NEXT: end

clang/test/dpct/python_migration/case_008/expected_pytorch.py

@@ -0,0 +1,2 @@
+# import torch
+import torch

clang/test/dpct/python_migration/case_009/case_009_cpp_extension.cpp

@@ -0,0 +1,17 @@
+// RUN: rm -rf %T && mkdir -p %T
+// RUN: cd %T
+// RUN: cp %S/input.py ./input.py
+
+// RUN: dpct -in-root ./ -out-root out_pytorch ./input.py --migrate-build-script-only
+// RUN: echo "begin" > %T/diff_pytorch.txt
+// RUN: diff --strip-trailing-cr %S/expected_pytorch.py %T/out_pytorch/input.py >> %T/diff_pytorch.txt
+// RUN: echo "end" >> %T/diff_pytorch.txt
+// CHECK: begin
+// CHECK-NEXT: end
+
+// RUN: dpct -in-root ./ -out-root out_ipex ./input.py --migrate-build-script-only --rule-file=%T/../../../../../../../extensions/python_rules/python_build_script_migration_rule_ipex.yaml
+// RUN: echo "begin" > %T/diff.txt
+// RUN: diff --strip-trailing-cr %S/expected_ipex.py %T/out_ipex/input.py >> %T/diff_ipex.txt
+// RUN: echo "end" >> %T/diff_ipex.txt
+// CHECK: begin
+// CHECK-NEXT: end

clang/test/dpct/python_migration/case_009/expected_ipex.py

@@ -0,0 +1,5 @@
+import torch
+import intel_extension_for_pytorch
+
+intel_extension_for_pytorch.xpu.cpp_extension.load(name=module_name, build_directory=cached_build_dir,
+                verbose=verbose_build, sources=cached_sources, **build_kwargs, extra_cflags=['-fsycl'], extra_ldflags=['-fsycl'])

clang/test/dpct/python_migration/case_009/expected_pytorch.py

@@ -0,0 +1,4 @@
+import torch
+
+torch.utils.cpp_extension.load(name=module_name, build_directory=cached_build_dir,
+                verbose=verbose_build, sources=cached_sources, **build_kwargs, extra_cflags=['-fsycl'], extra_ldflags=['-fsycl'])

clang/test/dpct/python_migration/case_009/input.py

@@ -0,0 +1,4 @@
+import torch
+
+torch.utils.cpp_extension.load(name=module_name, build_directory=cached_build_dir,
+                verbose=verbose_build, sources=cached_sources, **build_kwargs)

clang/tools/dpct/DpctOptRules/CMakeLists.txt

@@ -11,6 +11,7 @@ set(dpct_cmake_rule_files
 
 set(dpct_python_rule_files
   ${CMAKE_SOURCE_DIR}/../clang/tools/dpct/DpctOptRules/python_build_script_migration_rule_ipex.yaml
+  ${CMAKE_SOURCE_DIR}/../clang/tools/dpct/DpctOptRules/python_build_script_migration_rule_pytorch.yaml
 )
 
 set(dpct_pytorch_api_rule_files