diff --git a/.github/workflows/no-torch.yml b/.github/workflows/no-torch.yml
index 794c02b3c4db..eb3ac9b03161 100644
--- a/.github/workflows/no-torch.yml
+++ b/.github/workflows/no-torch.yml
@@ -5,6 +5,7 @@ on:
   pull_request:
     paths:
       - '.github/workflows/no-torch.yml'
+      - 'op_builder/**'
   schedule:
     - cron: "0 0 * * *"
 
diff --git a/op_builder/mlu/fused_adam.py b/op_builder/mlu/fused_adam.py
index 0198db7cb276..2343e55568c4 100644
--- a/op_builder/mlu/fused_adam.py
+++ b/op_builder/mlu/fused_adam.py
@@ -5,7 +5,11 @@
 # DeepSpeed Team
 
 from .builder import MLUOpBuilder
-import torch
+
+try:
+    import torch
+except ImportError as e:
+    pass
 
 
 class MLUFusedAdam: