bug fix

kubeflow · Jan 10, 2024 · b62ad9b · b62ad9b
1 parent 4488d02
commit b62ad9b
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 11 deletions.
diff --git a/sdk/python/kubeflow/training/api/training_client.py b/sdk/python/kubeflow/training/api/training_client.py
@@ -139,15 +139,19 @@ def train(
 
         if isinstance(resources_per_worker, dict):
             if "gpu" in resources_per_worker:
-                resources_per_worker["nvidia.com/gpu"] = resources_per_worker.pop("gpu")
-
-            if (
-                resources_per_worker["gpu"] is not None
-                and num_procs_per_worker > resources_per_worker["gpu"]
-            ) or (resources_per_worker["gpu"] is None and num_procs_per_worker != 0):
-                raise ValueError(
-                    "Insufficient gpu resources allocated to the container."
-                )
+                if (
+                    resources_per_worker["gpu"] is not None
+                    and (num_procs_per_worker > resources_per_worker["gpu"])
+                ) or (
+                    resources_per_worker["gpu"] is None and num_procs_per_worker != 0
+                ):
+                    raise ValueError(
+                        "Insufficient gpu resources allocated to the container."
+                    )
+                if resources_per_worker["gpu"] is not None:
+                    resources_per_worker["nvidia.com/gpu"] = resources_per_worker.pop(
+                        "gpu"
+                    )
 
             if (
                 "cpu" not in resources_per_worker
@@ -171,7 +175,8 @@ def train(
                 ),
             )
         except Exception as e:
-            raise RuntimeError("failed to create pvc")
+            pass  # local
+            # raise RuntimeError("failed to create pvc")
 
         if isinstance(model_provider_parameters, HuggingFaceModelParams):
             mp = "hf"
@@ -189,7 +194,7 @@ def train(
                 "--model_provider",
                 mp,
                 "--model_provider_parameters",
-                json.dumps(model_provider_parameters.__dict__),
+                json.dumps(model_provider_parameters.__dict__, cls=utils.SetEncoder),
                 "--dataset_provider",
                 dp,
                 "--dataset_provider_parameters",

diff --git a/sdk/python/kubeflow/training/utils/utils.py b/sdk/python/kubeflow/training/utils/utils.py
@@ -372,4 +372,6 @@ class SetEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, set):
             return list(obj)
+        if isinstance(obj, type):
+            return obj.__name__
         return json.JSONEncoder.default(self, obj)