don't override spark.app.name if provided by the user

Yelp · Jul 22, 2024 · b659432 · b659432
1 parent 18a6a50
commit b659432
Showing 1 changed file with 2 additions and 0 deletions.
diff --git a/paasta_tools/tron_tools.py b/paasta_tools/tron_tools.py
@@ -373,6 +373,8 @@ def build_spark_config(self) -> Dict[str, str]:
         if "spark.app.name" in spark_conf:
             spark_conf["spark.app.name"] = limit_size_with_hash(
                 f"tron_spark_{self.get_service()}_{self.get_instance()}_{self.get_action_name()}"
+                if "spark.app.name" not in stringified_spark_args else
+                stringified_spark_args["spark.app.name"]
             )
         # TODO: Remove this once dynamic pod template is generated inside the driver using spark-submit wrapper
         if "spark.kubernetes.executor.podTemplateFile" in spark_conf: