diff --git a/service_configuration_lib/spark_config.py b/service_configuration_lib/spark_config.py index 9aaccba..9d1389f 100644 --- a/service_configuration_lib/spark_config.py +++ b/service_configuration_lib/spark_config.py @@ -1109,6 +1109,10 @@ def get_spark_conf( else: raise UnsupportedClusterManagerException(cluster_manager) + # Enabling auto-decommission feature to move data around makes sense for only spot nodes + if 'batch' != paasta_pool and 'spark.decommission.enabled' in spark_conf: + spark_conf.update({'spark.decommission.enabled': 'false'}) + # configure dynamic resource allocation configs spark_conf = self.get_dra_configs(spark_conf)