Fixed logging in calculate_n_partitions

askap-vast · Jul 26, 2024 · 27e7e19 · 27e7e19
1 parent 7962e0a
commit 27e7e19
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/vast_pipeline/utils/utils.py b/vast_pipeline/utils/utils.py
@@ -397,7 +397,9 @@ def calculate_n_partitions(df, n_cpu, partition_size_mb=100):
     # n_partitions should be >= n_cpu for optimal parallel processing
     if n_partitions < n_cpu:
         n_partitions=n_cpu
+
+    partition_size_mb = int(np.ceil(mem_usage_mb/n_partitions))
 
-    logger.debug("Using {n_partitions} partions of {partition_size}MB")
+    logger.debug(f"Using {n_partitions} partions of {partition_size_mb}MB")
 
     return n_partitions