diff --git a/cluster_tools/cluster_tools/schedulers/slurm.py b/cluster_tools/cluster_tools/schedulers/slurm.py index 2990e9bdf..a151707c2 100644 --- a/cluster_tools/cluster_tools/schedulers/slurm.py +++ b/cluster_tools/cluster_tools/schedulers/slurm.py @@ -420,10 +420,10 @@ def investigate_failed_job( # resource limit. def parse_key_value_pairs( - pair_delimiter: str, key_value_delimiter: str + text: str, pair_delimiter: str, key_value_delimiter: str ) -> Dict[str, str]: properties = {} - for key_value_pair in stdout.split(pair_delimiter): + for key_value_pair in text.split(pair_delimiter): if key_value_delimiter not in key_value_pair: continue key, value = key_value_pair.split(key_value_delimiter, 1) @@ -436,11 +436,11 @@ def parse_key_value_pairs( if exit_code == 0: # Parse stdout into a key-value object - properties = parse_key_value_pairs(" ", "=") + properties = parse_key_value_pairs(stdout, " ", "=") - investigation = self._investigate_time_limit(properties) - if investigation: - return investigation + time_limit_investigation = self._investigate_time_limit(properties) + if time_limit_investigation: + return time_limit_investigation # Call `seff job_id` which should return some output including a line, # such as: "Memory Efficiency: 25019.18% of 1.00 GB" @@ -449,11 +449,11 @@ def parse_key_value_pairs( return None # Parse stdout into a key-value object - properties = parse_key_value_pairs("\n", ":") + properties = parse_key_value_pairs(stdout, "\n", ":") - investigation = self._investigate_memory_consumption(properties) - if investigation: - return investigation + memory_limit_investigation = self._investigate_memory_consumption(properties) + if memory_limit_investigation: + return memory_limit_investigation return self._investigate_exit_code(properties) diff --git a/cluster_tools/tests/test_slurm.py b/cluster_tools/tests/test_slurm.py index c90cea866..6e2185695 100644 --- a/cluster_tools/tests/test_slurm.py +++ b/cluster_tools/tests/test_slurm.py @@ -44,6 +44,12 @@ def expect_fork() -> bool: return True +def search_and_replace_in_slurm_config(search_string: str, replace_string: str) -> None: + chcall( + f"sed -ci 's/{search_string}/{replace_string}/g' /etc/slurm/slurm.conf && scontrol reconfigure" + ) + + def test_map_with_spawn() -> None: with cluster_tools.get_executor( "slurm", max_workers=5, start_method="spawn" @@ -291,9 +297,7 @@ def test_slurm_max_array_size() -> None: assert all(array_size <= max_array_size for array_size in occurences) finally: - chcall( - f"sed -ci 's/{command}//g' /etc/slurm/slurm.conf && scontrol reconfigure" - ) + search_and_replace_in_slurm_config(command, "") reset_max_array_size = executor.get_max_array_size() assert reset_max_array_size == original_max_array_size @@ -331,8 +335,8 @@ def test_slurm_memory_limit() -> None: try: # Increase the frequency at which slurm checks whether a job uses too much memory - chcall( - f"sed -ci 's/{original_gather_frequency_config}/{new_gather_frequency_config}/g' /etc/slurm/slurm.conf && scontrol reconfigure" + search_and_replace_in_slurm_config( + original_gather_frequency_config, new_gather_frequency_config ) with executor: @@ -350,8 +354,8 @@ def test_slurm_memory_limit() -> None: for fut in futures ) finally: - chcall( - f"sed -ci 's/{new_gather_frequency_config}/{original_gather_frequency_config}/g' /etc/slurm/slurm.conf && scontrol reconfigure" + search_and_replace_in_slurm_config( + new_gather_frequency_config, original_gather_frequency_config ) diff --git a/cluster_tools/typecheck.sh b/cluster_tools/typecheck.sh index 100b21797..4b3fa785f 100755 --- a/cluster_tools/typecheck.sh +++ b/cluster_tools/typecheck.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -eEuo pipefail -echo "typecheck.sh is not available for cluster_tools yet" - echo "Typecheck cluster_tools..." poetry run python -m mypy -p cluster_tools