diff --git a/cluster_tools/cluster_tools/__init__.py b/cluster_tools/cluster_tools/__init__.py index 28bc3ce67..e698c9276 100644 --- a/cluster_tools/cluster_tools/__init__.py +++ b/cluster_tools/cluster_tools/__init__.py @@ -1,20 +1,10 @@ -import logging -import multiprocessing -import os -import tempfile -from concurrent import futures -from concurrent.futures import ProcessPoolExecutor -from functools import partial -from pathlib import Path -from shutil import rmtree from typing import Any, Union, overload from typing_extensions import Literal -from cluster_tools._utils.warning import enrich_future_with_uncaught_warning from cluster_tools.executors.debug_sequential import DebugSequentialExecutor from cluster_tools.executors.multiprocessing_ import MultiprocessingExecutor -from cluster_tools.executors.pickle import PickleExecutor +from cluster_tools.executors.pickle_ import PickleExecutor from cluster_tools.executors.sequential import SequentialExecutor from cluster_tools.schedulers.cluster_executor import ( ClusterExecutor, diff --git a/cluster_tools/cluster_tools/_utils/call.py b/cluster_tools/cluster_tools/_utils/call.py index f726b699a..28929e13a 100644 --- a/cluster_tools/cluster_tools/_utils/call.py +++ b/cluster_tools/cluster_tools/_utils/call.py @@ -13,6 +13,8 @@ def call(command: str, stdin: Optional[str] = None) -> Tuple[str, str, int]: stdin_flag = None p = subprocess.run( command, + stdin=stdin_flag, + check=False, shell=True, capture_output=True, text=True, diff --git a/cluster_tools/cluster_tools/_utils/file_wait_thread.py b/cluster_tools/cluster_tools/_utils/file_wait_thread.py index 2bdb8046f..80f2ddfd2 100644 --- a/cluster_tools/cluster_tools/_utils/file_wait_thread.py +++ b/cluster_tools/cluster_tools/_utils/file_wait_thread.py @@ -2,7 +2,7 @@ import os import threading import time -from typing import TYPE_CHECKING, Callable, Dict, Tuple +from typing import TYPE_CHECKING, Callable, Dict if TYPE_CHECKING: from cluster_tools.schedulers.cluster_executor import ClusterExecutor diff --git a/cluster_tools/cluster_tools/_utils/multiprocessing_logging_handler.py b/cluster_tools/cluster_tools/_utils/multiprocessing_logging_handler.py index 0c4975082..96ef62119 100644 --- a/cluster_tools/cluster_tools/_utils/multiprocessing_logging_handler.py +++ b/cluster_tools/cluster_tools/_utils/multiprocessing_logging_handler.py @@ -9,7 +9,7 @@ from logging.handlers import QueueHandler from queue import Empty as QueueEmpty from queue import Queue -from typing import Any, Callable, List, Tuple +from typing import Any, Callable, List # Inspired by https://stackoverflow.com/a/894284 diff --git a/cluster_tools/cluster_tools/_utils/reflection.py b/cluster_tools/cluster_tools/_utils/reflection.py index ed0ea89b1..3853d172b 100644 --- a/cluster_tools/cluster_tools/_utils/reflection.py +++ b/cluster_tools/cluster_tools/_utils/reflection.py @@ -1,6 +1,4 @@ import os -import pickle -import sys from typing import Callable WARNING_TIMEOUT = 10 * 60 # seconds diff --git a/cluster_tools/cluster_tools/_utils/string.py b/cluster_tools/cluster_tools/_utils/string_.py similarity index 84% rename from cluster_tools/cluster_tools/_utils/string.py rename to cluster_tools/cluster_tools/_utils/string_.py index 1c5f573c6..225584c69 100644 --- a/cluster_tools/cluster_tools/_utils/string.py +++ b/cluster_tools/cluster_tools/_utils/string_.py @@ -2,6 +2,8 @@ import random import string +# The module name includes a _-suffix to avoid name clashes with the standard library string module. + def local_filename(filename: str = "") -> str: return os.path.join(os.getenv("CFUT_DIR", ".cfut"), filename) diff --git a/cluster_tools/cluster_tools/_utils/tailf.py b/cluster_tools/cluster_tools/_utils/tailf.py index 4689655ef..eafe8db56 100644 --- a/cluster_tools/cluster_tools/_utils/tailf.py +++ b/cluster_tools/cluster_tools/_utils/tailf.py @@ -33,7 +33,7 @@ def follow(self, seconds: int = 1) -> None: """ self.check_file_validity(self.tailed_file) - with open(self.tailed_file, errors="replace") as file_: + with open(self.tailed_file, encoding="utf-8", errors="replace") as file_: # Don't seek, since we want to print the entire file here. while True: line = file_.readline() diff --git a/cluster_tools/cluster_tools/_utils/warning.py b/cluster_tools/cluster_tools/_utils/warning.py index d708ae09d..be103a46b 100644 --- a/cluster_tools/cluster_tools/_utils/warning.py +++ b/cluster_tools/cluster_tools/_utils/warning.py @@ -1,6 +1,4 @@ import logging -import os -import string import threading import time from concurrent.futures import Future diff --git a/cluster_tools/cluster_tools/executors/multiprocessing_.py b/cluster_tools/cluster_tools/executors/multiprocessing_.py index 86ad9572c..4532c186a 100644 --- a/cluster_tools/cluster_tools/executors/multiprocessing_.py +++ b/cluster_tools/cluster_tools/executors/multiprocessing_.py @@ -1,7 +1,6 @@ import logging import multiprocessing import os -import sys import tempfile from concurrent import futures from concurrent.futures import Future, ProcessPoolExecutor @@ -12,7 +11,6 @@ from typing import ( Any, Callable, - Dict, Iterable, Iterator, List, @@ -22,7 +20,7 @@ cast, ) -from typing_extensions import Literal, ParamSpec, TypedDict +from typing_extensions import ParamSpec, TypedDict from cluster_tools._utils import pickling from cluster_tools._utils.multiprocessing_logging_handler import ( @@ -30,6 +28,8 @@ ) from cluster_tools._utils.warning import enrich_future_with_uncaught_warning +# The module name includes a _-suffix to avoid name clashes with the standard library multiprocessing module. + class CFutDict(TypedDict): output_pickle_path: str diff --git a/cluster_tools/cluster_tools/executors/pickle.py b/cluster_tools/cluster_tools/executors/pickle_.py similarity index 91% rename from cluster_tools/cluster_tools/executors/pickle.py rename to cluster_tools/cluster_tools/executors/pickle_.py index 405def3c6..9e837255a 100644 --- a/cluster_tools/cluster_tools/executors/pickle.py +++ b/cluster_tools/cluster_tools/executors/pickle_.py @@ -4,6 +4,8 @@ from cluster_tools._utils import pickling from cluster_tools.executors.multiprocessing_ import MultiprocessingExecutor +# The module name includes a _-suffix to avoid name clashes with the standard library pickle module. + _T = TypeVar("_T") diff --git a/cluster_tools/cluster_tools/remote.py b/cluster_tools/cluster_tools/remote.py index 6f3547d02..889a94084 100644 --- a/cluster_tools/cluster_tools/remote.py +++ b/cluster_tools/cluster_tools/remote.py @@ -6,7 +6,7 @@ from typing import Any, Dict, Optional, Type from cluster_tools._utils import pickling -from cluster_tools._utils.string import with_preliminary_postfix +from cluster_tools._utils.string_ import with_preliminary_postfix from cluster_tools.schedulers.cluster_executor import ClusterExecutor from cluster_tools.schedulers.kube import KubernetesExecutor from cluster_tools.schedulers.pbs import PBSExecutor diff --git a/cluster_tools/cluster_tools/schedulers/cluster_executor.py b/cluster_tools/cluster_tools/schedulers/cluster_executor.py index d30a14721..e5945f142 100644 --- a/cluster_tools/cluster_tools/schedulers/cluster_executor.py +++ b/cluster_tools/cluster_tools/schedulers/cluster_executor.py @@ -31,7 +31,7 @@ file_path_to_absolute_module, get_function_name, ) -from cluster_tools._utils.string import random_string, with_preliminary_postfix +from cluster_tools._utils.string_ import random_string, with_preliminary_postfix from cluster_tools._utils.tailf import Tail from cluster_tools._utils.warning import enrich_future_with_uncaught_warning from cluster_tools.executors.multiprocessing_ import CFutDict diff --git a/cluster_tools/cluster_tools/schedulers/pbs.py b/cluster_tools/cluster_tools/schedulers/pbs.py index 6537aee11..4c153ab86 100644 --- a/cluster_tools/cluster_tools/schedulers/pbs.py +++ b/cluster_tools/cluster_tools/schedulers/pbs.py @@ -9,7 +9,7 @@ from typing_extensions import Literal from cluster_tools._utils.call import call, chcall -from cluster_tools._utils.string import random_string +from cluster_tools._utils.string_ import random_string from cluster_tools.schedulers.cluster_executor import ClusterExecutor # qstat vs. checkjob diff --git a/cluster_tools/cluster_tools/schedulers/slurm.py b/cluster_tools/cluster_tools/schedulers/slurm.py index a151707c2..8f02abe33 100644 --- a/cluster_tools/cluster_tools/schedulers/slurm.py +++ b/cluster_tools/cluster_tools/schedulers/slurm.py @@ -24,7 +24,7 @@ from typing_extensions import Literal from cluster_tools._utils.call import call, chcall -from cluster_tools._utils.string import random_string +from cluster_tools._utils.string_ import random_string from cluster_tools.schedulers.cluster_executor import ( NOT_YET_SUBMITTED_STATE, ClusterExecutor, diff --git a/cluster_tools/lint.sh b/cluster_tools/lint.sh index b0094453e..01f601738 100755 --- a/cluster_tools/lint.sh +++ b/cluster_tools/lint.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash set -eEuo pipefail -poetry run python -m pylint -j2 cluster_tools \ No newline at end of file +# Pylint doesn't lint files in directories that don't have an __init__.py +# This is not fixed by the --recursive=y flag (https://github.com/PyCQA/pylint/issues/352) +# Inspired by https://stackoverflow.com/questions/4210042/how-to-exclude-a-directory-in-find-command +find cluster_tools -iname "*.py" | xargs poetry run python -m pylint -j2