fix linting setup and fix all linting errors

cluster_tools/cluster_tools/__init__.py

@@ -1,20 +1,10 @@
-import logging
-import multiprocessing
-import os
-import tempfile
-from concurrent import futures
-from concurrent.futures import ProcessPoolExecutor
-from functools import partial
-from pathlib import Path
-from shutil import rmtree
 from typing import Any, Union, overload
 
 from typing_extensions import Literal
 
-from cluster_tools._utils.warning import enrich_future_with_uncaught_warning
 from cluster_tools.executors.debug_sequential import DebugSequentialExecutor
 from cluster_tools.executors.multiprocessing_ import MultiprocessingExecutor
-from cluster_tools.executors.pickle import PickleExecutor
+from cluster_tools.executors.pickle_ import PickleExecutor
 from cluster_tools.executors.sequential import SequentialExecutor
 from cluster_tools.schedulers.cluster_executor import (
     ClusterExecutor,

cluster_tools/cluster_tools/_utils/call.py

@@ -13,6 +13,8 @@ def call(command: str, stdin: Optional[str] = None) -> Tuple[str, str, int]:
         stdin_flag = None
     p = subprocess.run(
         command,
+        stdin=stdin_flag,
+        check=False,
         shell=True,
         capture_output=True,
         text=True,

cluster_tools/cluster_tools/_utils/file_wait_thread.py

@@ -2,7 +2,7 @@
 import os
 import threading
 import time
-from typing import TYPE_CHECKING, Callable, Dict, Tuple
+from typing import TYPE_CHECKING, Callable, Dict
 
 if TYPE_CHECKING:
     from cluster_tools.schedulers.cluster_executor import ClusterExecutor

cluster_tools/cluster_tools/_utils/multiprocessing_logging_handler.py

@@ -9,7 +9,7 @@
 from logging.handlers import QueueHandler
 from queue import Empty as QueueEmpty
 from queue import Queue
-from typing import Any, Callable, List, Tuple
+from typing import Any, Callable, List
 
 # Inspired by https://stackoverflow.com/a/894284
 

cluster_tools/cluster_tools/_utils/reflection.py

@@ -1,6 +1,4 @@
 import os
-import pickle
-import sys
 from typing import Callable
 
 WARNING_TIMEOUT = 10 * 60  # seconds

cluster_tools/cluster_tools/_utils/string.py → cluster_tools/cluster_tools/_utils/string_.py

@@ -2,6 +2,8 @@
 import random
 import string
 
+# The module name includes a _-suffix to avoid name clashes with the standard library string module.
+
 
 def local_filename(filename: str = "") -> str:
     return os.path.join(os.getenv("CFUT_DIR", ".cfut"), filename)

cluster_tools/cluster_tools/_utils/tailf.py

@@ -33,7 +33,7 @@ def follow(self, seconds: int = 1) -> None:
         """
 
         self.check_file_validity(self.tailed_file)
-        with open(self.tailed_file, errors="replace") as file_:
+        with open(self.tailed_file, encoding="utf-8", errors="replace") as file_:
             # Don't seek, since we want to print the entire file here.
             while True:
                 line = file_.readline()

cluster_tools/cluster_tools/_utils/warning.py

@@ -1,6 +1,4 @@
 import logging
-import os
-import string
 import threading
 import time
 from concurrent.futures import Future

cluster_tools/cluster_tools/executors/multiprocessing_.py

@@ -1,7 +1,6 @@
 import logging
 import multiprocessing
 import os
-import sys
 import tempfile
 from concurrent import futures
 from concurrent.futures import Future, ProcessPoolExecutor
@@ -12,7 +11,6 @@
 from typing import (
     Any,
     Callable,
-    Dict,
     Iterable,
     Iterator,
     List,
@@ -22,14 +20,16 @@
     cast,
 )
 
-from typing_extensions import Literal, ParamSpec, TypedDict
+from typing_extensions import ParamSpec, TypedDict
 
 from cluster_tools._utils import pickling
 from cluster_tools._utils.multiprocessing_logging_handler import (
     _MultiprocessingLoggingHandlerPool,
 )
 from cluster_tools._utils.warning import enrich_future_with_uncaught_warning
 
+# The module name includes a _-suffix to avoid name clashes with the standard library multiprocessing module.
+
 
 class CFutDict(TypedDict):
     output_pickle_path: str

cluster_tools/cluster_tools/executors/pickle.py → cluster_tools/cluster_tools/executors/pickle_.py

@@ -4,6 +4,8 @@
 from cluster_tools._utils import pickling
 from cluster_tools.executors.multiprocessing_ import MultiprocessingExecutor
 
+# The module name includes a _-suffix to avoid name clashes with the standard library pickle module.
+
 _T = TypeVar("_T")
 
 

cluster_tools/cluster_tools/remote.py

@@ -6,7 +6,7 @@
 from typing import Any, Dict, Optional, Type
 
 from cluster_tools._utils import pickling
-from cluster_tools._utils.string import with_preliminary_postfix
+from cluster_tools._utils.string_ import with_preliminary_postfix
 from cluster_tools.schedulers.cluster_executor import ClusterExecutor
 from cluster_tools.schedulers.kube import KubernetesExecutor
 from cluster_tools.schedulers.pbs import PBSExecutor

cluster_tools/cluster_tools/schedulers/cluster_executor.py

@@ -31,7 +31,7 @@
     file_path_to_absolute_module,
     get_function_name,
 )
-from cluster_tools._utils.string import random_string, with_preliminary_postfix
+from cluster_tools._utils.string_ import random_string, with_preliminary_postfix
 from cluster_tools._utils.tailf import Tail
 from cluster_tools._utils.warning import enrich_future_with_uncaught_warning
 from cluster_tools.executors.multiprocessing_ import CFutDict

cluster_tools/cluster_tools/schedulers/pbs.py

@@ -9,7 +9,7 @@
 from typing_extensions import Literal
 
 from cluster_tools._utils.call import call, chcall
-from cluster_tools._utils.string import random_string
+from cluster_tools._utils.string_ import random_string
 from cluster_tools.schedulers.cluster_executor import ClusterExecutor
 
 # qstat vs. checkjob

cluster_tools/cluster_tools/schedulers/slurm.py

@@ -24,7 +24,7 @@
 from typing_extensions import Literal
 
 from cluster_tools._utils.call import call, chcall
-from cluster_tools._utils.string import random_string
+from cluster_tools._utils.string_ import random_string
 from cluster_tools.schedulers.cluster_executor import (
     NOT_YET_SUBMITTED_STATE,
     ClusterExecutor,

cluster_tools/lint.sh

@@ -1,4 +1,7 @@
 #!/usr/bin/env bash
 set -eEuo pipefail
 
-poetry run python -m pylint -j2 cluster_tools
+# Pylint doesn't lint files in directories that don't have an __init__.py
+# This is not fixed by the --recursive=y flag (https://github.com/PyCQA/pylint/issues/352)
+# Inspired by https://stackoverflow.com/questions/4210042/how-to-exclude-a-directory-in-find-command
+find cluster_tools -iname "*.py" | xargs poetry run python -m pylint -j2