From 440e5ff28b45fa2981226235384a3200e71bc8bc Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Tue, 22 Aug 2023 06:09:26 -0400 Subject: [PATCH 1/5] simplify typing --- neurodocker/cli/generate.py | 38 +++++------ neurodocker/cli/minify/_prune.py | 8 +-- neurodocker/cli/minify/trace.py | 20 +++--- neurodocker/reproenv/renderers.py | 64 +++++++++---------- neurodocker/reproenv/state.py | 15 +++-- neurodocker/reproenv/template.py | 40 ++++++------ ..._build_images_from_registered_templates.py | 6 +- neurodocker/reproenv/tests/utils.py | 22 +++---- neurodocker/reproenv/types.py | 17 ++--- 9 files changed, 115 insertions(+), 115 deletions(-) diff --git a/neurodocker/cli/generate.py b/neurodocker/cli/generate.py index 9c72542d..3e1cd6a9 100644 --- a/neurodocker/cli/generate.py +++ b/neurodocker/cli/generate.py @@ -5,10 +5,12 @@ # TODO: add a dedicated class for key=value in the eat-all class. +from __future__ import annotations + import json as json_lib import sys -import typing as ty from pathlib import Path +from typing import IO, Any, Optional, Type, cast import click @@ -48,15 +50,15 @@ def __init__(self, *args, **kwds): ) ] - def get_command(self, ctx: click.Context, name: str) -> ty.Optional[click.Command]: + def get_command(self, ctx: click.Context, name: str) -> Optional[click.Command]: command = self.commands.get(name) if command is None: return command # return immediately to error can be logged # This is only set if a subcommand is called. Calling --help on the group # does not set --template-path. - template_path: ty.Tuple[str] = ctx.params.get("template_path", tuple()) - yamls: ty.List[Path] = [] + template_path: tuple[str] = ctx.params.get("template_path", tuple()) + yamls: list[Path] = [] for p in template_path: path = Path(p) for pattern in ("*.yaml", "*.yml"): @@ -65,7 +67,7 @@ def get_command(self, ctx: click.Context, name: str) -> ty.Optional[click.Comman for path in yamls: _ = register_template(path) - params: ty.List[click.Parameter] = [ + params: list[click.Parameter] = [ click.Option( ["-p", "--pkg-manager"], type=click.Choice(list(allowed_pkg_managers), case_sensitive=False), @@ -86,11 +88,11 @@ class OrderedParamsCommand(click.Command): parameters. """ - def parse_args(self, ctx: click.Context, args: ty.List[str]): - self._options: ty.List[ty.Tuple[click.Parameter, ty.Any]] = [] + def parse_args(self, ctx: click.Context, args: list[str]): + self._options: list[tuple[click.Parameter, Any]] = [] # run the parser for ourselves to preserve the passed order parser = self.make_parser(ctx) - param_order: ty.List[click.Parameter] + param_order: list[click.Parameter] opts, _, param_order = parser.parse_args(args=list(args)) for param in param_order: # We need the parameter name... so if it's None, let's panic. @@ -186,8 +188,8 @@ def fn(v: str): return fn(value) -def _get_common_renderer_params() -> ty.List[click.Parameter]: - params: ty.List[click.Parameter] = [ +def _get_common_renderer_params() -> list[click.Parameter]: + params: list[click.Parameter] = [ click.Option( ["-p", "--pkg-manager"], type=click.Choice(list(allowed_pkg_managers), case_sensitive=False), @@ -290,9 +292,9 @@ def _create_help_for_template(template: Template) -> str: return h -def _get_params_for_registered_templates() -> ty.List[click.Parameter]: +def _get_params_for_registered_templates() -> list[click.Parameter]: """Return list of click parameters for registered templates.""" - params: ty.List[click.Parameter] = [] + params: list[click.Parameter] = [] names_tmpls = list(registered_templates_items()) names_tmpls.sort(key=lambda r: r[0]) # sort by name for name, tmpl in names_tmpls: @@ -308,7 +310,7 @@ def _params_to_renderer_dict(ctx: click.Context, pkg_manager) -> dict: """Return dictionary compatible with compatible with `_Renderer.from_dict()`.""" renderer_dict = {"pkg_manager": pkg_manager, "instructions": []} cmd = ctx.command - cmd = ty.cast(OrderedParamsCommand, cmd) + cmd = cast(OrderedParamsCommand, cmd) for param, value in cmd._options: d = _get_instruction_for_param(ctx=ctx, param=param, value=value) # TODO: what happens if `d is None`? @@ -319,9 +321,7 @@ def _params_to_renderer_dict(ctx: click.Context, pkg_manager) -> dict: return renderer_dict -def _get_instruction_for_param( - ctx: click.Context, param: click.Parameter, value: ty.Any -): +def _get_instruction_for_param(ctx: click.Context, param: click.Parameter, value: Any): # TODO: clean this up. d = None if param.name == "from_": @@ -415,7 +415,7 @@ def generate(*, template_path): def _base_generate( - ctx: click.Context, renderer: ty.Type[_Renderer], pkg_manager: str, **kwds + ctx: click.Context, renderer: Type[_Renderer], pkg_manager: str, **kwds ): """Function that does all of the work of `generate docker` and `generate singularity`. The difference between those two is the renderer used. @@ -475,14 +475,14 @@ def singularity(ctx: click.Context, pkg_manager: str, **kwds): type=click.File("r"), default=sys.stdin, ) -def genfromjson(*, container_type: str, input: ty.IO): +def genfromjson(*, container_type: str, input: IO): """Generate a container from a ReproEnv JSON file. INPUT is standard input by default or a path to a JSON file. """ d = json_lib.load(input) - renderer: ty.Type[_Renderer] + renderer: Type[_Renderer] if container_type.lower() == "docker": renderer = DockerRenderer elif container_type.lower() == "singularity": diff --git a/neurodocker/cli/minify/_prune.py b/neurodocker/cli/minify/_prune.py index 2419a4e7..533eebc7 100644 --- a/neurodocker/cli/minify/_prune.py +++ b/neurodocker/cli/minify/_prune.py @@ -1,6 +1,6 @@ """Remove all files under a directory but not caught by `reprozip trace`.""" +from __future__ import annotations -import typing as ty from pathlib import Path import yaml @@ -18,8 +18,8 @@ def _in_docker() -> bool: def main( *, - yaml_file: ty.Union[str, Path], - directories_to_prune: ty.Union[ty.List[str], ty.List[Path]], + yaml_file: str | Path, + directories_to_prune: list[str] | list[Path], ): if not _in_docker(): raise RuntimeError( @@ -50,7 +50,7 @@ def main( if not d.is_dir(): raise ValueError(f"Directory does not exist: {d}") - all_files: ty.Set[Path] = set() + all_files: set[Path] = set() for d in directories_to_prune: all_files.update(Path(d).rglob("*")) diff --git a/neurodocker/cli/minify/trace.py b/neurodocker/cli/minify/trace.py index 43cbd500..33234f2b 100644 --- a/neurodocker/cli/minify/trace.py +++ b/neurodocker/cli/minify/trace.py @@ -7,11 +7,13 @@ # TODO: consider implementing custom types for Docker container and paths within a # Docker container. +from __future__ import annotations + import io import logging import tarfile -import typing as ty from pathlib import Path +from typing import Generator, cast import click @@ -36,9 +38,9 @@ def copy_file_to_container( - container: ty.Union[str, docker.models.containers.Container], - src: ty.Union[str, Path], - dest: ty.Union[str, Path], + container: str | docker.models.containers.Container, + src: str | Path, + dest: str | Path, ) -> bool: """Copy `local_filepath` into `container`:`container_path`. @@ -98,10 +100,10 @@ def _get_mounts(container: docker.models.containers.Container) -> dict: @click.option("--yes", is_flag=True, help="Reply yes to all prompts.") @click.argument("command", nargs=-1, required=True) def minify( - container: ty.Union[str, docker.models.containers.Container], - directories_to_prune: ty.Tuple[str], + container: str | docker.models.containers.Container, + directories_to_prune: tuple[str], yes: bool, - command: ty.Tuple[str], + command: tuple[str], ) -> None: """Minify a container. @@ -118,7 +120,7 @@ def minify( "python -c 'a = 1 + 1; print(a)'" """ container = client.containers.get(container) - container = ty.cast(docker.models.containers.Container, container) + container = cast(docker.models.containers.Container, container) cmds = " ".join(f'"{c}"' for c in command) @@ -132,7 +134,7 @@ def minify( # iteration. exec_dict: dict = container.client.api.exec_create(container.id, cmd=trace_cmd) exec_id: str = exec_dict["Id"] - log_gen: ty.Generator[bytes, None, None] = container.client.api.exec_start( + log_gen: Generator[bytes, None, None] = container.client.api.exec_start( exec_id, stream=True ) for log in log_gen: diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index c2fb1f6b..1d1ca460 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -8,7 +8,7 @@ import os import pathlib import types -import typing as ty +from typing import Callable, Mapping, NoReturn, Optional import jinja2 @@ -34,7 +34,7 @@ # template. -def _raise_helper(msg: str) -> ty.NoReturn: +def _raise_helper(msg: str) -> NoReturn: raise RendererError(msg) @@ -43,7 +43,7 @@ def _raise_helper(msg: str) -> ty.NoReturn: # TODO: add a flag that avoids buggy behavior when basing a new container on # one created with ReproEnv. -PathType = ty.Union[str, pathlib.Path, os.PathLike] +PathType = str | pathlib.Path | os.PathLike def _render_string_from_template( @@ -86,7 +86,7 @@ def _render_string_from_template( return source -def _log_instruction(func: ty.Callable): +def _log_instruction(func: Callable): """Decorator that logs instructions passed to a Renderer. This adds the logs to the `_instructions` attribute of the Renderer instance. @@ -131,7 +131,7 @@ def with_logging(self, *args, **kwds): class _Renderer: def __init__( - self, pkg_manager: pkg_managers_type, users: ty.Optional[ty.Set[str]] = None + self, pkg_manager: pkg_managers_type, users: Optional[set[str]] = None ) -> None: if pkg_manager not in allowed_pkg_managers: raise RendererError( @@ -145,7 +145,7 @@ def __init__( # specification to JSON, because if we are not root, we can change to root, # write the file, and return to whichever user we were. self._current_user = "root" - self._instructions: ty.Mapping = { + self._instructions: Mapping = { "pkg_manager": self.pkg_manager, "existing_users": list(self._users), "instructions": [], @@ -178,11 +178,11 @@ def __str__(self) -> str: return f"{masthead}\n\n{image_spec}" @property - def users(self) -> ty.Set[str]: + def users(self) -> set[str]: return self._users @classmethod - def from_dict(cls, d: ty.Mapping) -> _Renderer: + def from_dict(cls, d: Mapping) -> _Renderer: """Instantiate a new renderer from a dictionary of instructions.""" # raise error if invalid _validate_renderer(d) @@ -274,7 +274,7 @@ def add_template( # Add environment (render any jinja templates). if template_method.env: - d: ty.Mapping[str, str] = { + d: Mapping[str, str] = { _render_string_from_template( k, template_method ): _render_string_from_template(v, template_method) @@ -285,7 +285,7 @@ def add_template( # Patch the `template_method.install_dependencies` instance method so it can be # used (ie rendered) in a template and have access to the pkg_manager requested. def install_patch( - inner_self: _BaseInstallationTemplate, pkgs: ty.List[str], opts: str = None + inner_self: _BaseInstallationTemplate, pkgs: list[str], opts: str = None ) -> str: return _install(pkgs=pkgs, pkg_manager=self.pkg_manager) @@ -369,21 +369,21 @@ def arg(self, key: str, value: str = None): def copy( self, - source: ty.Union[PathType, ty.List[PathType]], - destination: ty.Union[PathType, ty.List[PathType]], + source: PathType | list[PathType], + destination: PathType | list[PathType], ) -> _Renderer: raise NotImplementedError() def env(self, **kwds: str) -> _Renderer: raise NotImplementedError() - def entrypoint(self, args: ty.List[str]) -> _Renderer: + def entrypoint(self, args: list[str]) -> _Renderer: raise NotImplementedError() def from_(self, base_image: str) -> _Renderer: raise NotImplementedError() - def install(self, pkgs: ty.List[str], opts: str = None) -> _Renderer: + def install(self, pkgs: list[str], opts: str = None) -> _Renderer: raise NotImplementedError() def label(self, **kwds: str) -> _Renderer: @@ -436,11 +436,9 @@ def _get_instructions(self) -> str: class DockerRenderer(_Renderer): - def __init__( - self, pkg_manager: pkg_managers_type, users: ty.Set[str] = None - ) -> None: + def __init__(self, pkg_manager: pkg_managers_type, users: set[str] = None) -> None: super().__init__(pkg_manager=pkg_manager, users=users) - self._parts: ty.List[str] = [] + self._parts: list[str] = [] def render(self) -> str: """Return the rendered Dockerfile.""" @@ -467,7 +465,7 @@ def arg(self, key: str, value: str = None) -> DockerRenderer: @_log_instruction def copy( self, - source: ty.Union[PathType, ty.List[PathType]], + source: PathType | list[PathType], destination: PathType, from_: str = None, chown: str = None, @@ -494,7 +492,7 @@ def env(self, **kwds: str) -> DockerRenderer: return self @_log_instruction - def entrypoint(self, args: ty.List[str]) -> DockerRenderer: + def entrypoint(self, args: list[str]) -> DockerRenderer: s = 'ENTRYPOINT ["{}"]'.format('", "'.join(args)) self._parts.append(s) return self @@ -510,7 +508,7 @@ def from_(self, base_image: str, as_: str = None) -> DockerRenderer: return self @_log_instruction - def install(self, pkgs: ty.List[str], opts=None) -> DockerRenderer: + def install(self, pkgs: list[str], opts=None) -> DockerRenderer: """Install system packages.""" command = _install(pkgs, pkg_manager=self.pkg_manager, opts=opts) command = _indent_run_instruction(command) @@ -563,18 +561,18 @@ def workdir(self, path: PathType) -> DockerRenderer: class SingularityRenderer(_Renderer): def __init__( - self, pkg_manager: pkg_managers_type, users: ty.Optional[ty.Set[str]] = None + self, pkg_manager: pkg_managers_type, users: Optional[set[str]] = None ) -> None: super().__init__(pkg_manager=pkg_manager, users=users) self._header: _SingularityHeaderType = {} # The '%setup' section is intentionally omitted. - self._files: ty.List[str] = [] - self._environment: ty.List[ty.Tuple[str, str]] = [] - self._post: ty.List[str] = [] + self._files: list[str] = [] + self._environment: list[tuple[str, str]] = [] + self._post: list[str] = [] self._runscript = "" # TODO: is it OK to use a dict here? Labels could be overwritten. - self._labels: ty.Dict[str, str] = {} + self._labels: dict[str, str] = {} def render(self) -> str: s = "" @@ -633,7 +631,7 @@ def arg(self, key: str, value: str = None) -> SingularityRenderer: @_log_instruction def copy( self, - source: ty.Union[PathType, ty.List[PathType]], + source: PathType | list[PathType], destination: PathType, ) -> SingularityRenderer: if not isinstance(source, (list, tuple)): @@ -649,7 +647,7 @@ def env(self, **kwds: str) -> SingularityRenderer: return self @_log_instruction - def entrypoint(self, args: ty.List[str]) -> SingularityRenderer: + def entrypoint(self, args: list[str]) -> SingularityRenderer: self._runscript = " ".join(args) return self @@ -671,7 +669,7 @@ def from_(self, base_image: str) -> SingularityRenderer: return self @_log_instruction - def install(self, pkgs: ty.List[str], opts=None) -> SingularityRenderer: + def install(self, pkgs: list[str], opts=None) -> SingularityRenderer: """Install system packages.""" command = _install(pkgs, pkg_manager=self.pkg_manager, opts=opts) self.run(command) @@ -734,7 +732,7 @@ def _indent_run_instruction(string: str, indent=4) -> str: return "\n".join(out) -def _install(pkgs: ty.List[str], pkg_manager: str, opts: str = None) -> str: +def _install(pkgs: list[str], pkg_manager: str, opts: str = None) -> str: if pkg_manager == "apt": return _apt_install(pkgs, opts) elif pkg_manager == "yum": @@ -744,7 +742,7 @@ def _install(pkgs: ty.List[str], pkg_manager: str, opts: str = None) -> str: raise RendererError(f"Unknown package manager '{pkg_manager}'.") -def _apt_install(pkgs: ty.List[str], opts: str = None, sort=True) -> str: +def _apt_install(pkgs: list[str], opts: str = None, sort=True) -> str: """Return command to install deb packages with `apt-get` (Debian-based distros). `opts` are options passed to `yum install`. Default is "-q --no-install-recommends". @@ -762,7 +760,7 @@ def _apt_install(pkgs: ty.List[str], opts: str = None, sort=True) -> str: return s.strip() -def _apt_install_debs(urls: ty.List[str], opts: str = None, sort=True) -> str: +def _apt_install_debs(urls: list[str], opts: str = None, sort=True) -> str: """Return command to install deb packages with `apt-get` (Debian-based distros). `opts` are options passed to `yum install`. Default is "-q". @@ -786,7 +784,7 @@ def install_one(url: str): return s -def _yum_install(pkgs: ty.List[str], opts: str = None, sort=True) -> str: +def _yum_install(pkgs: list[str], opts: str = None, sort=True) -> str: """Return command to install packages with `yum` (CentOS, Fedora). `opts` are options passed to `yum install`. Default is "-q". diff --git a/neurodocker/reproenv/state.py b/neurodocker/reproenv/state.py index e092b89a..6601c068 100644 --- a/neurodocker/reproenv/state.py +++ b/neurodocker/reproenv/state.py @@ -1,10 +1,11 @@ """Stateful objects in reproenv runtime.""" +from __future__ import annotations import copy import json import os -import typing as ty from pathlib import Path +from typing import ItemsView, KeysView import jsonschema import yaml @@ -26,10 +27,10 @@ _schemas_path = Path(__file__).parent / "schemas" with (_schemas_path / "template.json").open("r") as f: - _TEMPLATE_SCHEMA: ty.Dict = json.load(f) + _TEMPLATE_SCHEMA: dict = json.load(f) with (_schemas_path / "renderer.json").open("r") as f: - _RENDERER_SCHEMA: ty.Dict = json.load(f) + _RENDERER_SCHEMA: dict = json.load(f) def _validate_template(template: TemplateType): @@ -59,7 +60,7 @@ def _validate_renderer(d): class _TemplateRegistry: """Object to hold templates in memory.""" - _templates: ty.Dict[str, TemplateType] = {} + _templates: dict[str, TemplateType] = {} @classmethod def _reset(cls): @@ -69,7 +70,7 @@ def _reset(cls): @classmethod def register( cls, - path_or_template: ty.Union[str, os.PathLike, TemplateType], + path_or_template: str | os.PathLike | TemplateType, name: str = None, ) -> TemplateType: """Register a template. This will overwrite an existing template with the @@ -150,12 +151,12 @@ def get(cls, name: str) -> TemplateType: ) @classmethod - def keys(cls) -> ty.KeysView[str]: + def keys(cls) -> KeysView[str]: """Return names of registered templates.""" return cls._templates.keys() @classmethod - def items(cls) -> ty.ItemsView[str, TemplateType]: + def items(cls) -> ItemsView[str, TemplateType]: return cls._templates.items() diff --git a/neurodocker/reproenv/template.py b/neurodocker/reproenv/template.py index 953720fa..1d365ef6 100644 --- a/neurodocker/reproenv/template.py +++ b/neurodocker/reproenv/template.py @@ -3,7 +3,7 @@ from __future__ import annotations import copy -import typing as ty +from typing import Mapping, Optional, cast from neurodocker.reproenv.exceptions import TemplateKeywordArgumentError from neurodocker.reproenv.state import _validate_template @@ -41,8 +41,8 @@ class Template: def __init__( self, template: TemplateType, - binaries_kwds: ty.Mapping[str, str] = None, - source_kwds: ty.Mapping[str, str] = None, + binaries_kwds: Mapping[str, str] = None, + source_kwds: Mapping[str, str] = None, ): # Validate against JSON schema. Registered templates were already validated at # registration time, but if we do not validate here, then in-memory templates @@ -50,9 +50,9 @@ def __init__( _validate_template(template) self._template = copy.deepcopy(template) - self._binaries: ty.Optional[_BinariesTemplate] = None + self._binaries: Optional[_BinariesTemplate] = None self._binaries_kwds = {} if binaries_kwds is None else binaries_kwds - self._source: ty.Optional[_SourceTemplate] = None + self._source: Optional[_SourceTemplate] = None self._source_kwds = {} if source_kwds is None else source_kwds if "binaries" in self._template: @@ -69,11 +69,11 @@ def name(self) -> str: return self._template["name"] @property - def binaries(self) -> ty.Union[None, _BinariesTemplate]: + def binaries(self) -> None | _BinariesTemplate: return self._binaries @property - def source(self) -> ty.Union[None, _SourceTemplate]: + def source(self) -> None | _SourceTemplate: return self._source @property @@ -104,7 +104,7 @@ class _BaseInstallationTemplate: def __init__( self, - template: ty.Union[_BinariesTemplateType, _SourceTemplateType], + template: _BinariesTemplateType | _SourceTemplateType, **kwds: str, ) -> None: self._template = copy.deepcopy(template) @@ -192,7 +192,7 @@ def template(self): return self._template @property - def env(self) -> ty.Mapping[str, str]: + def env(self) -> Mapping[str, str]: return self._template.get("env", {}) @property @@ -200,29 +200,29 @@ def instructions(self) -> str: return self._template.get("instructions", "") @property - def arguments(self) -> ty.Mapping: + def arguments(self) -> Mapping: return self._template.get("arguments", {}) @property - def required_arguments(self) -> ty.Set[str]: + def required_arguments(self) -> set[str]: args = self.arguments.get("required", None) return set(args) if args is not None else set() @property - def optional_arguments(self) -> ty.Dict[str, str]: + def optional_arguments(self) -> dict[str, str]: args = self.arguments.get("optional", None) return args if args is not None else {} @property - def versions(self) -> ty.Set[str]: + def versions(self) -> set[str]: raise NotImplementedError() - def dependencies(self, pkg_manager: str) -> ty.List[str]: + def dependencies(self, pkg_manager: str) -> list[str]: deps_dict = self._template.get("dependencies", {}) # TODO: not sure why the following line raises a type error in mypy. return deps_dict.get(pkg_manager, []) # type: ignore - def install(self, pkgs: ty.List[str], opts: str = None) -> str: + def install(self, pkgs: list[str], opts: str = None) -> str: raise NotImplementedError( "This method is meant to be patched by renderer objects, so it can be used" " in templates and have access to the pkg_manager being used." @@ -240,15 +240,15 @@ def __init__(self, template: _BinariesTemplateType, **kwds: str): super().__init__(template=template, **kwds) @property - def urls(self) -> ty.Mapping[str, str]: + def urls(self) -> Mapping[str, str]: # TODO: how can the code be changed so this cast is not necessary? - self._template = ty.cast(_BinariesTemplateType, self._template) + self._template = cast(_BinariesTemplateType, self._template) return self._template.get("urls", {}) @property - def versions(self) -> ty.Set[str]: + def versions(self) -> set[str]: # TODO: how can the code be changed so this cast is not necessary? - self._template = ty.cast(_BinariesTemplateType, self._template) + self._template = cast(_BinariesTemplateType, self._template) return set(self.urls.keys()) @@ -257,5 +257,5 @@ def __init__(self, template: _SourceTemplateType, **kwds: str): super().__init__(template=template, **kwds) @property - def versions(self) -> ty.Set[str]: + def versions(self) -> set[str]: return {"ANY"} diff --git a/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py b/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py index b857bca3..b616af46 100644 --- a/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py +++ b/neurodocker/reproenv/tests/test_build_images_from_registered_templates.py @@ -1,7 +1,7 @@ # TODO: add more tests for `from_dict` method. -import typing as ty from pathlib import Path +from typing import cast import pytest @@ -101,8 +101,8 @@ def test_build_using_renderer_instance_methods( _TemplateRegistry._reset() _TemplateRegistry.register(_template_filepath) - pkg_manager = ty.cast(pkg_managers_type, pkg_manager) - method = ty.cast(installation_methods_type, method) + pkg_manager = cast(pkg_managers_type, pkg_manager) + method = cast(installation_methods_type, method) fd_exe = "fdfind" if pkg_manager == "apt" else "fd" diff --git a/neurodocker/reproenv/tests/utils.py b/neurodocker/reproenv/tests/utils.py index b5997f48..d1111150 100644 --- a/neurodocker/reproenv/tests/utils.py +++ b/neurodocker/reproenv/tests/utils.py @@ -1,10 +1,12 @@ +from __future__ import annotations + import contextlib import getpass import os import subprocess -import typing as ty import uuid from pathlib import Path +from typing import Generator import pytest @@ -49,7 +51,7 @@ def _singularity_available(): @contextlib.contextmanager -def build_docker_image(context: Path, remove=False) -> ty.Generator[str, None, None]: +def build_docker_image(context: Path, remove=False) -> Generator[str, None, None]: """Context manager that builds a Docker image and removes it on exit. The argument `remove` is `False` by default because we clean up all images at the @@ -64,7 +66,7 @@ def build_docker_image(context: Path, remove=False) -> ty.Generator[str, None, N if not df.exists(): raise FileNotFoundError(f"Dockerfile not found: {df}") tag = "reproenv-pytest-" + uuid.uuid4().hex - cmd: ty.List[str] = ["docker", "build", "--tag", tag, str(context)] + cmd: list[str] = ["docker", "build", "--tag", tag, str(context)] try: _ = subprocess.check_output(cmd, cwd=context) yield tag @@ -80,9 +82,7 @@ def build_docker_image(context: Path, remove=False) -> ty.Generator[str, None, N @contextlib.contextmanager -def build_singularity_image( - context: Path, remove=True -) -> ty.Generator[str, None, None]: +def build_singularity_image(context: Path, remove=True) -> Generator[str, None, None]: """Context manager that builds a Apptainer image and removes it on exit. If `sudo singularity` is not available, the full path to `apptainer` can be set @@ -101,7 +101,7 @@ def build_singularity_image( user = getpass.getuser() cachedir = Path("/") / "dev" / "shm" / user / "apptainer" singularity = os.environ.get("REPROENV_APPTAINER_PROGRAM", "apptainer") - cmd: ty.List[str] = [ + cmd: list[str] = [ "sudo", f"APPTAINER_CACHEDIR={cachedir}", singularity, @@ -120,9 +120,7 @@ def build_singularity_image( pass -def run_docker_image( - img: str, args: ty.List[str] = None, entrypoint: ty.List[str] = None -): +def run_docker_image(img: str, args: list[str] = None, entrypoint: list[str] = None): """Wrapper for `docker run`. Returns @@ -144,7 +142,7 @@ def run_docker_image( def run_singularity_image( - img: str, args: ty.List[str] = None, entrypoint: ty.List[str] = None + img: str, args: list[str] = None, entrypoint: list[str] = None ): """Wrapper for `singularity run` or `singularity exec`. @@ -155,7 +153,7 @@ def run_singularity_image( """ scmd = "run" if entrypoint is None else "exec" # sudo not required - cmd: ty.List[str] = ["singularity", scmd, "--cleanenv", img] + cmd: list[str] = ["singularity", scmd, "--cleanenv", img] if entrypoint is not None: cmd.extend(entrypoint) if args is not None: diff --git a/neurodocker/reproenv/types.py b/neurodocker/reproenv/types.py index def4c76c..d74e5954 100644 --- a/neurodocker/reproenv/types.py +++ b/neurodocker/reproenv/types.py @@ -1,6 +1,7 @@ """Define types used in ReproEnv.""" +from __future__ import annotations -import typing as ty +from typing import Mapping from mypy_extensions import TypedDict from typing_extensions import Literal @@ -28,23 +29,23 @@ class _InstallationDependenciesType(TypedDict, total=False): `yum`, and Debian and Ubuntu use `apt` and `dpkg`. """ - apt: ty.List[str] - debs: ty.List[str] - yum: ty.List[str] + apt: list[str] + debs: list[str] + yum: list[str] class _TemplateArgumentsType(TypedDict): """Arguments (i.e., variables) that are used in the template.""" - required: ty.List[str] - optional: ty.Mapping[str, str] + required: list[str] + optional: Mapping[str, str] class _BaseTemplateType(TypedDict, total=False): """Keys common to both types of templates: binaries and source.""" arguments: _TemplateArgumentsType - env: ty.Mapping[str, str] + env: Mapping[str, str] dependencies: _InstallationDependenciesType instructions: str @@ -58,7 +59,7 @@ class _SourceTemplateType(_BaseTemplateType): class _BinariesTemplateType(_BaseTemplateType): """Template that defines how to install software from pre-compiled binaries.""" - urls: ty.Mapping[str, str] + urls: Mapping[str, str] class TemplateType(TypedDict, total=False): From d7033f0d1d68a629f8419262c7dd8590b0c7d283 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Tue, 22 Aug 2023 06:23:56 -0400 Subject: [PATCH 2/5] pacify mypy --- neurodocker/reproenv/renderers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index 1d1ca460..2a82c1a2 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -8,7 +8,7 @@ import os import pathlib import types -from typing import Callable, Mapping, NoReturn, Optional +from typing import Callable, Mapping, NoReturn, Optional, Union import jinja2 @@ -43,7 +43,7 @@ def _raise_helper(msg: str) -> NoReturn: # TODO: add a flag that avoids buggy behavior when basing a new container on # one created with ReproEnv. -PathType = str | pathlib.Path | os.PathLike +PathType = Union[str | pathlib.Path | os.PathLike] def _render_string_from_template( From bcf2114678aa5868aad929d40e7412523d02b34c Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Tue, 22 Aug 2023 06:28:01 -0400 Subject: [PATCH 3/5] do not fail fast --- .github/workflows/pull-request.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 8924c7e8..1976b0e0 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -10,6 +10,7 @@ jobs: strategy: matrix: python-version: ['3.11', '3.10', '3.9', '3.8'] + fail-fast: false steps: - name: Install Apptainer env: From 3cdcc8e1a46a7cfb7dacbc0b3c15dbb204413358 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Tue, 22 Aug 2023 06:31:21 -0400 Subject: [PATCH 4/5] fix typo --- neurodocker/reproenv/renderers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index 2a82c1a2..4217d75a 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -43,7 +43,7 @@ def _raise_helper(msg: str) -> NoReturn: # TODO: add a flag that avoids buggy behavior when basing a new container on # one created with ReproEnv. -PathType = Union[str | pathlib.Path | os.PathLike] +PathType = Union[str, pathlib.Path, os.PathLike] def _render_string_from_template( From 732bfe2a9a95bde889bfa86f45eb770b1a1c2a08 Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Tue, 22 Aug 2023 06:40:45 -0400 Subject: [PATCH 5/5] cancel previous runs --- .github/workflows/pull-request.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 1976b0e0..cb55ac56 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -1,5 +1,9 @@ name: CI +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + on: pull_request: branches: [ master ]