diff --git a/CHANGELOG.md b/CHANGELOG.md index 16a63d6..2e4a349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,16 @@ and this project adheres to [Semantic Versioning][]. do not contain a separate `.gitignore` anymore. This means empty folders won't be tracked by git, but this solves issues with dvc refusing to track the output folder because it is already partly tracked by git. +### New Features + +- Python API that mirrors `dso-r` functionality (e.g. to be used from Jupyter notebooks) ([#30](https://github.com/Boehringer-Ingelheim/dso/pull/30)) + +### Chore + +- Refactor CLI into separate module ([#30](https://github.com/Boehringer-Ingelheim/dso/pull/30)) +- Defer imports in CLI until they are actually needed to speed up CLI ([#30](https://github.com/Boehringer-Ingelheim/dso/pull/30)) +- Make all modules explicitly private that are not part of the public API ([#30](https://github.com/Boehringer-Ingelheim/dso/pull/30)) + ## v0.10.1 ### Fixes diff --git a/pyproject.toml b/pyproject.toml index 359a685..6c6f230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ optional-dependencies.test = [ urls.Documentation = "https://github.com/Boehringer-Ingelheim/dso" urls.Home-page = "https://github.com/Boehringer-Ingelheim/dso" urls.Source = "https://github.com/Boehringer-Ingelheim/dso" -scripts.dso = "dso:cli" +scripts.dso = "dso.cli:cli" [tool.hatch.version] source = "vcs" diff --git a/src/dso/__init__.py b/src/dso/__init__.py index 66feadb..5160e86 100644 --- a/src/dso/__init__.py +++ b/src/dso/__init__.py @@ -1,62 +1,4 @@ -import logging -import os +from ._metadata import __version__ # noqa +from .api import here, read_params, set_stage, stage_here -import rich_click as click - -from ._logging import log -from ._metadata import __version__ -from .compile_config import cli as compile_config_cli -from .create import cli as create_cli -from .exec import cli as exec_cli -from .get_config import cli as get_config_cli -from .init import cli as init_cli -from .lint import cli as lint_cli -from .repro import cli as repro_cli -from .watermark import cli as watermark_cli - -click.rich_click.USE_MARKDOWN = True - - -@click.group() -@click.option( - "-q", - "--quiet", - count=True, - help=( - "Reduce verbosity. `-q` disables info messages, `-qq` disables warnings. Errors messages cannot be disabled. " - "The same can be achieved by setting the env var `DSO_QUIET=1` or `DSO_QUIET=2`, respectively." - ), - default=int(os.environ.get("DSO_QUIET", 0)), -) -@click.option( - "-v", - "--verbose", - help=( - "Increase logging verbosity to include debug messages. " - "The same can be achieved by setting the env var `DSO_VERBOSE=1`." - ), - default=bool(int(os.environ.get("DSO_VERBOSE", 0))), - is_flag=True, -) -@click.version_option(version=__version__, prog_name="dso") -def cli(quiet: int, verbose: bool): - """Root command""" - if quiet >= 2: - log.setLevel(logging.ERROR) - os.environ["DSO_QUIET"] = "2" - elif quiet == 1: - log.setLevel(logging.WARNING) - os.environ["DSO_QUIET"] = "1" - elif verbose: - log.setLevel(logging.DEBUG) - os.environ["DSO_VERBOSE"] = "1" - - -cli.add_command(create_cli) -cli.add_command(init_cli) -cli.add_command(compile_config_cli) -cli.add_command(repro_cli) -cli.add_command(exec_cli) -cli.add_command(lint_cli) -cli.add_command(get_config_cli) -cli.add_command(watermark_cli) +__all__ = ["read_params", "here", "stage_here", "set_stage"] diff --git a/src/dso/compile_config.py b/src/dso/_compile_config.py similarity index 90% rename from src/dso/compile_config.py rename to src/dso/_compile_config.py index d38605f..1f1e052 100644 --- a/src/dso/compile_config.py +++ b/src/dso/_compile_config.py @@ -9,11 +9,10 @@ from textwrap import dedent import hiyapyco -import rich_click as click from ruamel.yaml import YAML, yaml_object from ._logging import log -from ._util import _find_in_parent, check_project_roots, get_project_root +from ._util import check_project_roots, find_in_parent, get_project_root PARAMS_YAML_DISCLAIMER = dedent( """\ @@ -117,7 +116,7 @@ def _get_list_of_configs_to_compile(paths: Sequence[Path], project_root: Path): # Check each parent directory if it contains a "params.in.yaml" - If yes, add it to the list of all configs. # We don't need to re-check the parents of added items, because their parent is per definition also a parent # of a config that was already part of the list. - while (tmp_path := _find_in_parent(tmp_path.parent, "params.in.yaml", project_root)) is not None: + while (tmp_path := find_in_parent(tmp_path.parent, "params.in.yaml", project_root)) is not None: all_configs.add(tmp_path) # we don't want to find the current config again, therefore .parent tmp_path = tmp_path.parent @@ -198,19 +197,3 @@ def compile_all_configs(paths: Sequence[Path]): log.debug(f"./{config.relative_to(project_root)} [green]is already up-to-date!") log.info("[green]Configuration compiled successfully.") - - -@click.command(name="compile-config") -@click.argument("args", nargs=-1) -def cli(args): - """Compile params.in.yaml into params.yaml using Jinja2 templating and resolving recursive templates. - - If passing no arguments, configs will be resolved for the current working directory (i.e. all parent configs, - and all configs in child directories). Alternatively a list of paths can be specified. In that case, all configs - related to these paths will be compiled (useful for using with pre-commit). - """ - if not len(args): - paths = [Path.cwd()] - else: - paths = [Path(x) for x in args] - compile_all_configs(paths) diff --git a/src/dso/get_config.py b/src/dso/_get_config.py similarity index 75% rename from src/dso/get_config.py rename to src/dso/_get_config.py index 2cfa374..3cf269b 100644 --- a/src/dso/get_config.py +++ b/src/dso/_get_config.py @@ -1,16 +1,15 @@ -import os +"""Get configuration for a stage based on params.in.yaml and dvc.yaml""" + import re import sys from collections.abc import Collection from itertools import groupby from pathlib import Path -import rich_click as click from ruamel.yaml import YAML from dso._logging import log from dso._util import get_project_root -from dso.compile_config import compile_all_configs def _filter_nested_dict(data: dict, keys: Collection[str]) -> dict: @@ -45,8 +44,11 @@ def get_config(stage: str, *, all: bool = False, skip_compile: bool = False) -> all If true, the config is not filtered based on the `dvc.yaml` file. skip_compile - If true, do not compile the config before loading it + If `True`, do not compile the config before loading it. + If `False`, always compile. """ + from dso._compile_config import compile_all_configs + proj_root = get_project_root(Path.cwd()) log.info(f"Retrieving config for stage ./{stage}") if ":" in stage: @@ -117,41 +119,3 @@ def get_config(stage: str, *, all: bool = False, skip_compile: bool = False) -> keep_params = {p for p in keep_params if not (p.startswith("item.") or p == "item")} return _filter_nested_dict(config, keep_params) - - -@click.command(name="get-config") -@click.option( - "--all", - is_flag=True, - type=bool, - default=False, - help="Include all parameters, not only those mentioned in `dvc.yaml`", -) -@click.option( - "--skip-compile", - is_flag=True, - type=bool, - default=bool(int(os.environ.get("DSO_SKIP_COMPILE", 0))), - help="Do not compile configs before loading it. The same can be achieved by setting the `DSO_SKIP_COMPILE=1` env var.", -) -@click.argument( - "stage", -) -def cli(stage, all, skip_compile): - """Get the configuration for a given stage and print it to STDOUT in yaml format. - - The path to the stage must be relative to the root dir of the project. - - By default, the configuration is filtered to include only the keys that are mentioned in `dvc.yaml` to force - declaring all dependencies. - - If multiple stages are defined in a single `dvc.yaml`, the stage name MUST be specified using - `path/to/stage:stage_name` unless `--all` is given. - """ - try: - out_config = get_config(stage, all=all, skip_compile=skip_compile) - yaml = YAML() - yaml.dump(out_config, sys.stdout) - except KeyError as e: - log.error(f"dvc.yaml defines parameter {e} that is not in params.yaml") - sys.exit(1) diff --git a/src/dso/lint.py b/src/dso/_lint.py similarity index 85% rename from src/dso/lint.py rename to src/dso/_lint.py index 3dad81c..399eda4 100644 --- a/src/dso/lint.py +++ b/src/dso/_lint.py @@ -1,4 +1,5 @@ -import os +"""Linting functions for DSO projects""" + import re import sys from abc import ABC, abstractmethod @@ -7,12 +8,10 @@ from os import chdir from pathlib import Path -import rich_click as click from ruamel.yaml import YAML from dso._logging import log -from dso._util import _find_in_parent, _git_list_files, check_project_roots, get_project_root -from dso.compile_config import compile_all_configs +from dso._util import check_project_roots, find_in_parent, get_project_root, git_list_files class LintError(Exception): @@ -67,7 +66,7 @@ def is_applicable(cls: type["QuartoRule"], file: Path) -> bool: Return true, if "dso exec quarto" is found in the dvc.yaml associated with this stage AND the file matches the pattern """ - dvc_yaml = _find_in_parent(file, "dvc.yaml", get_project_root(file)) + dvc_yaml = find_in_parent(file, "dvc.yaml", get_project_root(file)) assert dvc_yaml is not None, "No dvc.yaml found in project" is_quarto_stage = "dso exec quarto ." in dvc_yaml.read_text() return is_quarto_stage and Rule._match_filename_pattern(cls.PATTERN, file) @@ -82,7 +81,7 @@ class DSO001(QuartoRule): def check(cls, file): """Check that the file passes the linting step.""" root_path = get_project_root(file) - stage_path_expected = _find_in_parent(file, "dvc.yaml", root_path) + stage_path_expected = find_in_parent(file, "dvc.yaml", root_path) assert stage_path_expected is not None, "No dvc.yaml found in project" # .parent to remove the dvc.yaml filename stage_path_expected = str(stage_path_expected.parent.relative_to(root_path)) @@ -204,7 +203,7 @@ def lint(self, file: Path): if not file.is_file(): raise ValueError("Only existing files (not directories) may be passed to linter") - config_path = _find_in_parent(file, "params.yaml", get_project_root(file)) + config_path = find_in_parent(file, "params.yaml", get_project_root(file)) assert config_path is not None, "No params.yaml found in project" config = DSOLinter._get_linting_config(config_path) rules = [r for r in self.rules if r.__name__ not in config.get("exclude", [])] @@ -240,7 +239,7 @@ def lint(paths: Sequence[Path]): if p.is_file(): files.add(p) else: - files.update(_git_list_files(p)) + files.update(git_list_files(p)) log.info(f"Compiled a list of {len(files)} to be linted") @@ -258,31 +257,3 @@ def lint(paths: Sequence[Path]): log.warning(f"Linting completed with {warn} warnings and {error} errors") if error: sys.exit(1) - - -@click.command(name="lint") -@click.option( - "--skip-compile", - help="Do not compile configs before linting. The same can be achieved by setting the `DSO_SKIP_COMPILE=1` env var.", - type=bool, - default=bool(int(os.environ.get("DSO_SKIP_COMPILE", 0))), - is_flag=True, -) -@click.argument("args", nargs=-1) -def cli(args, skip_compile: bool = False): - """Lint a dso project - - Performs consistency checks according to a set of rules. - - If passing no arguments, linting will be performed for the current working directory. Alternatively a list of paths - can be specified. In that case, all stages related to any of the files are linted (useful for using with pre-commit). - - Configurations are compiled before linting. - """ - if not len(args): - paths = [Path.cwd()] - else: - paths = [Path(x) for x in args] - if not skip_compile: - compile_all_configs(paths) - lint(paths) diff --git a/src/dso/_quarto.py b/src/dso/_quarto.py new file mode 100644 index 0000000..5aed78e --- /dev/null +++ b/src/dso/_quarto.py @@ -0,0 +1,96 @@ +"""Helper functions for rendering quarto documents""" + +import os +import stat +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from pathlib import Path +from textwrap import dedent, indent + +from ruamel.yaml import YAML + + +def render_quarto(quarto_dir: Path, report_dir: Path, before_script: str, cwd: Path, with_pandocfilter: bool = False): + """ + Render a quarto project + + Parameters + ---------- + quarto_dir + Path that contains the _quarto.yml document + report_dir + Output directory of the rendered document + before_script + Bash snippet to execute before running quarto (e.g. to setup the enviornment) + """ + before_script = indent(before_script, " " * 8) + report_dir = report_dir.absolute() + report_dir.mkdir(exist_ok=True) + + # clean up existing `.rmarkdown` files that may interfere with rendering + # these are leftovers from a previous, failed `quarto render` attempt. If they still exist, the next attempt + # fails. We remove them *before* the run instead of cleaning them up *after* the run, because they + # may be usefule for debugging failures. + # see https://github.com/Boehringer-Ingelheim/dso/issues/54 + for f in quarto_dir.glob("*.rmarkdown"): + if f.is_file(): + f.unlink() + + # Enable pandocfilter if requested. + # We create a temporary script that then calls the current python binary with the dso.pandocfilter module + # This may seem cumbersome, but we do it this way because + # * pandoc only supports a single binary for `--filter`, referring to subcommands or `-m` is not possible here + # * we want to ensure that exactly the same python/dso version is used for the pandocfilter as for the + # parent command (important when running through dso-mgr) + filter_script = None + if with_pandocfilter: + with tempfile.NamedTemporaryFile(delete=False, mode="w") as f: + f.write("#!/bin/bash\n") + f.write(f'{sys.executable} -m dso.pandocfilter "$@"\n') + filter_script = Path(f.name) + + filter_script.chmod(filter_script.stat().st_mode | stat.S_IEXEC) + + pandocfilter = f"--filter {filter_script}" + else: + pandocfilter = "" + + # propagate quiet setting to quarto + quiet = "--quiet" if bool(int(os.environ.get("DSO_QUIET", 0))) else "" + script = dedent( + f"""\ + #!/bin/bash + set -euo pipefail + + # this flags enables building larger reports with embedded resources + export QUARTO_DENO_V8_OPTIONS=--max-old-space-size=8192 + + {before_script} + + quarto render "{quarto_dir}" --output-dir "{report_dir}" {quiet} {pandocfilter} + """ + ) + res = subprocess.run(script, shell=True, executable="/bin/bash", cwd=cwd) + + # clean up + if filter_script is not None: + filter_script.unlink() + + if res.returncode: + sys.exit(res.returncode) + + +@contextmanager +def quarto_config_yml(quarto_config: dict | None, quarto_dir: Path): + """Context manager that temporarily creates a _quarto.yml file and cleans up after itself""" + if quarto_config is None: + quarto_config = {} + config_file = quarto_dir / "_quarto.yml" + yaml = YAML(typ="safe") + yaml.dump(quarto_config, config_file) + try: + yield + finally: + config_file.unlink() diff --git a/src/dso/_util.py b/src/dso/_util.py index d8ab40a..21deaff 100644 --- a/src/dso/_util.py +++ b/src/dso/_util.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import importlib import json import subprocess @@ -5,23 +7,17 @@ from collections.abc import Sequence from functools import cache from importlib import resources - -try: - # has been added in Python 3.11 - from importlib.resources.abc import Traversable -except ImportError: - # will be removed in Python 3.14 - from importlib.abc import Traversable from os import environ from pathlib import Path -from typing import Literal +from typing import TYPE_CHECKING, Literal -from git.repo import Repo -from jinja2 import StrictUndefined, Template from rich.prompt import Confirm from dso._logging import console, log +if TYPE_CHECKING: + from importlib.resources.abc import Traversable + DEFAULT_BRANCH = "master" @@ -38,7 +34,7 @@ def check_project_roots(paths: Sequence[Path]) -> Path: return tmp_project_roots.pop() -def _find_in_parent(start_directory: Path, file_or_folder: str, recurse_barrier: Path | None = None) -> Path | None: +def find_in_parent(start_directory: Path, file_or_folder: str, recurse_barrier: Path | None = None) -> Path | None: """ Recursively walk up to the folder directory until we either find `file_or_folder` or reach the root. @@ -49,7 +45,9 @@ def _find_in_parent(start_directory: Path, file_or_folder: str, recurse_barrier: If the root is reached without finding the file, None is returned. """ return _find_in_parent_abs( - start_directory.absolute(), file_or_folder, recurse_barrier.absolute() if recurse_barrier is not None else None + start_directory.absolute(), + file_or_folder, + recurse_barrier.absolute() if recurse_barrier is not None else None, ) @@ -93,7 +91,7 @@ def get_project_root(start_directory: Path) -> Path: FileNotFoundError If the .git folder is not found. """ - proj_root = _find_in_parent(start_directory, ".git") + proj_root = find_in_parent(start_directory, ".git") if proj_root is None: raise FileNotFoundError("Not within a dso project (No .git directory found)") else: @@ -101,13 +99,15 @@ def get_project_root(start_directory: Path) -> Path: return proj_root.parent -def _get_template_path(template_type: Literal["init", "folder", "stage"], template_name: str) -> Traversable: +def get_template_path(template_type: Literal["init", "folder", "stage"], template_name: str) -> Traversable: template_module = importlib.import_module(f"dso.templates.{template_type}") return resources.files(template_module) / template_name def _copy_with_render(source: Traversable, destination: Path, params: dict): """Fill all placeholders in a file with jinja2 and save file to destination""" + from jinja2 import StrictUndefined, Template + with source.open() as f: template = Template(f.read(), undefined=StrictUndefined) rendered_content = template.render(params) @@ -118,8 +118,10 @@ def _copy_with_render(source: Traversable, destination: Path, params: dict): file.write("\n") -def _instantiate_template(template_path: Traversable, target_dir: Path | str, **params) -> None: +def instantiate_template(template_path: Traversable, target_dir: Path | str, **params) -> None: """Copy a template folder to a target directory, filling all placeholder values.""" + from jinja2 import Template + target_dir = Path(target_dir) def _traverse_template(curr_path, subdir): @@ -138,16 +140,18 @@ def _traverse_template(curr_path, subdir): _traverse_template(template_path, Path(".")) -def _instantiate_with_repo(template: Traversable, target_dir: Path | str, **params) -> None: +def instantiate_with_repo(template: Traversable, target_dir: Path | str, **params) -> None: """Create a git repo in a directory and render a template inside. Creates an initial commit. """ + from git.repo import Repo + target_dir = Path(target_dir) target_dir.mkdir(exist_ok=True) log.info("Created project directory.") - _instantiate_template(template, target_dir, **params) + instantiate_template(template, target_dir, **params) log.info("Created folder structure from template.") if not (target_dir / ".git").exists(): @@ -161,7 +165,7 @@ def _instantiate_with_repo(template: Traversable, target_dir: Path | str, **para log.info("Initalized local git repo.") -def _git_list_files(dir: Path) -> list[Path]: +def git_list_files(dir: Path) -> list[Path]: """ Recursively list all files in `dir` that are not .gitignored. @@ -169,7 +173,9 @@ def _git_list_files(dir: Path) -> list[Path]: Source: https://stackoverflow.com/a/77197460/2340703 """ res = subprocess.run( - ["git", "ls-files", "--cached", "--others", "--exclude-standard"], cwd=dir, capture_output=True + ["git", "ls-files", "--cached", "--others", "--exclude-standard"], + cwd=dir, + capture_output=True, ) if res.returncode: sys.exit(res.returncode) diff --git a/src/dso/watermark.py b/src/dso/_watermark.py similarity index 85% rename from src/dso/watermark.py rename to src/dso/_watermark.py index 578c46d..5abcac4 100644 --- a/src/dso/watermark.py +++ b/src/dso/_watermark.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import Generic, TypeVar -import rich_click as click from PIL import Image, ImageDraw, ImageFont from pypdf import PdfReader, PdfWriter from svgutils import compose @@ -180,27 +179,3 @@ def apply_and_save(self, input_image: Path | str, output_image: Path | str): with open(output_image, "wb") as f: writer.write(f) reader.close() - - -@click.command(name="watermark") -@click.argument("input_image", type=Path) -@click.argument("output_image", type=Path) -@click.option("--text", help="Text to use as watermark", required=True) -@click.option( - "--tile_size", - type=(int, int), - help="watermark text will be arranged in tile of this size (once at top left, once at middle right). Specify the tile size as e.g. `120 80`", -) -@click.option("--font_size", type=int) -@click.option("--font_outline", type=int) -@click.option("--font_color", help="Use RGBA (e.g. `#AAAAAA88`) to specify transparency") -@click.option("--font_outline_color", help="Use RGBA (e.g. `#AAAAAA88`) to specify transparency") -def cli(input_image, output_image, text, **kwargs): - """Add a watermark to an image - - To be called from the dso-r package for implementing a custom graphics device. - Can also be used standalone for watermarking images. - """ - kwargs = {k: v for k, v in kwargs.items() if v is not None} - - Watermarker.add_watermark(input_image, output_image, text=text, **kwargs) diff --git a/src/dso/api.py b/src/dso/api.py new file mode 100644 index 0000000..9924f86 --- /dev/null +++ b/src/dso/api.py @@ -0,0 +1,94 @@ +"""Python API, e.g. to be called from jupyter notebooks. + +The functionality is the same as provided by the dso-r package. +""" + +import os +from dataclasses import dataclass +from pathlib import Path +from textwrap import dedent + +from dso._get_config import get_config +from dso._logging import log + +from ._util import get_project_root + + +@dataclass +class Config: + """DSO config class""" + + stage_here: Path | None = None + """Absolute path to the current stage""" + + +CONFIG = Config() +"""Global configuration storage of the DSO API""" + + +def here(rel_path: str | Path | None = None) -> Path: + """Get project root as a Path object + + Parameters + ---------- + rel_path + Relative path to be appended to the project root + """ + proj_root = get_project_root(Path.cwd()) + if rel_path is None: + return proj_root + else: + return proj_root / rel_path + + +def stage_here(rel_path: str | Path | None = None) -> Path: + """ + Get the absolute path to the current stage + + The current stage is stored in `dso.CONFIG` and can be set using `dso.set_stage` or + `dso.read_params`. + + Parameters + ---------- + rel_path + Relative path to be appended to the project root + """ + if CONFIG.stage_here is None: + raise RuntimeError("No stage has been set. Run `read_params` or `set_stage` first!") + if rel_path is None: + return CONFIG.stage_here + else: + return CONFIG.stage_here / rel_path + + +def set_stage(stage: str | Path) -> None: + """ + Set the active stage for `stage_here()` + + This sets the stage dir in `dso.CONFIG`. + + Parameters + ---------- + stage + Path to stage, relative to the project root + """ + proj_root = get_project_root(Path.cwd()) + if not (proj_root / stage).exists(): + raise ValueError( + dedent( + f"""\ + The stage `{stage}` could not be found. + + Current working directory: `{Path.cwd()}` + Inferred project root: `{proj_root}` + """ + ) + ) + CONFIG.stage_here = proj_root / stage + log.info(f"stage_here() starts at {CONFIG.stage_here}") + + +def read_params(stage: str | Path) -> dict: + """Set stage dir and load parameters from params.yaml""" + set_stage(stage) + return get_config(str(stage), skip_compile=bool(int(os.environ.get("DSO_SKIP_COMPILE", 0)))) diff --git a/src/dso/cli/__init__.py b/src/dso/cli/__init__.py new file mode 100644 index 0000000..ddf8897 --- /dev/null +++ b/src/dso/cli/__init__.py @@ -0,0 +1,234 @@ +"""Main entry point for CLI""" + +import logging +import os +import subprocess +import sys +from os import getcwd +from pathlib import Path + +import rich_click as click +from rich.prompt import Confirm, Prompt +from ruamel.yaml import YAML + +from dso._logging import log +from dso._metadata import __version__ +from dso._util import get_project_root, get_template_path, instantiate_with_repo + +from ._create import create_cli +from ._exec import exec_cli + +click.rich_click.USE_MARKDOWN = True + + +@click.command(name="compile-config") +@click.argument("args", nargs=-1) +def compile_config_cli(args): + """Compile params.in.yaml into params.yaml using Jinja2 templating and resolving recursive templates. + + If passing no arguments, configs will be resolved for the current working directory (i.e. all parent configs, + and all configs in child directories). Alternatively a list of paths can be specified. In that case, all configs + related to these paths will be compiled (useful for using with pre-commit). + """ + from dso._compile_config import compile_all_configs + + if not len(args): + paths = [Path.cwd()] + else: + paths = [Path(x) for x in args] + + compile_all_configs(paths) + + +@click.command(name="get-config") +@click.option( + "--all", + is_flag=True, + type=bool, + default=False, + help="Include all parameters, not only those mentioned in `dvc.yaml`", +) +@click.option( + "--skip-compile", + is_flag=True, + type=bool, + default=bool(int(os.environ.get("DSO_SKIP_COMPILE", 0))), + help="Do not compile configs before loading it. The same can be achieved by setting the `DSO_SKIP_COMPILE=1` env var.", +) +@click.argument( + "stage", +) +def get_config_cli(stage, all, skip_compile): + """Get the configuration for a given stage and print it to STDOUT in yaml format. + + The path to the stage must be relative to the root dir of the project. + + By default, the configuration is filtered to include only the keys that are mentioned in `dvc.yaml` to force + declaring all dependencies. + + If multiple stages are defined in a single `dvc.yaml`, the stage name MUST be specified using + `path/to/stage:stage_name` unless `--all` is given. + """ + from dso._get_config import get_config + + try: + out_config = get_config(stage, all=all, skip_compile=skip_compile) + yaml = YAML() + yaml.dump(out_config, sys.stdout) + except KeyError as e: + log.error(f"dvc.yaml defines parameter {e} that is not in params.yaml") + sys.exit(1) + + +@click.option("--description") +@click.argument("name", required=False) +@click.command( + "init", +) +def init_cli(name: str | None = None, description: str | None = None): + """ + Initialize a new project. A project can contain several stages organized in arbitrary subdirectories. + + If you wish to initialize DSO in an existing project, you can specify an existing directory. In + this case, it will initialize files from the template that do not exist yet, but never overwrite existing files. + """ + from dso._compile_config import compile_all_configs + + if name is None: + name = Prompt.ask('[bold]Please enter the name of the project, e.g. "single_cell_lung_atlas"') + + target_dir = Path(getcwd()) / name + + if target_dir.exists(): + if not Confirm.ask("[bold]Directory already exists. Do you want to initialize DSO in an existing project?"): + sys.exit(1) + + if description is None: + description = Prompt.ask("[bold]Please add a short description of the project") + + instantiate_with_repo( + get_template_path("init", "default"), target_dir, project_name=name, project_description=description + ) + log.info("[green]Project initalized successfully.") + compile_all_configs([target_dir]) + + +@click.command(name="lint") +@click.option( + "--skip-compile", + help="Do not compile configs before linting. The same can be achieved by setting the `DSO_SKIP_COMPILE=1` env var.", + type=bool, + default=bool(int(os.environ.get("DSO_SKIP_COMPILE", 0))), + is_flag=True, +) +@click.argument("args", nargs=-1) +def lint_cli(args, skip_compile: bool = False): + """Lint a dso project + + Performs consistency checks according to a set of rules. + + If passing no arguments, linting will be performed for the current working directory. Alternatively a list of paths + can be specified. In that case, all stages related to any of the files are linted (useful for using with pre-commit). + + Configurations are compiled before linting. + """ + from dso._compile_config import compile_all_configs + from dso._lint import lint + + if not len(args): + paths = [Path.cwd()] + else: + paths = [Path(x) for x in args] + if not skip_compile: + compile_all_configs(paths) + lint(paths) + + +@click.command( + name="repro", + context_settings={"ignore_unknown_options": True}, +) +@click.argument("args", nargs=-1, type=click.UNPROCESSED) +def repro_cli(args): + """Wrapper around dvc repro, compiling configuration before running.""" + from dso._compile_config import compile_all_configs + from dso._util import check_ask_pre_commit + + check_ask_pre_commit(Path.cwd()) + compile_all_configs([get_project_root(Path.cwd())]) + os.environ["DSO_SKIP_COMPILE"] = "1" + cmd = ["dvc", "repro", *args] + log.info(f"Running `{' '.join(cmd)}`") + res = subprocess.run(cmd) + sys.exit(res.returncode) + + +@click.command(name="watermark") +@click.argument("input_image", type=Path) +@click.argument("output_image", type=Path) +@click.option("--text", help="Text to use as watermark", required=True) +@click.option( + "--tile_size", + type=(int, int), + help="watermark text will be arranged in tile of this size (once at top left, once at middle right). Specify the tile size as e.g. `120 80`", +) +@click.option("--font_size", type=int) +@click.option("--font_outline", type=int) +@click.option("--font_color", help="Use RGBA (e.g. `#AAAAAA88`) to specify transparency") +@click.option("--font_outline_color", help="Use RGBA (e.g. `#AAAAAA88`) to specify transparency") +def watermark_cli(input_image, output_image, text, **kwargs): + """Add a watermark to an image + + To be called from the dso-r package for implementing a custom graphics device. + Can also be used standalone for watermarking images. + """ + kwargs = {k: v for k, v in kwargs.items() if v is not None} + + from dso._watermark import Watermarker + + Watermarker.add_watermark(input_image, output_image, text=text, **kwargs) + + +@click.group() +@click.option( + "-q", + "--quiet", + count=True, + help=( + "Reduce verbosity. `-q` disables info messages, `-qq` disables warnings. Errors messages cannot be disabled. " + "The same can be achieved by setting the env var `DSO_QUIET=1` or `DSO_QUIET=2`, respectively." + ), + default=int(os.environ.get("DSO_QUIET", 0)), +) +@click.option( + "-v", + "--verbose", + help=( + "Increase logging verbosity to include debug messages. " + "The same can be achieved by setting the env var `DSO_VERBOSE=1`." + ), + default=bool(int(os.environ.get("DSO_VERBOSE", 0))), + is_flag=True, +) +@click.version_option(version=__version__, prog_name="dso") +def cli(quiet: int, verbose: bool): + """Root command""" + if quiet >= 2: + log.setLevel(logging.ERROR) + os.environ["DSO_QUIET"] = "2" + elif quiet == 1: + log.setLevel(logging.WARNING) + os.environ["DSO_QUIET"] = "1" + elif verbose: + log.setLevel(logging.DEBUG) + os.environ["DSO_VERBOSE"] = "1" + + +cli.add_command(create_cli) +cli.add_command(init_cli) +cli.add_command(compile_config_cli) +cli.add_command(repro_cli) +cli.add_command(exec_cli) +cli.add_command(lint_cli) +cli.add_command(get_config_cli) +cli.add_command(watermark_cli) diff --git a/src/dso/create.py b/src/dso/cli/_create.py similarity index 83% rename from src/dso/create.py rename to src/dso/cli/_create.py index 909ae21..6eda77b 100644 --- a/src/dso/create.py +++ b/src/dso/cli/_create.py @@ -5,15 +5,12 @@ from pathlib import Path from textwrap import dedent, indent -import questionary import rich_click as click from rich.prompt import Confirm, Prompt from dso._logging import log -from dso._util import _get_template_path, _instantiate_template, get_project_root -from dso.compile_config import compile_all_configs +from dso._util import get_project_root, get_template_path, instantiate_template -DEFAULT_BRANCH = "master" # list of stage template with description - can be later populated also from external directories STAGE_TEMPLATES = { "bash": "Execute a simple bash snippet or call an external script (e.g. nextflow)", @@ -36,8 +33,12 @@ @click.option("--template", type=click.Choice(list(STAGE_TEMPLATES))) @click.argument("name", required=False) @click.command("stage", help=CREATE_STAGE_HELP_TEXT) -def create_stage(name: str | None = None, template: str | None = None, description: str | None = None): +def create_stage_cli(name: str | None = None, template: str | None = None, description: str | None = None): """Create a new stage.""" + import questionary + + from dso._compile_config import compile_all_configs + if template is None: template = str(questionary.select("Choose a template:", choices=list(STAGE_TEMPLATES)).ask()) @@ -57,8 +58,8 @@ def create_stage(name: str | None = None, template: str | None = None, descripti project_root = get_project_root(target_dir) stage_path = target_dir.relative_to(project_root) - _instantiate_template( - _get_template_path("stage", template), + instantiate_template( + get_template_path("stage", template), target_dir, stage_name=name, stage_description=description, @@ -70,7 +71,7 @@ def create_stage(name: str | None = None, template: str | None = None, descripti @click.argument("name", required=False) @click.command("folder") -def create_folder(name: str | None = None): +def create_folder_cli(name: str | None = None): """Create a new folder. A folder can contain subfolders or stages. Technically, nothing prevents you from just using `mkdir`. This command additionally adds some default @@ -80,6 +81,8 @@ def create_folder(name: str | None = None): be copied to the folder. Existing files will never be overwritten. """ # currently there's only one template for folders + from dso._compile_config import compile_all_configs + template = "default" if name is None: @@ -93,16 +96,16 @@ def create_folder(name: str | None = None): target_dir.mkdir(exist_ok=True) - _instantiate_template(_get_template_path("folder", template), target_dir, stage_name=name) + instantiate_template(get_template_path("folder", template), target_dir, stage_name=name) log.info("[green]Folder created successfully.") compile_all_configs([target_dir]) @click.group(name="create") -def cli(): +def create_cli(): """Create stage folder structure subcommand.""" pass -cli.add_command(create_stage) -cli.add_command(create_folder) +create_cli.add_command(create_stage_cli) +create_cli.add_command(create_folder_cli) diff --git a/src/dso/cli/_exec.py b/src/dso/cli/_exec.py new file mode 100644 index 0000000..ca65090 --- /dev/null +++ b/src/dso/cli/_exec.py @@ -0,0 +1,92 @@ +import os +from pathlib import Path + +import rich_click as click +from ruamel.yaml import YAML + +from dso._logging import log + + +@click.command("quarto") +@click.argument("stage", required=True) +@click.option( + "--skip-compile", + help="Do not compile configs before linting. The same can be achieved by setting the `DSO_SKIP_COMPILE=1` env var.", + type=bool, + default=bool(int(os.environ.get("DSO_SKIP_COMPILE", 0))), + is_flag=True, +) +def exec_quarto_cli(stage: str, skip_compile: bool = True): + """ + Render a quarto stage. Quarto parameters are inherited from params.yaml + + A quarto stage is assmed to have the following structure: + * One or multiple `.qmd` files in `src` + * Reports will be stored in `report` + + No `_quarto.yml` shall be present as it will be automatically created tempoarily. Instead + supply quarto parameters in `params.in.yaml` under the key `dso.quarto`. + + Parameters + ---------- + stage + Path to the stage, e.g. `.` for the current directory. + """ + from dso._compile_config import compile_all_configs + from dso._quarto import quarto_config_yml, render_quarto + + stage_dir = Path(stage).absolute() + log.info(f"Executing quarto stage {stage_dir}") + if not skip_compile: + log.debug("Skipping compilation of config files.") + compile_all_configs([stage_dir]) + os.environ["DSO_SKIP_COMPILE"] = ( + "1" # no need to re-compile the config when calling `read_params` in the script + ) + yaml = YAML(typ="safe") + params = yaml.load(stage_dir / "params.yaml") + dso_config = params.get("dso", {}) + if dso_config is None: + dso_config = {} + quarto_config = dso_config.get("quarto", {}) + # before script is dso-specific - we retrieve and remove it from the quarto config + before_script = quarto_config.pop("before_script", "") + + # The following keys in the quarto configuration are paths. They need to be amended by a ".." to compensate + # for the `src` directory in the quarto stage. I couln't find a comprehensive specification of the + # _quarto.yml file to find all possible keys that are affected. Let's just grow this list as issues appear. + QUARTO_PATH_KEYS = ["bibliography", "css"] + for key in QUARTO_PATH_KEYS: + try: + tmp_list = quarto_config[key] + new_conf = [] + # can be either a str or a list of strs (in case of multiple files). Let's force this to be a list. + if isinstance(tmp_list, str): + tmp_list = [tmp_list] + for tmp_val in tmp_list: + tmp_path = Path(tmp_val) + if not tmp_path.is_absolute(): + tmp_path = ".." / tmp_path + new_conf.append(str(tmp_path)) + # if only one entry, don't add a list + quarto_config[key] = new_conf if len(new_conf) > 1 else new_conf[0] + except KeyError: + pass + + with quarto_config_yml(quarto_config, stage_dir / "src"): + render_quarto( + stage_dir / "src", + report_dir=stage_dir / "report", + before_script=before_script, + cwd=stage_dir, + with_pandocfilter="watermark" in quarto_config or "disclaimer" in quarto_config, + ) + + +@click.group(name="exec") +def exec_cli(): + """Dso wrappers around various tools""" + pass + + +exec_cli.add_command(exec_quarto_cli) diff --git a/src/dso/exec.py b/src/dso/exec.py deleted file mode 100644 index 78b97d3..0000000 --- a/src/dso/exec.py +++ /dev/null @@ -1,181 +0,0 @@ -import os -import stat -import subprocess -import sys -import tempfile -from contextlib import contextmanager -from pathlib import Path -from textwrap import dedent, indent - -import rich_click as click -from ruamel.yaml import YAML - -from dso.compile_config import compile_all_configs - -from ._logging import log - - -def _render_quarto(quarto_dir: Path, report_dir: Path, before_script: str, cwd: Path, with_pandocfilter: bool = False): - """ - Render a quarto project - - Parameters - ---------- - quarto_dir - Path that contains the _quarto.yml document - report_dir - Output directory of the rendered document - before_script - Bash snippet to execute before running quarto (e.g. to setup the enviornment) - """ - before_script = indent(before_script, " " * 8) - report_dir = report_dir.absolute() - report_dir.mkdir(exist_ok=True) - - # clean up existing `.rmarkdown` files that may interfere with rendering - # these are leftovers from a previous, failed `quarto render` attempt. If they still exist, the next attempt - # fails. We remove them *before* the run instead of cleaning them up *after* the run, because they - # may be usefule for debugging failures. - # see https://github.com/Boehringer-Ingelheim/dso/issues/54 - for f in quarto_dir.glob("*.rmarkdown"): - if f.is_file(): - f.unlink() - - # Enable pandocfilter if requested. - # We create a temporary script that then calls the current python binary with the dso.pandocfilter module - # This may seem cumbersome, but we do it this way because - # * pandoc only supports a single binary for `--filter`, referring to subcommands or `-m` is not possible here - # * we want to ensure that exactly the same python/dso version is used for the pandocfilter as for the - # parent command (important when running through dso-mgr) - filter_script = None - if with_pandocfilter: - with tempfile.NamedTemporaryFile(delete=False, mode="w") as f: - f.write("#!/bin/bash\n") - f.write(f'{sys.executable} -m dso.pandocfilter "$@"\n') - filter_script = Path(f.name) - - filter_script.chmod(filter_script.stat().st_mode | stat.S_IEXEC) - - pandocfilter = f"--filter {filter_script}" - else: - pandocfilter = "" - - # propagate quiet setting to quarto - quiet = "--quiet" if bool(int(os.environ.get("DSO_QUIET", 0))) else "" - script = dedent( - f"""\ - #!/bin/bash - set -euo pipefail - - # this flags enables building larger reports with embedded resources - export QUARTO_DENO_V8_OPTIONS=--max-old-space-size=8192 - - {before_script} - - quarto render "{quarto_dir}" --output-dir "{report_dir}" {quiet} {pandocfilter} - """ - ) - res = subprocess.run(script, shell=True, executable="/bin/bash", cwd=cwd) - - # clean up - if filter_script is not None: - filter_script.unlink() - - if res.returncode: - sys.exit(res.returncode) - - -@contextmanager -def _quarto_config_yml(quarto_config: dict | None, quarto_dir: Path): - """Context manager that temporarily creates a _quarto.yml file and cleans up after itself""" - if quarto_config is None: - quarto_config = {} - config_file = quarto_dir / "_quarto.yml" - yaml = YAML(typ="safe") - yaml.dump(quarto_config, config_file) - try: - yield - finally: - config_file.unlink() - - -@click.command("quarto") -@click.argument("stage", required=True) -@click.option( - "--skip-compile", - help="Do not compile configs before linting. The same can be achieved by setting the `DSO_SKIP_COMPILE=1` env var.", - type=bool, - default=bool(int(os.environ.get("DSO_SKIP_COMPILE", 0))), - is_flag=True, -) -def exec_quarto(stage: str, skip_compile: bool = True): - """ - Render a quarto stage. Quarto parameters are inherited from params.yaml - - A quarto stage is assmed to have the following structure: - * One or multiple `.qmd` files in `src` - * Reports will be stored in `report` - - No `_quarto.yml` shall be present as it will be automatically created tempoarily. Instead - supply quarto parameters in `params.in.yaml` under the key `dso.quarto`. - - Parameters - ---------- - stage - Path to the stage, e.g. `.` for the current directory. - """ - stage_dir = Path(stage).absolute() - log.info(f"Executing quarto stage {stage_dir}") - if not skip_compile: - log.debug("Skipping compilation of config files.") - compile_all_configs([stage_dir]) - os.environ["DSO_SKIP_COMPILE"] = ( - "1" # no need to re-compile the config when calling `read_params` in the script - ) - yaml = YAML(typ="safe") - params = yaml.load(stage_dir / "params.yaml") - dso_config = params.get("dso", {}) - if dso_config is None: - dso_config = {} - quarto_config = dso_config.get("quarto", {}) - # before script is dso-specific - we retrieve and remove it from the quarto config - before_script = quarto_config.pop("before_script", "") - - # The following keys in the quarto configuration are paths. They need to be amended by a ".." to compensate - # for the `src` directory in the quarto stage. I couln't find a comprehensive specification of the - # _quarto.yml file to find all possible keys that are affected. Let's just grow this list as issues appear. - QUARTO_PATH_KEYS = ["bibliography", "css"] - for key in QUARTO_PATH_KEYS: - try: - tmp_list = quarto_config[key] - new_conf = [] - # can be either a str or a list of strs (in case of multiple files). Let's force this to be a list. - if isinstance(tmp_list, str): - tmp_list = [tmp_list] - for tmp_val in tmp_list: - tmp_path = Path(tmp_val) - if not tmp_path.is_absolute(): - tmp_path = ".." / tmp_path - new_conf.append(str(tmp_path)) - # if only one entry, don't add a list - quarto_config[key] = new_conf if len(new_conf) > 1 else new_conf[0] - except KeyError: - pass - - with _quarto_config_yml(quarto_config, stage_dir / "src"): - _render_quarto( - stage_dir / "src", - report_dir=stage_dir / "report", - before_script=before_script, - cwd=stage_dir, - with_pandocfilter="watermark" in quarto_config or "disclaimer" in quarto_config, - ) - - -@click.group(name="exec") -def cli(): - """Dso wrappers around various tools""" - pass - - -cli.add_command(exec_quarto) diff --git a/src/dso/init.py b/src/dso/init.py deleted file mode 100644 index 85082e5..0000000 --- a/src/dso/init.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Initializes the project folder structure and git""" - -import sys -from os import getcwd -from pathlib import Path - -import rich_click as click -from rich.prompt import Confirm, Prompt - -from dso._logging import log -from dso._util import _get_template_path, _instantiate_with_repo -from dso.compile_config import compile_all_configs - -DEFAULT_BRANCH = "master" - - -@click.option("--description") -@click.argument("name", required=False) -@click.command( - "init", -) -def cli(name: str | None = None, description: str | None = None): - """ - Initialize a new project. A project can contain several stages organized in arbitrary subdirectories. - - If you wish to initialize DSO in an existing project, you can specify an existing directory. In - this case, it will initialize files from the template that do not exist yet, but never overwrite existing files. - """ - if name is None: - name = Prompt.ask('[bold]Please enter the name of the project, e.g. "single_cell_lung_atlas"') - - target_dir = Path(getcwd()) / name - - if target_dir.exists(): - if not Confirm.ask("[bold]Directory already exists. Do you want to initialize DSO in an existing project?"): - sys.exit(1) - - if description is None: - description = Prompt.ask("[bold]Please add a short description of the project") - - _instantiate_with_repo( - _get_template_path("init", "default"), target_dir, project_name=name, project_description=description - ) - log.info("[green]Project initalized successfully.") - compile_all_configs([target_dir]) diff --git a/src/dso/pandocfilter.py b/src/dso/pandocfilter.py index 63abdf8..59709dd 100644 --- a/src/dso/pandocfilter.py +++ b/src/dso/pandocfilter.py @@ -4,7 +4,7 @@ * warning box at the top * watermark to all PNG images -Called internally by `dso exec quarto`. +Called internally by `dso exec quarto` via `python -m dso.pandocfilter`. """ import sys @@ -20,7 +20,7 @@ from panflute import Div, Image, RawBlock, run_filter from dso._logging import log -from dso.watermark import Watermarker +from dso._watermark import Watermarker def _get_disclaimer_box(title, text): diff --git a/src/dso/repro.py b/src/dso/repro.py deleted file mode 100644 index 4e8accd..0000000 --- a/src/dso/repro.py +++ /dev/null @@ -1,26 +0,0 @@ -import os -import subprocess -import sys -from pathlib import Path - -import rich_click as click - -from dso._logging import log -from dso._util import check_ask_pre_commit, get_project_root -from dso.compile_config import compile_all_configs - - -@click.command( - name="repro", - context_settings={"ignore_unknown_options": True}, -) -@click.argument("args", nargs=-1, type=click.UNPROCESSED) -def cli(args): - """Wrapper around dvc repro, compiling configuration before running.""" - check_ask_pre_commit(Path.cwd()) - compile_all_configs([get_project_root(Path.cwd())]) - os.environ["DSO_SKIP_COMPILE"] = "1" - cmd = ["dvc", "repro", *args] - log.info(f"Running `{' '.join(cmd)}`") - res = subprocess.run(cmd) - sys.exit(res.returncode) diff --git a/src/dso/templates/init/default/.gitignore b/src/dso/templates/init/default/.gitignore index f0742ef..cb2e7c0 100644 --- a/src/dso/templates/init/default/.gitignore +++ b/src/dso/templates/init/default/.gitignore @@ -47,3 +47,6 @@ sccprj/ # Windows Thumbs.db ~$* + +# dso +.dso.json diff --git a/tests/conftest.py b/tests/conftest.py index 0b048a6..c37adc0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,9 +5,8 @@ from click.testing import CliRunner from pytest import fixture -from dso.compile_config import compile_all_configs -from dso.create import cli as dso_create -from dso.init import cli as dso_init +from dso._compile_config import compile_all_configs +from dso.cli import create_cli, init_cli TESTDATA = Path(__file__).parent / "data" @@ -18,7 +17,7 @@ def dso_project(tmp_path) -> Path: runner = CliRunner() proj_name = "dso_project" chdir(tmp_path) - runner.invoke(dso_init, [proj_name, "--description", "a test project"]) + runner.invoke(init_cli, [proj_name, "--description", "a test project"]) return tmp_path / proj_name @@ -32,7 +31,7 @@ def quarto_stage(dso_project) -> Path: runner = CliRunner() stage_name = "quarto_stage" chdir(dso_project) - runner.invoke(dso_create, ["stage", stage_name, "--template", "quarto", "--description", "a quarto stage"]) + runner.invoke(create_cli, ["stage", stage_name, "--template", "quarto", "--description", "a quarto stage"]) with (Path(stage_name) / "src" / f"{stage_name}.qmd").open("w") as f: f.write( dedent( diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 0000000..4439985 --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,33 @@ +from os import chdir + +import pytest + +from dso import here, read_params, set_stage, stage_here + + +def test_api(quarto_stage): + chdir(quarto_stage) + + assert here() == (quarto_stage / "..").resolve() + assert here("quarto_stage") == quarto_stage + + # stage not yet initialized + with pytest.raises(RuntimeError): + stage_here() + + # stage must exist + with pytest.raises(ValueError): + set_stage("doesnt/exist") + + set_stage("quarto_stage") + + assert stage_here() == quarto_stage + assert stage_here("dvc.yaml") == quarto_stage / "dvc.yaml" + + +def test_api_read_params(quarto_stage): + # more extensive tests of the same functionality are in `test_get_config.py` + chdir(quarto_stage) + + params = read_params("quarto_stage") + assert "dso" in params diff --git a/tests/test_cli.py b/tests/test_cli.py index 8cb0eb6..3268691 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,6 @@ from click.testing import CliRunner -from dso import cli +from dso.cli import cli def test_root_command(): diff --git a/tests/test_compile_config.py b/tests/test_compile_config.py index 2c1f616..d1bf3de 100644 --- a/tests/test_compile_config.py +++ b/tests/test_compile_config.py @@ -9,12 +9,12 @@ from click.testing import CliRunner from ruamel.yaml import YAML -from dso.compile_config import ( +from dso._compile_config import ( _get_list_of_configs_to_compile, _get_parent_configs, _load_yaml_with_auto_adjusting_paths, - cli, ) +from dso.cli import compile_config_cli def _setup_yaml_configs(tmp_path, configs: dict[str, dict]): @@ -94,7 +94,7 @@ def test_auto_adjusting_path_with_jinja(tmp_path, test_yaml, expected): with test_file.open("w") as f: f.write(dedent(test_yaml)) - result = runner.invoke(cli, []) + result = runner.invoke(compile_config_cli, []) print(result.output) td = Path(td) assert result.exit_code == 0 @@ -115,7 +115,7 @@ def test_compile_configs(tmp_path): "A/B/C/params.in.yaml": {"value": "C", "jinja2": "{{ only_root }}", "list": [5]}, }, ) - result = runner.invoke(cli, []) + result = runner.invoke(compile_config_cli, []) print(result.output) td = Path(td) assert result.exit_code == 0 @@ -140,7 +140,7 @@ def test_compile_configs_null_override(tmp_path): "A/B/params.in.yaml": {"str": None, "list": None, "dict": None, "null": None}, }, ) - result = runner.invoke(cli, []) + result = runner.invoke(compile_config_cli, []) print(result.output) td = Path(td) assert result.exit_code == 0 diff --git a/tests/test_create.py b/tests/test_create.py index d1b1560..9db93ee 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -4,14 +4,16 @@ import pytest from click.testing import CliRunner -from dso.create import cli +from dso.cli import create_cli @pytest.mark.parametrize("template", ["bash", "quarto"]) def test_create_stage(dso_project, template): runner = CliRunner() chdir(dso_project) - result = runner.invoke(cli, ["stage", "teststage", "--template", template, "--description", "testdescription"]) + result = runner.invoke( + create_cli, ["stage", "teststage", "--template", template, "--description", "testdescription"] + ) print(result.output) assert result.exit_code == 0 assert (dso_project / "teststage").is_dir() @@ -29,7 +31,7 @@ def test_create_stage(dso_project, template): def test_create_folder(dso_project): runner = CliRunner() chdir(dso_project) - result = runner.invoke(cli, ["folder", "testfolder"]) + result = runner.invoke(create_cli, ["folder", "testfolder"]) print(result.output) assert result.exit_code == 0 assert (dso_project / "testfolder").is_dir() @@ -46,7 +48,7 @@ def test_create_folder_existing_dir(dso_project): (dso_project / "testfolder").mkdir() (dso_project / "testfolder" / "dvc.yaml").touch() chdir(dso_project) - result = runner.invoke(cli, ["folder", "testfolder"], input="y") + result = runner.invoke(create_cli, ["folder", "testfolder"], input="y") print(result.output) assert result.exit_code == 0 assert (dso_project / "testfolder").is_dir() diff --git a/tests/test_exec.py b/tests/test_exec.py index 0799d3b..d7a2efe 100644 --- a/tests/test_exec.py +++ b/tests/test_exec.py @@ -4,7 +4,7 @@ import pytest from click.testing import CliRunner -from dso.exec import cli +from dso.cli import exec_cli @pytest.mark.parametrize("quiet", [None, "2"]) @@ -21,7 +21,7 @@ def test_exec_quarto(quarto_stage, quiet, launch_dir): if quiet is not None: os.environ["DSO_QUIET"] = quiet - result = runner.invoke(cli, ["quarto", stage_path]) + result = runner.invoke(exec_cli, ["quarto", stage_path]) assert result.exit_code == 0 assert (quarto_stage / "report" / "quarto_stage.html").is_file() assert "Hello World!" in (quarto_stage / "report" / "quarto_stage.html").read_text() @@ -32,7 +32,7 @@ def test_exec_quarto_empty_params(quarto_stage_empty_configs): chdir(quarto_stage_empty_configs) stage_path = "." - result = runner.invoke(cli, ["quarto", stage_path]) + result = runner.invoke(exec_cli, ["quarto", stage_path]) assert result.exit_code == 0 assert (quarto_stage_empty_configs / "report" / "quarto_stage.html").is_file() assert "Hello World!" in (quarto_stage_empty_configs / "report" / "quarto_stage.html").read_text() @@ -43,7 +43,7 @@ def test_exec_quarto_bibliography(quarto_stage_bibliography): chdir(quarto_stage_bibliography) stage_path = "." - result = runner.invoke(cli, ["quarto", stage_path]) + result = runner.invoke(exec_cli, ["quarto", stage_path]) assert result.exit_code == 0 assert (quarto_stage_bibliography / "report" / "quarto_stage.html").is_file() assert "Knuth" in (quarto_stage_bibliography / "report" / "quarto_stage.html").read_text() @@ -54,7 +54,7 @@ def test_exec_quarto_stylesheet(quarto_stage_css): chdir(quarto_stage_css) stage_path = "." - result = runner.invoke(cli, ["quarto", stage_path]) + result = runner.invoke(exec_cli, ["quarto", stage_path]) assert result.exit_code == 0 assert (quarto_stage_css / "report" / "quarto_stage.html").is_file() assert "h2.veryspecialclass1" in (quarto_stage_css / "report" / "quarto_stage.html").read_text() diff --git a/tests/test_get_config.py b/tests/test_get_config.py index e607ce3..173e337 100644 --- a/tests/test_get_config.py +++ b/tests/test_get_config.py @@ -4,7 +4,8 @@ import pytest from click.testing import CliRunner -from dso.get_config import _filter_nested_dict, cli, get_config +from dso._get_config import _filter_nested_dict, get_config +from dso.cli import get_config_cli @pytest.mark.parametrize( @@ -199,6 +200,6 @@ def test_get_config_invalid_stage(dso_project): def test_get_config_cli(quarto_stage): runner = CliRunner() chdir(quarto_stage) - result = runner.invoke(cli, ["quarto_stage"]) + result = runner.invoke(get_config_cli, ["quarto_stage"]) assert result.exit_code == 0 assert "quarto:" in result.output diff --git a/tests/test_init.py b/tests/test_init.py index 790296c..0fa13e6 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -4,13 +4,13 @@ from click.testing import CliRunner -from dso.init import cli +from dso.cli import init_cli def test_init(tmp_path): runner = CliRunner() with runner.isolated_filesystem(temp_dir=tmp_path) as td: - result = runner.invoke(cli, ["testproject", "--description", "testdescription"]) + result = runner.invoke(init_cli, ["testproject", "--description", "testdescription"]) print(result.output) td = Path(td) assert result.exit_code == 0 @@ -29,7 +29,7 @@ def test_init_existing_dir(dso_project): (dso_project / "README.md").unlink() rmtree(dso_project / ".git") - result = runner.invoke(cli, [str(dso_project), "--description", "testdescription"], input="y") + result = runner.invoke(init_cli, [str(dso_project), "--description", "testdescription"], input="y") assert result.exit_code == 0 assert (dso_project).is_dir() assert (dso_project / ".git").is_dir() diff --git a/tests/test_lint.py b/tests/test_lint.py index bfc6718..3b86b66 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -5,7 +5,8 @@ import pytest from click.testing import CliRunner -from dso.lint import DSO001, DSOLinter, LintError, QuartoRule, Rule, cli +from dso._lint import DSO001, DSOLinter, LintError, QuartoRule, Rule +from dso.cli import lint_cli @pytest.mark.parametrize( @@ -203,6 +204,6 @@ def check(cls, file): def test_lint_cli(dso_project, paths): runner = CliRunner() chdir(dso_project) # proj root - result = runner.invoke(cli, paths) + result = runner.invoke(lint_cli, paths) print(result.output) assert result.exit_code == 0 diff --git a/tests/test_pandocfilter.py b/tests/test_pandocfilter.py index 431d799..b0fe088 100644 --- a/tests/test_pandocfilter.py +++ b/tests/test_pandocfilter.py @@ -4,7 +4,8 @@ from click.testing import CliRunner -from dso.exec import _render_quarto, cli +from dso._quarto import render_quarto +from dso.cli import exec_cli from tests.conftest import TESTDATA @@ -59,8 +60,12 @@ def test_pandocfilter(quarto_stage): ) ) - _render_quarto( - quarto_stage / "src", quarto_stage / "report", before_script="", cwd=quarto_stage, with_pandocfilter=True + render_quarto( + quarto_stage / "src", + quarto_stage / "report", + before_script="", + cwd=quarto_stage, + with_pandocfilter=True, ) out_html = (quarto_stage / "report" / "quarto_stage.html").read_text() assert "Disclaimer" in out_html @@ -100,7 +105,7 @@ def test_override_config(quarto_stage): chdir(quarto_stage) stage_path = "." - result = runner.invoke(cli, ["quarto", stage_path]) + result = runner.invoke(exec_cli, ["quarto", stage_path]) assert result.exit_code == 0 out_html = (quarto_stage / "report" / "quarto_stage.html").read_text() diff --git a/tests/test_util.py b/tests/test_util.py index 6a178ad..630c464 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,6 +1,6 @@ import pytest -from dso._util import _find_in_parent, _git_list_files, _read_dot_dso_json, _update_dot_dso_json +from dso._util import _read_dot_dso_json, _update_dot_dso_json, find_in_parent, git_list_files @pytest.mark.parametrize( @@ -24,7 +24,7 @@ def test_find_in_parent(tmp_path, file_or_folder, recurse_barrier, expected): expected = tmp_path / expected if recurse_barrier is not None: recurse_barrier = tmp_path / recurse_barrier - assert _find_in_parent(subfolder, file_or_folder, recurse_barrier) == expected + assert find_in_parent(subfolder, file_or_folder, recurse_barrier) == expected def test_dot_dso_json(dso_project): @@ -44,7 +44,7 @@ def test_dot_dso_json(dso_project): def test_git_list_files(dso_project): - files = _git_list_files(dso_project) + files = git_list_files(dso_project) assert files == [ dso_project / x for x in [ diff --git a/tests/test_watermark.py b/tests/test_watermark.py index 8e24e5b..aa12aec 100644 --- a/tests/test_watermark.py +++ b/tests/test_watermark.py @@ -4,7 +4,8 @@ from click.testing import CliRunner from PIL import Image -from dso.watermark import PDFWatermarker, SVGWatermarker, Watermarker, cli +from dso._watermark import PDFWatermarker, SVGWatermarker, Watermarker +from dso.cli import watermark_cli from tests.conftest import TESTDATA @@ -66,6 +67,6 @@ def test_add_watermark_cli(tmp_path, params): test_image = _get_test_image(tmp_path, format="png", size=(500, 500)) test_image_out = tmp_path / "test_image_out.png" - result = runner.invoke(cli, [str(test_image), str(test_image_out), "--text", "test text", *params]) + result = runner.invoke(watermark_cli, [str(test_image), str(test_image_out), "--text", "test text", *params]) assert result.exit_code == 0 assert test_image_out.is_file()