Update gitignores in stage template

CHANGELOG.md

@@ -10,6 +10,12 @@ and this project adheres to [Semantic Versioning][].
 
 ## [Unreleased]
 
+### Template updates
+
+-   Single `.gitignore` file per stage. Content of input/output/report folders is ignored. These folders
+    do not contain a separate `.gitignore` anymore. This means empty folders won't be tracked by git, but
+    this solves issues with dvc refusing to track the output folder because it is already partly tracked by git ([#73](https://github.com/Boehringer-Ingelheim/dso/pull/73)).
+
 ### Fixes
 
 -   Do not change the configuration of the root logger, only the `dso` logger. Changing the root logger
@@ -19,6 +25,8 @@ and this project adheres to [Semantic Versioning][].
 
 -   Paths in `params.in.yaml` files declared with `!path` can now be compiled to absolute instead of relative paths ([#78](https://github.com/Boehringer-Ingelheim/dso/pull/78)).
 -   Python API that mirrors `dso-r` functionality (e.g. to be used from Jupyter notebooks) ([#30](https://github.com/Boehringer-Ingelheim/dso/pull/30))
+-   `dso exec quarto` automatically creates an `output` directory in the stage if it doesn't exist. If it doesn't contain any file,
+    it will be removed again after completion ([#73](https://github.com/Boehringer-Ingelheim/dso/pull/73)).
 
 ## Documentation
 

src/dso/_get_config.py

@@ -102,7 +102,10 @@ def get_config(stage: str, *, all: bool = False, skip_compile: bool = False) ->
         # We want to include parameters mentioned in either `params`, `deps`, `outs`.
         # The parameters in `deps`/`outs` are encapsulated in `${ <param> }`
         is_matrix_stage = "matrix" in dvc_stage_config
-        keep_params = set(dvc_stage_config.get("params", []))
+        if (params := dvc_stage_config.get("params", [])) is None:
+            keep_params = set()
+        else:
+            keep_params = set(params)
         dvc_param_pat = re.compile(r"\$\{\s*(.*?)\s*\}")
         for dep in dvc_stage_config.get("deps", []):
             if match := dvc_param_pat.findall(dep):

src/dso/_util.py

@@ -5,6 +5,7 @@
 import subprocess
 import sys
 from collections.abc import Sequence
+from contextlib import contextmanager
 from functools import cache
 from importlib import resources
 from os import environ
@@ -135,9 +136,11 @@ def _traverse_template(curr_path, subdir):
         for p in curr_path.iterdir():
             if p.is_file():
                 name_rendered = Template(p.name).render(params)
-                target_file = target_dir / subdir / name_rendered
-                if not target_file.exists():
-                    _copy_with_render(p, target_file, params)
+                # this file is used for checking in empty folders in git.
+                if name_rendered != ".gitkeeptemplate":
+                    target_file = target_dir / subdir / name_rendered
+                    if not target_file.exists():
+                        _copy_with_render(p, target_file, params)
             else:
                 (target_dir / subdir / p.name).mkdir(exist_ok=True)
                 _traverse_template(p, subdir / p.name)
@@ -237,6 +240,20 @@ def _update_dot_dso_json(dir: Path, update_dict: dict):
         json.dump(config, f)
 
 
+@contextmanager
+def add_directory(dir: Path):
+    """Context manager that temporarily creates a directory and removes it again if it's empty"""
+    dir.mkdir(exist_ok=True)
+    try:
+        yield
+    finally:
+        try:
+            dir.rmdir()
+        except OSError:
+            # directory not empty
+            pass
+
+
 def check_ask_pre_commit(dir: Path):
     """
     Check if pre-commit hooks are installed and asks to install them

src/dso/_watermark.py

@@ -75,7 +75,7 @@ def _get_watermark_tile(self) -> Image.Image:
         img = Image.new("RGBA", self.tile_size, color=(255, 255, 255, 0))
 
         d = ImageDraw.Draw(img)
-        with resources.path(assets, "open_sans.ttf") as watermark_font:
+        with resources.open_binary(assets, "open_sans.ttf") as watermark_font:
             font = ImageFont.truetype(watermark_font, self.font_size)
 
         # Add text in top left corner

src/dso/cli/_exec.py

@@ -5,6 +5,7 @@
 from ruamel.yaml import YAML
 
 from dso._logging import log
+from dso._util import add_directory
 
 
 @click.command("quarto")
@@ -73,14 +74,15 @@ def dso_exec_quarto(stage: str, skip_compile: bool = True):
         except KeyError:
             pass
 
-    with quarto_config_yml(quarto_config, stage_dir / "src"):
-        render_quarto(
-            stage_dir / "src",
-            report_dir=stage_dir / "report",
-            before_script=before_script,
-            cwd=stage_dir,
-            with_pandocfilter="watermark" in quarto_config or "disclaimer" in quarto_config,
-        )
+    with add_directory(stage_dir / "output"):
+        with quarto_config_yml(quarto_config, stage_dir / "src"):
+            render_quarto(
+                stage_dir / "src",
+                report_dir=stage_dir / "report",
+                before_script=before_script,
+                cwd=stage_dir,
+                with_pandocfilter="watermark" in quarto_config or "disclaimer" in quarto_config,
+            )
 
 
 @click.group(name="exec")

src/dso/templates/.gitignore

@@ -0,0 +1 @@
+!*.gitkeep*

src/dso/templates/stage/bash/.gitignore

@@ -1,4 +1,7 @@
 /log
 /tmp
 /temp
-*.rmarkdown
+/input/*
+!/input/*.dvc
+/output/*
+/report/*

src/dso/templates/stage/bash/dvc.yaml

@@ -9,5 +9,6 @@ stages:
         bash -euo pipefail << EOF
 
         # add bash code here
+        mkdir -p output && echo "Hello World" > output/hello.txt
 
         EOF

src/dso/templates/stage/quarto/.gitignore

@@ -1,4 +1,7 @@
 /log
 /tmp
 /temp
-*.rmarkdown
+/input/*
+!/input/*.dvc
+/output/*
+/report/*

tests/conftest.py

@@ -38,6 +38,9 @@ def quarto_stage(dso_project) -> Path:
                 """\
                 ```{python}
                 print("Hello World!")
+                from dso import read_params, stage_here
+                read_params("quarto_stage")
+                (stage_here("output") / "hello.txt").touch()
                 ```
                 """
             )
@@ -55,6 +58,9 @@ def quarto_stage_empty_configs(quarto_stage) -> Path:
         f.write("\n")
     with (quarto_stage / "params.in.yaml").open("w") as f:
         f.write("\n")
+    # remove param from `dvc.yaml` because it's not in the empty config anymore
+    lines = [line for line in (quarto_stage / "dvc.yaml").read_text().splitlines() if "dso.quarto" not in line]
+    (quarto_stage / "dvc.yaml").write_text("\n".join(lines) + "\n")
     compile_all_configs([quarto_stage])
 
     return quarto_stage

tests/test_exec.py

@@ -24,6 +24,7 @@ def test_exec_quarto(quarto_stage, quiet, launch_dir):
     result = runner.invoke(dso_exec, ["quarto", stage_path])
     assert result.exit_code == 0
     assert (quarto_stage / "report" / "quarto_stage.html").is_file()
+    assert (quarto_stage / "output" / "hello.txt").is_file()
     assert "Hello World!" in (quarto_stage / "report" / "quarto_stage.html").read_text()