From 021607ae6019b9e34a42b13934ab8c8248b920e5 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 15 Oct 2024 13:49:55 -0400 Subject: [PATCH 01/13] Rework HTML slate renderer. --- lablib/generators/slate_html.py | 311 +++++++++++------- lablib/lib/utils.py | 7 - lablib/renderers/slate_render.py | 142 +++++--- .../slates/slate_generic/slate_generic.html | 2 +- tests/test_renderers_slate.py | 148 +++++++++ 5 files changed, 439 insertions(+), 171 deletions(-) create mode 100644 tests/test_renderers_slate.py diff --git a/lablib/generators/slate_html.py b/lablib/generators/slate_html.py index 0a90134..0c4bc3f 100644 --- a/lablib/generators/slate_html.py +++ b/lablib/generators/slate_html.py @@ -1,49 +1,60 @@ from __future__ import annotations +import datetime +import os import shutil from typing import List, Dict -from dataclasses import dataclass, field from pathlib import Path from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By -from ..lib import utils +from ..lib import utils, imageio -@dataclass class SlateHtmlGenerator: """Class to generate a slate from a template. - Attention: - This class is functional but not yet tested. - - TODO: - * refactor into a plain python class - * add tests - Attributes: + slate_template_path: The path to the template. data: A dictionary containing the data to be formatted in the template. width: The width of the slate. height: The height of the slate. staging_dir: The directory where the slate will be staged. - slate_template_path: The path to the template. source_files: A list of source files. is_source_linear: A boolean to set whether the source files are linear. + + Raises: + ValueError: When the provided slate template path is invalid. """ + def __init__( + self, + data: Dict, + slate_template_path: str, + width: int = None, + height: int = None, + staging_dir: str = None, + source_files: List = None, + is_source_linear: bool = None + ): + self.data = data + self.width = width or 1920 + self.height = height or 1080 + self._staging_dir = staging_dir or utils.get_staging_dir() + self.source_files = source_files or [] + self.is_source_linear = is_source_linear if is_source_linear is not None else True + + try: + slate_template_path = slate_template_path + self._slate_template_path = Path(slate_template_path).resolve().as_posix() + + except Exception as error: + raise ValueError( + "Could not load slate template" + f" from {slate_template_path}" + ) from error - data: Dict = field(default_factory=dict) - width: int = 1920 - height: int = 1080 - staging_dir: str = None - slate_template_path: str = None - source_files: List = field(default_factory=list) - is_source_linear: bool = True - - def __post_init__(self): - if not self.staging_dir: - self.staging_dir = utils.get_staging_dir() self._thumbs = [] self._charts = [] self._thumb_class_name: str = "thumb" @@ -52,8 +63,10 @@ def __post_init__(self): self._slate_staged_path: str = None self._slate_computed: str = None self._slate_base_image_path: str = None - self._remove_missing_parents: bool = True - self._slate_base_name = "slate_base.png" + self.remove_missing_parents: bool = True + self.slate_base_name = "slate_base" + self.slate_extension = "png" + options = Options() # THIS WILL NEED TO BE SWITCHED TO NEW MODE, BUT THERE ARE BUGS. # WE SHOULD BE FINE FOR A COUPLE OF YEARS UNTIL DEPRECATION. @@ -71,101 +84,157 @@ def __post_init__(self): options.add_experimental_option("excludeSwitches", ["enable-logging"]) self._driver = webdriver.Chrome(options=options) - def get_staging_dir(self) -> str: - return self.staging_dir + @property + def staging_dir(self) -> str: + """Return the path to the staging directory. - def get_thumb_placeholder(self) -> str: - self._driver.get(self._slate_staged_path) - thumb_placeholder = self._driver.find_elements( - By.CLASS_NAME, self._thumb_class_name - )[0] - src = thumb_placeholder.get_attribute("src").replace("file:///", "") - return src + Returns: + str: The path to the staging directory. + """ + return self._staging_dir - def set_slate_base_name(self, name: str) -> None: - self._slate_base_name = "{}.png".format(name) + @property + def slate_filename(self) -> str: + """Return the slate filename. - def set_remove_missing_parent(self, remove: bool = True) -> None: - self._remove_missing_parents = remove + Returns: + str: The slate filename. + """ + return f"{self.slate_base_name}.{self.slate_extension}" - def set_linear_working_space(self, is_linear: bool) -> None: - self.is_source_linear = is_linear + @property + def template_path(self): + """Return the slate template path. - def set_source_files(self, files: List) -> None: - self.source_files = files + Returns: + str: The slate template path. + """ + return self._slate_template_path - def set_template_path(self, path: str) -> None: - self.slate_template_path = Path(path).resolve().as_posix() + @template_path.setter + def template_path(self, path: str) -> None: + """Set the slate template path. - def set_data(self, data: Dict) -> None: - self.data = data + Arguments: + path (str): The new path to the slate template path. + """ + self._slate_template_path = Path(path).resolve().as_posix() def set_size(self, width: int, height: int) -> None: + """Set the slate resolution. + + Args: + width (int): The width of the slate. + height (int): The height of the slate. + """ self.width = width self.height = height - def set_thumb_class_name(self, name: str) -> None: - self._thumb_class_name = name - - def set_chart_class_name(self, name: str) -> None: - self._chart_class_name = name + def _stage_slate(self) -> str: + """Prepare staging content for slate generation. - def set_viewport_size(self, width: int, height: int) -> None: - window_size = self._driver.execute_script( - "return [window.outerWidth - window.innerWidth + arguments[0]," - "window.outerHeight - window.innerHeight + arguments[1]];", - width, - height, - ) - self._driver.set_window_size(*window_size) + Returns: + str: The path to the prepped staging directory. - def stage_slate(self) -> str: - if not self.staging_dir: - raise ValueError("Missing staging dir!") - if not self.slate_template_path: - raise ValueError("Missing slate template path!") - slate_path = Path(self.slate_template_path).resolve() + Raises: + ValueError: When the provided template path is invalid. + """ + slate_path = Path(self.template_path).resolve() slate_dir = slate_path.parent slate_name = slate_path.name slate_staging_dir = Path( self.staging_dir, self._template_staging_dirname ).resolve() slate_staged_path = Path(slate_staging_dir, slate_name).resolve() + + # Clear staging path directory shutil.rmtree(slate_staging_dir.as_posix(), ignore_errors=True) + + # Copy over template root directory shutil.copytree(src=slate_dir.as_posix(), dst=slate_staging_dir.as_posix()) + self._slate_staged_path = slate_staged_path.as_posix() return self._slate_staged_path - def format_slate(self) -> None: - if not self.data: - raise ValueError("Missing subst_data to format template!") + def _format_slate(self) -> None: + """Format template with generator data values. + + Raises: + ValueError: When the provided data is incomplete. + """ + now = datetime.datetime.now() + default_data = { + "dd": now.day, + "mmm": now.month, + "yyyy": now.year, + "frame_start": self.source_files[0].frame_number, + "frame_end": self.source_files[-1].frame_number, + "resolution_width": self.width, + "resolution_height": self.height, + } + formatted_dict = default_data.copy() + formatted_dict.update(self.data) # overides with provided data. + + # Read template content with open(self._slate_staged_path, "r+") as f: - formatted_slate = f.read().format_map(utils.format_dict(self.data)) + template_content = f.read() + + # Attempt to replace/format template with content. + try: + content = template_content.format(**formatted_dict) + except KeyError as error: + raise ValueError( + f"Key mismatch, cannot fill template: {error}" + ) from error + + # Override template file content with formatted data. + with open(self._slate_staged_path, "w+") as f: f.seek(0) - f.write(formatted_slate) + f.write(content) f.truncate() self._driver.get(self._slate_staged_path) - elements = self._driver.find_elements( - By.XPATH, - "//*[contains(text(),'{}')]".format(utils.format_dict._placeholder), - ) - for el in elements: - self._driver.execute_script( - "var element = arguments[0];\n" "element.style.display = 'none';", el - ) - if self._remove_missing_parents: - parent = el.find_element(By.XPATH, "..") - self._driver.execute_script( - "var element = arguments[0];\n" "element.style.display = 'none';", - parent, - ) + + # TODO: Revisit this. + # Currently this function will fail with a KeyError + # if any data expected by the template is missing from + # the data dict. + # + # The code below turns this into a silent fails where + # missig fields are hidden from the resulting slate. +# elements = self._driver.find_elements( +# By.XPATH, +# "//*[contains(text(),'{}')]".format("**MISSING"), +# ) +# for el in elements: +# self._driver.execute_script( +# "var element = arguments[0];\n" "element.style.display = 'none';", el +# ) +# if self._remove_missing_parents: +# parent = el.find_element(By.XPATH, "..") +# self._driver.execute_script( +# "var element = arguments[0];\n" "element.style.display = 'none';", +# parent, +# ) with open(self._slate_staged_path, "w") as f: f.write(self._driver.page_source) - def setup_base_slate(self) -> str: + def _setup_base_slate(self) -> str: + """Setup the slate template in selenium. + + Returns: + str: The slate final destination path. + """ self._driver.get(self._slate_staged_path) - self.set_viewport_size(self.width, self.height) + + window_size = self._driver.execute_script( + "return [window.outerWidth - window.innerWidth + arguments[0]," + "window.outerHeight - window.innerHeight + arguments[1]];", + self.width, + self.height, + ) + self._driver.set_window_size(*window_size) + thumbs = self._driver.find_elements(By.CLASS_NAME, self._thumb_class_name) for thumb in thumbs: src_path = thumb.get_attribute("src") @@ -180,15 +249,14 @@ def setup_base_slate(self) -> str: ), thumb, ) - self._thumbs.append( - utils.ImageInfo( - filename=src_path.replace("file:///", ""), - origin_x=thumb.location["x"], - origin_y=thumb.location["y"], - width=thumb.size["width"], - height=thumb_height, - ) - ) + + path = src_path.replace("file:///", "") + thumb_image = imageio.ImageInfo(path=path) + thumb_image.origin_x = thumb.location["x"] + thumb_image.origin_y = thumb.location["y"] + thumb_image.width = thumb.size["width"] + thumb_image.height = thumb_height + self._thumbs.append(thumb_image) for thumb in thumbs: self._driver.execute_script( @@ -201,15 +269,13 @@ def setup_base_slate(self) -> str: for chart in charts: src_path = chart.get_attribute("src") if src_path: - self._charts.append( - utils.ImageInfo( - filename=src_path.replace("file:///", ""), - origin_x=chart.location["x"], - origin_y=chart.location["y"], - width=chart.size["width"], - height=chart.size["height"], - ) - ) + path=src_path.replace("file:///", "") + chart_image = imageio.ImageInfo(path=path) + chart_image.origin_x = chart.location["x"] + chart_image.origin_y = chart.location["y"] + chart_image.width = chart.size["width"] + chart_image.height = chart.size["height"] + self._charts.append(chart_image) for chart in charts: self._driver.execute_script( @@ -219,34 +285,37 @@ def setup_base_slate(self) -> str: ) template_staged_path = Path(self._slate_staged_path).resolve().parent - slate_base_path = Path(template_staged_path, self._slate_base_name).resolve() + slate_base_path = Path(template_staged_path, self.slate_filename).resolve() self._driver.save_screenshot(slate_base_path.as_posix()) self._driver.quit() self._slate_base_image_path = slate_base_path return slate_base_path - def set_thumbnail_sources(self) -> None: + def _set_thumbnail_sources(self) -> None: + """Set thumbnail sources before processing slate. + """ thumb_steps = int(len(self.source_files) / (len(self._thumbs) + 1)) for i, t in enumerate(self._thumbs): - self._thumbs[i].filename = ( - Path(self.source_files[thumb_steps * (i + 1)]).resolve().as_posix() + src_file = self.source_files[thumb_steps * (i + 1)] + src_file_path = src_file.filepath + self._thumbs[i].path = ( + Path(src_file_path).resolve().as_posix() ) def create_base_slate(self) -> None: - self.stage_slate() - self.format_slate() - # thumb_info = utils.read_image_info(self.get_thumb_placeholder()) - # thumb_cmd = [ - # "oiiotool", - # "-i", thumb_info.filename, - # "-resize", "{}x{}".format(self.width, self.height), - # "-o", thumb_info.filename - # ] - # subprocess.run(thumb_cmd) - self.setup_base_slate() - self.set_thumbnail_sources() + """Prepare and create base slate. + """ + self._stage_slate() + self._format_slate() + self._setup_base_slate() + self._set_thumbnail_sources() def get_oiiotool_cmd(self) -> List: + """ Get the oiiotool command to run for slate generation. + + Returns: + list: The oiiotool command to run. + """ label = "base" cmd = [ "-i", @@ -267,7 +336,7 @@ def get_oiiotool_cmd(self) -> List: label, ] for thumb in self._thumbs: - cmd.extend(["-i", thumb.filename]) + cmd.extend(["-i", thumb.path]) if not self.is_source_linear: cmd.extend(["--colorconvert", "sRGB", "linear"]) @@ -291,7 +360,7 @@ def get_oiiotool_cmd(self) -> List: cmd.extend( [ "-i", - chart.filename, + chart.path, "--colorconvert", "sRGB", "linear", diff --git a/lablib/lib/utils.py b/lablib/lib/utils.py index 4b89cef..cb89b91 100644 --- a/lablib/lib/utils.py +++ b/lablib/lib/utils.py @@ -193,13 +193,6 @@ def call_ffprobe(filepath: str | Path) -> dict: return result -class format_dict(dict): - _placeholder = "**MISSING**" - - def __missing__(self, key) -> str: - return self._placeholder - - def offset_timecode(tc: str, frame_offset: int = None, fps: float = None) -> str: if not frame_offset: frame_offset = -1 diff --git a/lablib/renderers/slate_render.py b/lablib/renderers/slate_render.py index 0e68c9e..6711b30 100644 --- a/lablib/renderers/slate_render.py +++ b/lablib/renderers/slate_render.py @@ -1,5 +1,4 @@ from __future__ import annotations -from dataclasses import dataclass import subprocess import shutil @@ -7,63 +6,121 @@ from pathlib import Path -from ..lib.utils import call_iinfo, offset_timecode +from ..lib.utils import call_iinfo, call_cmd, offset_timecode from ..generators import SlateHtmlGenerator from ..lib import SequenceInfo +from .basic import BasicRenderer -@dataclass -class SlateRenderer: +class SlateRenderer(BasicRenderer): """Class for rendering slates. - Attention: - This class is functional but not yet tested. + .. admonition:: Example - TODO: - * refactor into a plain python class - * add tests + .. code-block:: python + + # render slate image to 1080p + slate_generator = SlateHtmlGenerator( + # data used to fill up the slate template + { + "project": {"name": "test_project"}, + "intent": {"value": "test_intent"}, + "task": {"short": "test_task"}, + "asset": "test_asset", + "comment": "some random comment", + "scope": "test_scope", + "@version": "123", + }, + "/templates/slates/slate_generic/slate_generic.html", + ) + rend = SlateRenderer( + slate_generator, + SequenceInfo.scan("resources/public/plateMain/v002")[0], + ) + rend.render(debug=True) """ - slate_proc: SlateHtmlGenerator = None - source_sequence: SequenceInfo = None - dest: str = None + def __init__( + self, + slate_generator: SlateHtmlGenerator, + source_sequence: SequenceInfo, + destination: str = None # default prepend to the source sequence + ): + self._slate_proc = slate_generator + self._dest = None # default destination + self._forced_dest = destination # explicit destination + + self._source_sequence = None + self.source_sequence = source_sequence - def __post_init__(self) -> None: self._thumbs: List = None - self._debug: bool = False self._command: List = [] - if self.source_sequence: - self.set_source_sequence(self.source_sequence) - self.slate_proc.source_files = self.source_sequence.frames - if self.dest: - self.set_destination(self.dest) - def set_slate_processor(self, processor: SlateHtmlGenerator) -> None: - self.slate_proc = processor + @property + def slate_generator(self) -> SlateHtmlGenerator: + """ + Returns: + SlateHtmlGenerator: The generator associated to the renderer. + """ + return self._slate_proc - def set_debug(self, debug: bool) -> None: - self._debug = debug + @slate_generator.setter + def slate_generator(self, generator: SlateHtmlGenerator) -> None: + """ + Args: + generator (SlateHtmlGenerator): The new generator for the renderer. + """ + self._slate_proc = generator + self._slate_proc.source_files = self._source_sequence.frames - def set_source_sequence(self, source_sequence: SequenceInfo) -> None: - self.source_sequence = source_sequence - head, frame, tail = source_sequence._get_file_splits(source_sequence.frames[0]) - self.dest = f"{head}{str(int(frame) - 1).zfill(source_sequence.padding)}{tail}" # noqa + @property + def source_sequence(self) -> SequenceInfo: + """Return the source sequence. + + Returns: + SequenceInfo. The source sequence. + """ + return self._source_sequence + + @source_sequence.setter + def source_sequence(self, source_sequence: SequenceInfo) -> None: + """Set new source sequence. + """ + self._source_sequence = source_sequence + self._slate_proc.source_files = self._source_sequence.frames - def set_destination(self, dest: str) -> None: - self.dest = dest + first_frame = self._source_sequence.frames[0] + frame_number = first_frame.frame_number + slate_frame = str(frame_number - 1).zfill(source_sequence.padding) + ext = first_frame.extension + head, _, __ = first_frame.filepath.rsplit(".", 3) - def render(self) -> None: - iinfo_metadata = call_iinfo(self.source_sequence.frames[0]) + self._dest = f"{head}.{slate_frame}{ext}" + + @property + def destination(self): + """ + Returns: + str: The renderer destination. + """ + return self._forced_dest or self._dest + + def render(self, debug=False) -> None: + """Render the slate sequence. + + Arguments: + debug (Optional[bool]): Whether to increase log verbosity. + """ + first_frame = self.source_sequence.frames[0] timecode = offset_timecode( - tc=iinfo_metadata["timecode"], + tc=first_frame.timecode, frame_offset=-1, - fps=iinfo_metadata["fps"], + fps=first_frame.fps, ) - self.slate_proc.create_base_slate() - if not self.slate_proc: - raise ValueError("Missing valid SlateHtmlGenerator!") + self._slate_proc.create_base_slate() + cmd = ["oiiotool"] - cmd.extend(self.slate_proc.get_oiiotool_cmd()) + cmd.extend(self._slate_proc.get_oiiotool_cmd()) cmd.extend( [ "--ch", @@ -73,11 +130,12 @@ def render(self) -> None: f"""'{timecode.replace('"', "")}'""", ] ) - if self._debug: + if debug: cmd.extend(["--debug", "-v"]) - cmd.extend(["-o", self.dest]) - self._command = cmd - subprocess.run(cmd) - slate_base_image_path = Path(self.slate_proc._slate_base_image_path).resolve() + + cmd.extend(["-o", self.destination]) + call_cmd(cmd) + + slate_base_image_path = Path(self._slate_proc._slate_base_image_path).resolve() slate_base_image_path.unlink() shutil.rmtree(slate_base_image_path.parent) diff --git a/templates/slates/slate_generic/slate_generic.html b/templates/slates/slate_generic/slate_generic.html index 90b38d9..c7d6cf0 100644 --- a/templates/slates/slate_generic/slate_generic.html +++ b/templates/slates/slate_generic/slate_generic.html @@ -14,7 +14,7 @@
Length:
-
{frameStart} - {frameEnd}
+
{frame_start} - {frame_end}
Resolution:
diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py new file mode 100644 index 0000000..3dc1394 --- /dev/null +++ b/tests/test_renderers_slate.py @@ -0,0 +1,148 @@ +import os +import pathlib +import tempfile +import shutil + +import pytest + +from lablib.lib import SequenceInfo +from lablib.generators import SlateHtmlGenerator +from lablib.renderers import SlateRenderer + + +SLATE_TEMPLATE_FILE = os.path.join( + os.path.dirname(__file__), + "..", + "templates", + "slates", + "slate_generic", + "slate_generic.html" +) +DEFAULT_SEQUENCE_PATH = "resources/public/plateMain/v002" +DEFAULT_SEQUENCE = SequenceInfo.scan(DEFAULT_SEQUENCE_PATH)[0] + + +def _run_slate_renderer( + generator: SlateHtmlGenerator, + sequence: SequenceInfo = None, + output_path: str = None, + ): + """ Run a renderer associated with the provided + generator and source sequence. + """ + sequence = sequence or DEFAULT_SEQUENCE + renderer = SlateRenderer( + generator, + sequence, + destination=output_path, + ) + + renderer.render() + + +@pytest.fixture() +def source_dir(): + """ Prepare are clean source sequence + and after each tests. + """ + # Prepare a temporary directory with a copy of default sequence + temp_dir = tempfile.mkdtemp() + for img in DEFAULT_SEQUENCE.frames: + new_path = pathlib.Path(temp_dir) / img.filename + + try: + new_path.symlink_to(img.path) + except OSError: + shutil.copy(img.filepath, new_path) + + yield temp_dir + + # Remove all temporary directory content. + shutil.rmtree(temp_dir) + + +def test_Slaterenderer_missing_keys(): + """ An Exception should raise if any key defined in the + template but missing in the provided data. + """ + with pytest.raises(ValueError): + generator = SlateHtmlGenerator( + {"missing": "data"}, + SLATE_TEMPLATE_FILE, + ) + _run_slate_renderer(generator) + + +def test_Slaterenderer_default(source_dir): + """ Ensure a valid HD slate is generated from default. + """ + source_sequence = SequenceInfo.scan(source_dir)[0] + generator = SlateHtmlGenerator( + { + "project": {"name": "test_project"}, + "intent": {"value": "test_intent"}, + "task": {"short": "test_task"}, + "asset": "test_asset", + "comment": "some random comment", + "scope": "test_scope", + "@version": "123", + }, + SLATE_TEMPLATE_FILE, + ) + _run_slate_renderer(generator, sequence=source_sequence) + + edited_sequence = SequenceInfo.scan(source_dir)[0] + slate_frame = edited_sequence.frames[0] + + assert len(edited_sequence.frames) == len(source_sequence.frames) + 1 + assert slate_frame.width == 1920 + assert slate_frame.height == 1080 + + +def test_Slaterenderer_4K(source_dir): + """ Ensure a valid 4K slate. + """ + source_sequence = SequenceInfo.scan(source_dir)[0] + generator = SlateHtmlGenerator( + { + "project": {"name": "test_project"}, + "intent": {"value": "test_intent"}, + "task": {"short": "test_task"}, + "asset": "test_asset", + "comment": "some random comment", + "scope": "test_scope", + "@version": "123", + }, + SLATE_TEMPLATE_FILE, + width=4096, + height=2048, + ) + _run_slate_renderer(generator, sequence=source_sequence) + + edited_sequence = SequenceInfo.scan(source_dir)[0] + slate_frame = edited_sequence.frames[0] + + assert len(edited_sequence.frames) == len(source_sequence.frames) + 1 + assert slate_frame.width == 4096 + assert slate_frame.height == 2048 + + +def test_Slaterenderer_explicit_output(source_dir): + """ Ensure a valid HD slate can be generated to a predefined output. + """ + expected_output = pathlib.Path(source_dir) / "output.exr" + generator = SlateHtmlGenerator( + { + "project": {"name": "test_project"}, + "intent": {"value": "test_intent"}, + "task": {"short": "test_task"}, + "asset": "test_asset", + "comment": "some random comment", + "scope": "test_scope", + "@version": "123", + }, + SLATE_TEMPLATE_FILE, + ) + _run_slate_renderer(generator, output_path=expected_output) + + assert os.path.exists(expected_output) From 0ffff06c4b63d21c449312d0ca7c1dd9061f4d58 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 15 Oct 2024 14:19:08 -0400 Subject: [PATCH 02/13] Disable tests on GH. --- tests/test_renderers_slate.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py index 3dc1394..6e14eb1 100644 --- a/tests/test_renderers_slate.py +++ b/tests/test_renderers_slate.py @@ -19,7 +19,8 @@ "slate_generic.html" ) DEFAULT_SEQUENCE_PATH = "resources/public/plateMain/v002" -DEFAULT_SEQUENCE = SequenceInfo.scan(DEFAULT_SEQUENCE_PATH)[0] +DEFAULT_SEQUENCE = SequenceInfo.scan(DEFAULT_SEQUENCE_PATH)[0] +IS_RUNNING_ON_GH_ACTION = bool(os.getenv("GITHUB_WORKFLOW")) def _run_slate_renderer( @@ -61,6 +62,7 @@ def source_dir(): shutil.rmtree(temp_dir) +@pytest.mark.skipif(IS_RUNNING_ON_GH_ACTION, reason="cause access violation on GH") def test_Slaterenderer_missing_keys(): """ An Exception should raise if any key defined in the template but missing in the provided data. @@ -73,6 +75,7 @@ def test_Slaterenderer_missing_keys(): _run_slate_renderer(generator) +@pytest.mark.skipif(IS_RUNNING_ON_GH_ACTION, reason="cause access violation on GH") def test_Slaterenderer_default(source_dir): """ Ensure a valid HD slate is generated from default. """ @@ -99,6 +102,7 @@ def test_Slaterenderer_default(source_dir): assert slate_frame.height == 1080 +@pytest.mark.skipif(IS_RUNNING_ON_GH_ACTION, reason="cause access violation on GH") def test_Slaterenderer_4K(source_dir): """ Ensure a valid 4K slate. """ @@ -127,6 +131,7 @@ def test_Slaterenderer_4K(source_dir): assert slate_frame.height == 2048 +@pytest.mark.skipif(IS_RUNNING_ON_GH_ACTION, reason="cause access violation on GH") def test_Slaterenderer_explicit_output(source_dir): """ Ensure a valid HD slate can be generated to a predefined output. """ From 7485a423ea47066ec7b9c9f8e4dfce4f17135db9 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 15 Oct 2024 16:35:45 -0400 Subject: [PATCH 03/13] Fix GH unit tests. --- tests/test_renderers_slate.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py index 6e14eb1..131e4a7 100644 --- a/tests/test_renderers_slate.py +++ b/tests/test_renderers_slate.py @@ -20,7 +20,6 @@ ) DEFAULT_SEQUENCE_PATH = "resources/public/plateMain/v002" DEFAULT_SEQUENCE = SequenceInfo.scan(DEFAULT_SEQUENCE_PATH)[0] -IS_RUNNING_ON_GH_ACTION = bool(os.getenv("GITHUB_WORKFLOW")) def _run_slate_renderer( @@ -61,8 +60,6 @@ def source_dir(): # Remove all temporary directory content. shutil.rmtree(temp_dir) - -@pytest.mark.skipif(IS_RUNNING_ON_GH_ACTION, reason="cause access violation on GH") def test_Slaterenderer_missing_keys(): """ An Exception should raise if any key defined in the template but missing in the provided data. @@ -75,7 +72,6 @@ def test_Slaterenderer_missing_keys(): _run_slate_renderer(generator) -@pytest.mark.skipif(IS_RUNNING_ON_GH_ACTION, reason="cause access violation on GH") def test_Slaterenderer_default(source_dir): """ Ensure a valid HD slate is generated from default. """ @@ -102,7 +98,6 @@ def test_Slaterenderer_default(source_dir): assert slate_frame.height == 1080 -@pytest.mark.skipif(IS_RUNNING_ON_GH_ACTION, reason="cause access violation on GH") def test_Slaterenderer_4K(source_dir): """ Ensure a valid 4K slate. """ @@ -131,7 +126,6 @@ def test_Slaterenderer_4K(source_dir): assert slate_frame.height == 2048 -@pytest.mark.skipif(IS_RUNNING_ON_GH_ACTION, reason="cause access violation on GH") def test_Slaterenderer_explicit_output(source_dir): """ Ensure a valid HD slate can be generated to a predefined output. """ From 44527f202e8ca2137fb52f3f0dda1441e6cd5d00 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 16 Oct 2024 10:47:59 -0400 Subject: [PATCH 04/13] Adjust install LabLib --- .github/workflows/run_all_tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index f1570b6..fa25969 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -36,8 +36,6 @@ jobs: - name: install LabLib run: | - $env:POETRY_HOME="$repo_root\.poetry" - .\.poetry\bin\poetry.exe lock .\start.ps1 install - name: get dependencies From 6e0563b14ef8b32f3473e27284c843abd94b667d Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 16 Oct 2024 11:02:43 -0400 Subject: [PATCH 05/13] Attempt to fix GH tests for slate. --- tests/test_renderers_slate.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py index 131e4a7..7ede471 100644 --- a/tests/test_renderers_slate.py +++ b/tests/test_renderers_slate.py @@ -46,14 +46,10 @@ def source_dir(): and after each tests. """ # Prepare a temporary directory with a copy of default sequence - temp_dir = tempfile.mkdtemp() + temp_dir = tempfile.mkdtemp(dir=os.getcwd()) for img in DEFAULT_SEQUENCE.frames: new_path = pathlib.Path(temp_dir) / img.filename - - try: - new_path.symlink_to(img.path) - except OSError: - shutil.copy(img.filepath, new_path) + shutil.copy(img.filepath, new_path) yield temp_dir From 932e690b655c1672102d73e52f2dba770a613643 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 16 Oct 2024 12:36:53 -0400 Subject: [PATCH 06/13] Implement slate_fill_mode for SlateHtmlProcessor. --- lablib/generators/__init__.py | 3 +- lablib/generators/slate_html.py | 100 +++++++++++------- .../slates/slate_generic/slate_generic.html | 6 +- tests/test_renderers_slate.py | 53 ++++++++-- 4 files changed, 111 insertions(+), 51 deletions(-) diff --git a/lablib/generators/__init__.py b/lablib/generators/__init__.py index bc62da4..89500fc 100644 --- a/lablib/generators/__init__.py +++ b/lablib/generators/__init__.py @@ -1,9 +1,10 @@ """Generator module to be used by Processors and Renderers.""" from .ocio_config import OCIOConfigFileGenerator -from .slate_html import SlateHtmlGenerator +from .slate_html import SlateHtmlGenerator, SlateFillMode __all__ = [ "OCIOConfigFileGenerator", "SlateHtmlGenerator", + "SlateFillMode" ] diff --git a/lablib/generators/slate_html.py b/lablib/generators/slate_html.py index 0c4bc3f..5816910 100644 --- a/lablib/generators/slate_html.py +++ b/lablib/generators/slate_html.py @@ -1,7 +1,8 @@ from __future__ import annotations +import collections import datetime -import os +from enum import Enum import shutil from typing import List, Dict from pathlib import Path @@ -13,21 +14,31 @@ from ..lib import utils, imageio +class SlateFillMode(Enum): + """Slate fill mode (fill up template from data). + """ + RAISE_WHEN_MISSING = 1 # missing data from template : raise + HIDE_FIELD_WHEN_MISSING = 2 # missing data from template : hide field + + class SlateHtmlGenerator: """Class to generate a slate from a template. Attributes: - slate_template_path: The path to the template. - data: A dictionary containing the data to be formatted in the template. - width: The width of the slate. - height: The height of the slate. - staging_dir: The directory where the slate will be staged. - source_files: A list of source files. - is_source_linear: A boolean to set whether the source files are linear. + data (dict): A dictionary containing the data to be formatted in the template. + slate_template_path (str): The path to the template. + width (int): The width of the slate. + height (int): The height of the slate. + staging_dir (str): The directory where the slate will be staged. + source_files (list): A list of source files. + is_source_linear (bool): A boolean to set whether the source files are linear. + slate_fill_mode (SlateFillMode): The template fill mode. Raises: ValueError: When the provided slate template path is invalid. """ + _MISSING_FIELD = "**MISSING**" + def __init__( self, data: Dict, @@ -36,7 +47,8 @@ def __init__( height: int = None, staging_dir: str = None, source_files: List = None, - is_source_linear: bool = None + is_source_linear: bool = None, + slate_fill_mode: SlateFillMode = None ): self.data = data self.width = width or 1920 @@ -44,6 +56,7 @@ def __init__( self._staging_dir = staging_dir or utils.get_staging_dir() self.source_files = source_files or [] self.is_source_linear = is_source_linear if is_source_linear is not None else True + self._slate_fill_mode = slate_fill_mode or SlateFillMode.RAISE_WHEN_MISSING try: slate_template_path = slate_template_path @@ -156,6 +169,35 @@ def _stage_slate(self) -> str: self._slate_staged_path = slate_staged_path.as_posix() return self._slate_staged_path + def __format_template(self, template_content: str, values: dict) -> str: + """ + Args: + template_content (str): The template content to format. + values (dict): The values to use for formatting. + + Returns: + str: The formatted string. + + Raises: + ValueError: When some key is missing or the mode is unknown. + """ + # Attempt to replace/format template with content. + if self._slate_fill_mode == SlateFillMode.RAISE_WHEN_MISSING: + try: + return template_content.format(**values) + except KeyError as error: + raise ValueError( + f"Key mismatch, cannot fill template: {error}" + ) from error + + elif self._slate_fill_mode == SlateFillMode.HIDE_FIELD_WHEN_MISSING: + default_values = collections.defaultdict(lambda: self._MISSING_FIELD) + default_values.update(values) + return template_content.format_map(default_values) + + else: + raise ValueError(f"Unknown slate fill mode {self._slate_fill_mode}.") + def _format_slate(self) -> None: """Format template with generator data values. @@ -179,13 +221,7 @@ def _format_slate(self) -> None: with open(self._slate_staged_path, "r+") as f: template_content = f.read() - # Attempt to replace/format template with content. - try: - content = template_content.format(**formatted_dict) - except KeyError as error: - raise ValueError( - f"Key mismatch, cannot fill template: {error}" - ) from error + content = self.__format_template(template_content, formatted_dict) # Override template file content with formatted data. with open(self._slate_staged_path, "w+") as f: @@ -195,27 +231,17 @@ def _format_slate(self) -> None: self._driver.get(self._slate_staged_path) - # TODO: Revisit this. - # Currently this function will fail with a KeyError - # if any data expected by the template is missing from - # the data dict. - # - # The code below turns this into a silent fails where - # missig fields are hidden from the resulting slate. -# elements = self._driver.find_elements( -# By.XPATH, -# "//*[contains(text(),'{}')]".format("**MISSING"), -# ) -# for el in elements: -# self._driver.execute_script( -# "var element = arguments[0];\n" "element.style.display = 'none';", el -# ) -# if self._remove_missing_parents: -# parent = el.find_element(By.XPATH, "..") -# self._driver.execute_script( -# "var element = arguments[0];\n" "element.style.display = 'none';", -# parent, -# ) + # HIDE_FIELD_WHEN_MISSING mode + # The code below hide detected missing fields from the resulting slate. + elements = self._driver.find_elements( + By.XPATH, + "//*[contains(text(),'{}')]".format(self._MISSING_FIELD), + ) + for el in elements: + self._driver.execute_script( + "var element = arguments[0];\n" "element.style.display = 'none';", el + ) + with open(self._slate_staged_path, "w") as f: f.write(self._driver.page_source) diff --git a/templates/slates/slate_generic/slate_generic.html b/templates/slates/slate_generic/slate_generic.html index c7d6cf0..5f85f0e 100644 --- a/templates/slates/slate_generic/slate_generic.html +++ b/templates/slates/slate_generic/slate_generic.html @@ -6,8 +6,8 @@
-
{project[name]}
-
{asset}_{task[short]}_v{@version}
+
{project_name}
+
{asset}_{task_short}_v{@version}
Date:
{dd} {mmm} {yyyy}
@@ -26,7 +26,7 @@
Intent:
-
{intent[value]}
+
{intent}
Notes:
diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py index 7ede471..64aa71f 100644 --- a/tests/test_renderers_slate.py +++ b/tests/test_renderers_slate.py @@ -6,7 +6,7 @@ import pytest from lablib.lib import SequenceInfo -from lablib.generators import SlateHtmlGenerator +from lablib.generators import SlateHtmlGenerator, SlateFillMode from lablib.renderers import SlateRenderer @@ -56,6 +56,7 @@ def source_dir(): # Remove all temporary directory content. shutil.rmtree(temp_dir) + def test_Slaterenderer_missing_keys(): """ An Exception should raise if any key defined in the template but missing in the provided data. @@ -68,15 +69,47 @@ def test_Slaterenderer_missing_keys(): _run_slate_renderer(generator) +def test_Slaterenderer_unknown_mode(): + """ An Exception should raise if the provided slate mode is unknown. + """ + with pytest.raises(ValueError): + generator = SlateHtmlGenerator( + {"missing": "data"}, + SLATE_TEMPLATE_FILE, + slate_fill_mode="UNKNOWN MODE" + ) + _run_slate_renderer(generator) + + +def test_Slaterenderer_missing_keys_hide(source_dir): + """ Slate should go through even with missing data + when using HIDE_WHEN_MISSING mode. + """ + source_sequence = SequenceInfo.scan(source_dir)[0] + generator = SlateHtmlGenerator( + {"missing": "data"}, + SLATE_TEMPLATE_FILE, + slate_fill_mode=SlateFillMode.HIDE_FIELD_WHEN_MISSING + ) + _run_slate_renderer(generator, sequence=source_sequence) + + edited_sequence = SequenceInfo.scan(source_dir)[0] + slate_frame = edited_sequence.frames[0] + + assert len(edited_sequence.frames) == len(source_sequence.frames) + 1 + assert slate_frame.width == 1920 + assert slate_frame.height == 1080 + + def test_Slaterenderer_default(source_dir): """ Ensure a valid HD slate is generated from default. """ source_sequence = SequenceInfo.scan(source_dir)[0] generator = SlateHtmlGenerator( { - "project": {"name": "test_project"}, - "intent": {"value": "test_intent"}, - "task": {"short": "test_task"}, + "project_name": "test_project", + "intent": "test_intent", + "task_short": "test_task", "asset": "test_asset", "comment": "some random comment", "scope": "test_scope", @@ -100,9 +133,9 @@ def test_Slaterenderer_4K(source_dir): source_sequence = SequenceInfo.scan(source_dir)[0] generator = SlateHtmlGenerator( { - "project": {"name": "test_project"}, - "intent": {"value": "test_intent"}, - "task": {"short": "test_task"}, + "project_name": "test_project", + "intent": "test_intent", + "task_short": "test_task", "asset": "test_asset", "comment": "some random comment", "scope": "test_scope", @@ -128,9 +161,9 @@ def test_Slaterenderer_explicit_output(source_dir): expected_output = pathlib.Path(source_dir) / "output.exr" generator = SlateHtmlGenerator( { - "project": {"name": "test_project"}, - "intent": {"value": "test_intent"}, - "task": {"short": "test_task"}, + "project_name": "test_project", + "intent": "test_intent", + "task_short": "test_task", "asset": "test_asset", "comment": "some random comment", "scope": "test_scope", From d30b23c4d9d4f6512cc7b7d690436e72095aca66 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 22 Oct 2024 14:33:17 -0400 Subject: [PATCH 07/13] Adress feedback from PR. --- lablib/renderers/slate_render.py | 1 + tests/test_renderers_slate.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lablib/renderers/slate_render.py b/lablib/renderers/slate_render.py index 6711b30..c506496 100644 --- a/lablib/renderers/slate_render.py +++ b/lablib/renderers/slate_render.py @@ -133,6 +133,7 @@ def render(self, debug=False) -> None: if debug: cmd.extend(["--debug", "-v"]) + cmd.extend(["-resize", f"{self._slate_proc.width}x{self._slate_proc.height}"]) cmd.extend(["-o", self.destination]) call_cmd(cmd) diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py index 64aa71f..856018e 100644 --- a/tests/test_renderers_slate.py +++ b/tests/test_renderers_slate.py @@ -41,21 +41,18 @@ def _run_slate_renderer( @pytest.fixture() -def source_dir(): +def source_dir(test_staging_dir): """ Prepare are clean source sequence and after each tests. """ # Prepare a temporary directory with a copy of default sequence - temp_dir = tempfile.mkdtemp(dir=os.getcwd()) + temp_dir = tempfile.mkdtemp(dir=test_staging_dir) for img in DEFAULT_SEQUENCE.frames: new_path = pathlib.Path(temp_dir) / img.filename shutil.copy(img.filepath, new_path) yield temp_dir - # Remove all temporary directory content. - shutil.rmtree(temp_dir) - def test_Slaterenderer_missing_keys(): """ An Exception should raise if any key defined in the From 0f0f84e46e83bb41d43346a7d300e0690c1efb04 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Tue, 22 Oct 2024 17:36:05 -0400 Subject: [PATCH 08/13] Adjust temporary_directory naming. --- tests/test_renderers_slate.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py index 856018e..911b0e9 100644 --- a/tests/test_renderers_slate.py +++ b/tests/test_renderers_slate.py @@ -1,6 +1,5 @@ import os import pathlib -import tempfile import shutil import pytest @@ -41,12 +40,17 @@ def _run_slate_renderer( @pytest.fixture() -def source_dir(test_staging_dir): +def source_dir(test_staging_dir, request): """ Prepare are clean source sequence and after each tests. """ # Prepare a temporary directory with a copy of default sequence - temp_dir = tempfile.mkdtemp(dir=test_staging_dir) + temp_dir = os.path.join( + test_staging_dir, + request.node.name + ) + os.mkdir(temp_dir) + for img in DEFAULT_SEQUENCE.frames: new_path = pathlib.Path(temp_dir) / img.filename shutil.copy(img.filepath, new_path) From 65382af02f1c872865461011b9231223aadef4b2 Mon Sep 17 00:00:00 2001 From: Robin De Lillo Date: Wed, 23 Oct 2024 07:21:39 -0400 Subject: [PATCH 09/13] Update tests/test_renderers_slate.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jakub Ježek --- tests/test_renderers_slate.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py index 911b0e9..0cb57a6 100644 --- a/tests/test_renderers_slate.py +++ b/tests/test_renderers_slate.py @@ -45,11 +45,13 @@ def source_dir(test_staging_dir, request): and after each tests. """ # Prepare a temporary directory with a copy of default sequence - temp_dir = os.path.join( - test_staging_dir, - request.node.name - ) - os.mkdir(temp_dir) + temp_dir = test_staging_dir / request.node.name + + # remove folder with content + shutil.rmtree(temp_dir, ignore_errors=True) + + # create temp folder again + temp_dir.mkdir() for img in DEFAULT_SEQUENCE.frames: new_path = pathlib.Path(temp_dir) / img.filename From 52f23d9adbcf55c321aa7b24807526d6d08d9470 Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 23 Oct 2024 07:42:11 -0400 Subject: [PATCH 10/13] Add scale factor to force 100%. --- lablib/generators/slate_html.py | 1 + lablib/renderers/slate_render.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/lablib/generators/slate_html.py b/lablib/generators/slate_html.py index 5816910..637dbdf 100644 --- a/lablib/generators/slate_html.py +++ b/lablib/generators/slate_html.py @@ -94,6 +94,7 @@ def __init__( options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-extensions") options.add_argument("--disable-gpu") + options.add_argument(f"--force-device-scale-factor=1") options.add_experimental_option("excludeSwitches", ["enable-logging"]) self._driver = webdriver.Chrome(options=options) diff --git a/lablib/renderers/slate_render.py b/lablib/renderers/slate_render.py index c506496..6711b30 100644 --- a/lablib/renderers/slate_render.py +++ b/lablib/renderers/slate_render.py @@ -133,7 +133,6 @@ def render(self, debug=False) -> None: if debug: cmd.extend(["--debug", "-v"]) - cmd.extend(["-resize", f"{self._slate_proc.width}x{self._slate_proc.height}"]) cmd.extend(["-o", self.destination]) call_cmd(cmd) From 30846d4a80be5ff08e9b4e6ca1b2e3d4e832651e Mon Sep 17 00:00:00 2001 From: "robin@ynput.io" Date: Wed, 23 Oct 2024 07:56:58 -0400 Subject: [PATCH 11/13] Adjust typo. --- lablib/generators/slate_html.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lablib/generators/slate_html.py b/lablib/generators/slate_html.py index 637dbdf..f7673fb 100644 --- a/lablib/generators/slate_html.py +++ b/lablib/generators/slate_html.py @@ -94,7 +94,7 @@ def __init__( options.add_argument("--disable-dev-shm-usage") options.add_argument("--disable-extensions") options.add_argument("--disable-gpu") - options.add_argument(f"--force-device-scale-factor=1") + options.add_argument("--force-device-scale-factor=1") options.add_experimental_option("excludeSwitches", ["enable-logging"]) self._driver = webdriver.Chrome(options=options) From 0b442c6151aa9f51b8451854eadfd59ee4eef225 Mon Sep 17 00:00:00 2001 From: doerp Date: Sat, 26 Oct 2024 11:58:44 +0200 Subject: [PATCH 12/13] fixes oiiotool command --- lablib/renderers/slate_render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lablib/renderers/slate_render.py b/lablib/renderers/slate_render.py index 6711b30..30f4aa0 100644 --- a/lablib/renderers/slate_render.py +++ b/lablib/renderers/slate_render.py @@ -127,7 +127,7 @@ def render(self, debug=False) -> None: "R,G,B", "--attrib:type=timecode", "smpte:TimeCode", - f"""'{timecode.replace('"', "")}'""", + timecode, ] ) if debug: From 9fdfea7982c15c429b775cc18b06d63335633d55 Mon Sep 17 00:00:00 2001 From: doerp Date: Sat, 26 Oct 2024 12:13:15 +0200 Subject: [PATCH 13/13] assert timecode --- tests/test_renderers_slate.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_renderers_slate.py b/tests/test_renderers_slate.py index 0cb57a6..a831e6b 100644 --- a/tests/test_renderers_slate.py +++ b/tests/test_renderers_slate.py @@ -102,6 +102,7 @@ def test_Slaterenderer_missing_keys_hide(source_dir): assert len(edited_sequence.frames) == len(source_sequence.frames) + 1 assert slate_frame.width == 1920 assert slate_frame.height == 1080 + assert slate_frame.timecode == "02:10:04:16" def test_Slaterenderer_default(source_dir): @@ -128,6 +129,7 @@ def test_Slaterenderer_default(source_dir): assert len(edited_sequence.frames) == len(source_sequence.frames) + 1 assert slate_frame.width == 1920 assert slate_frame.height == 1080 + assert slate_frame.timecode == "02:10:04:16" def test_Slaterenderer_4K(source_dir): @@ -156,6 +158,7 @@ def test_Slaterenderer_4K(source_dir): assert len(edited_sequence.frames) == len(source_sequence.frames) + 1 assert slate_frame.width == 4096 assert slate_frame.height == 2048 + assert slate_frame.timecode == "02:10:04:16" def test_Slaterenderer_explicit_output(source_dir):