From 282d1720ae958bbe8833ac64d6295e658a28b438 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 12 Sep 2024 16:39:09 +0200 Subject: [PATCH 01/13] Add staging directory functions and configurations - Added functions to handle custom staging directories - Updated imports and removed deprecated code - Created a new module for staging directory handling --- client/ayon_core/pipeline/__init__.py | 27 +-- .../ayon_core/pipeline/publish/constants.py | 1 - client/ayon_core/pipeline/publish/lib.py | 177 ++++++-------- client/ayon_core/pipeline/stagingdir.py | 220 ++++++++++++++++++ client/ayon_core/pipeline/tempdir.py | 94 ++++++-- .../publish/collect_custom_staging_dir.py | 76 ------ .../plugins/publish/extract_burnin.py | 11 +- .../publish/extract_color_transcode.py | 15 +- .../plugins/publish/extract_review.py | 7 +- 9 files changed, 399 insertions(+), 229 deletions(-) create mode 100644 client/ayon_core/pipeline/stagingdir.py delete mode 100644 client/ayon_core/plugins/publish/collect_custom_staging_dir.py diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index 8fd00ee6b6..d5c3140d37 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -8,6 +8,10 @@ from .anatomy import Anatomy +from .tempdir import get_temp_dir + +from .stagingdir import get_staging_dir + from .create import ( BaseCreator, Creator, @@ -116,10 +120,12 @@ "AYON_CONTAINER_ID", "AYON_INSTANCE_ID", "HOST_WORKFILE_EXTENSIONS", - # --- Anatomy --- "Anatomy", - + # --- Temp dir --- + "get_temp_dir", + # --- Staging dir --- + "get_staging_dir", # --- Create --- "BaseCreator", "Creator", @@ -127,42 +133,34 @@ "HiddenCreator", "CreatedInstance", "CreatorError", - "CreatorError", - # - legacy creation "LegacyCreator", "legacy_create", - "discover_creator_plugins", "discover_legacy_creator_plugins", "register_creator_plugin", "deregister_creator_plugin", "register_creator_plugin_path", "deregister_creator_plugin_path", - # --- Load --- "HeroVersionType", "IncompatibleLoaderError", "LoaderPlugin", "ProductLoaderPlugin", - "discover_loader_plugins", "register_loader_plugin", "deregister_loader_plugin_path", "register_loader_plugin_path", "deregister_loader_plugin", - "load_container", "remove_container", "update_container", "switch_container", - "loaders_from_representation", "get_representation_path", "get_representation_context", "get_repres_contexts", - # --- Publish --- "PublishValidationError", "PublishXmlValidationError", @@ -170,50 +168,41 @@ "AYONPyblishPluginMixin", "OpenPypePyblishPluginMixin", "OptionalPyblishPluginMixin", - # --- Actions --- "LauncherAction", "InventoryAction", - "discover_launcher_actions", "register_launcher_action", "register_launcher_action_path", - "discover_inventory_actions", "register_inventory_action", "register_inventory_action_path", "deregister_inventory_action", "deregister_inventory_action_path", - # --- Process context --- "install_ayon_plugins", "install_openpype_plugins", "install_host", "uninstall_host", "is_installed", - "register_root", "registered_root", - "register_host", "registered_host", "deregister_host", "get_process_id", - "get_global_context", "get_current_context", "get_current_host_name", "get_current_project_name", "get_current_folder_path", "get_current_task_name", - # Workfile templates "discover_workfile_build_plugins", "register_workfile_build_plugin", "deregister_workfile_build_plugin", "register_workfile_build_plugin_path", "deregister_workfile_build_plugin_path", - # Backwards compatible function names "install", "uninstall", diff --git a/client/ayon_core/pipeline/publish/constants.py b/client/ayon_core/pipeline/publish/constants.py index 38f5ffef3f..5240628365 100644 --- a/client/ayon_core/pipeline/publish/constants.py +++ b/client/ayon_core/pipeline/publish/constants.py @@ -8,4 +8,3 @@ DEFAULT_PUBLISH_TEMPLATE = "default" DEFAULT_HERO_PUBLISH_TEMPLATE = "default" -TRANSIENT_DIR_TEMPLATE = "default" diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 8b82622e4c..9cfcd3f71a 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -2,7 +2,6 @@ import sys import inspect import copy -import tempfile import xml.etree.ElementTree from typing import Optional, Union, List @@ -18,15 +17,11 @@ ) from ayon_core.settings import get_project_settings from ayon_core.addon import AddonsManager -from ayon_core.pipeline import ( - tempdir, - Anatomy -) +from ayon_core.pipeline import get_staging_dir from ayon_core.pipeline.plugin_discover import DiscoverResult from .constants import ( DEFAULT_PUBLISH_TEMPLATE, DEFAULT_HERO_PUBLISH_TEMPLATE, - TRANSIENT_DIR_TEMPLATE ) @@ -581,58 +576,6 @@ def context_plugin_should_run(plugin, context): return False -def get_instance_staging_dir(instance): - """Unified way how staging dir is stored and created on instances. - - First check if 'stagingDir' is already set in instance data. - In case there already is new tempdir will not be created. - - It also supports `AYON_TMPDIR`, so studio can define own temp - shared repository per project or even per more granular context. - Template formatting is supported also with optional keys. Folder is - created in case it doesn't exists. - - Available anatomy formatting keys: - - root[work | ] - - project[name | code] - - Note: - Staging dir does not have to be necessarily in tempdir so be careful - about its usage. - - Args: - instance (pyblish.lib.Instance): Instance for which we want to get - staging dir. - - Returns: - str: Path to staging dir of instance. - """ - staging_dir = instance.data.get('stagingDir') - if staging_dir: - return staging_dir - - anatomy = instance.context.data.get("anatomy") - - # get customized tempdir path from `AYON_TMPDIR` env var - custom_temp_dir = tempdir.create_custom_tempdir( - anatomy.project_name, anatomy) - - if custom_temp_dir: - staging_dir = os.path.normpath( - tempfile.mkdtemp( - prefix="pyblish_tmp_", - dir=custom_temp_dir - ) - ) - else: - staging_dir = os.path.normpath( - tempfile.mkdtemp(prefix="pyblish_tmp_") - ) - instance.data['stagingDir'] = staging_dir - - return staging_dir - - def get_publish_repre_path(instance, repre, only_published=False): """Get representation path that can be used for integration. @@ -685,6 +628,8 @@ def get_publish_repre_path(instance, repre, only_published=False): return None +# deprecated: backward compatibility only +# TODO: remove in the future def get_custom_staging_dir_info( project_name, host_name, @@ -694,67 +639,85 @@ def get_custom_staging_dir_info( product_name, project_settings=None, anatomy=None, - log=None + log=None, ): - """Checks profiles if context should use special custom dir as staging. + from ayon_core.pipeline.stagingdir import get_staging_dir_config + + tr_data = get_staging_dir_config( + host_name, + project_name, + task_type, + task_name, + product_type, + product_name, + project_settings=project_settings, + anatomy=anatomy, + log=log, + ) - Args: - project_name (str) - host_name (str) - product_type (str) - task_name (str) - task_type (str) - product_name (str) - project_settings(Dict[str, Any]): Prepared project settings. - anatomy (Dict[str, Any]) - log (Logger) (optional) + if not tr_data: + return None, None + + return tr_data["template"], tr_data["persistence"] + + +def get_instance_staging_dir(instance): + """Unified way how staging dir is stored and created on instances. + + First check if 'stagingDir' is already set in instance data. + In case there already is new tempdir will not be created. Returns: - (tuple) - Raises: - ValueError - if misconfigured template should be used + str: Path to staging dir """ - settings = project_settings or get_project_settings(project_name) - custom_staging_dir_profiles = (settings["core"] - ["tools"] - ["publish"] - ["custom_staging_dir_profiles"]) - if not custom_staging_dir_profiles: - return None, None + staging_dir = instance.data.get("stagingDir") - if not log: - log = Logger.get_logger("get_custom_staging_dir_info") + if staging_dir: + return staging_dir - filtering_criteria = { - "hosts": host_name, - "families": product_type, - "task_names": task_name, - "task_types": task_type, - "subsets": product_name - } - profile = filter_profiles(custom_staging_dir_profiles, - filtering_criteria, - logger=log) + anatomy_data = instance.data["anatomyData"] + formatting_data = copy.deepcopy(anatomy_data) - if not profile or not profile["active"]: - return None, None + product_type = instance.data["productType"] + product_name = instance.data["productName"] + + # context data based variables + project_entity = instance.context.data["projectEntity"] + folder_entity = instance.context.data["folderEntity"] + task_entity = instance.context.data["taskEntity"] + host_name = instance.context.data["hostName"] + project_settings = instance.context.data["project_settings"] + anatomy = instance.context.data["anatomy"] + current_file = instance.context.data.get("currentFile") + + # add current file as workfile name into formatting data + if current_file: + workfile = os.path.basename(current_file) + workfile_name, _ = os.path.splitext(workfile) + formatting_data["workfile_name"] = workfile_name + + dir_data = get_staging_dir( + host_name, + project_entity, + folder_entity, + task_entity, + product_type, + product_name, + anatomy, + project_settings=project_settings, + formatting_data=formatting_data, + ) - if not anatomy: - anatomy = Anatomy(project_name) + staging_dir_path = dir_data["stagingDir"] - template_name = profile["template_name"] or TRANSIENT_DIR_TEMPLATE + # TODO: not sure if this is necessary + # path might be already created by get_staging_dir + if not os.path.exists(staging_dir_path): + os.makedirs(staging_dir_path) - custom_staging_dir = anatomy.get_template_item( - "staging", template_name, "directory", default=None - ) - if custom_staging_dir is None: - raise ValueError(( - "Anatomy of project \"{}\" does not have set" - " \"{}\" template key!" - ).format(project_name, template_name)) - is_persistent = profile["custom_staging_dir_persistent"] + instance.data.update(dir_data) - return custom_staging_dir.template, is_persistent + return staging_dir_path def get_published_workfile_instance(context): diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py new file mode 100644 index 0000000000..e8fa1c4853 --- /dev/null +++ b/client/ayon_core/pipeline/stagingdir.py @@ -0,0 +1,220 @@ +from ayon_core.lib import Logger, filter_profiles, StringTemplate +from ayon_core.settings import get_project_settings +from .anatomy import Anatomy +from .tempdir import get_temp_dir +from ayon_core.pipeline.template_data import get_template_data + + +STAGING_DIR_TEMPLATES = "staging" + + +def get_staging_dir_config( + host_name, + project_name, + task_type, + task_name, + product_type, + product_name, + project_settings=None, + anatomy=None, + log=None, +): + """Get matching staging dir profile. + + Args: + host_name (str): Name of host. + project_name (str): Name of project. + task_type (str): Type of task. + task_name (str): Name of task. + product_type (str): Type of product. + product_name (str): Name of product. + project_settings(Dict[str, Any]): Prepared project settings. + anatomy (Dict[str, Any]) + log (Optional[logging.Logger]) + + Returns: + Dict or None: Data with directory template and is_persistent or None + Raises: + ValueError - if misconfigured template should be used + """ + settings = project_settings or get_project_settings(project_name) + + staging_dir_profiles = settings["core"]["tools"]["publish"][ + "custom_staging_dir_profiles" + ] + + if not staging_dir_profiles: + return None + + if not log: + log = Logger.get_logger("get_staging_dir_config") + + filtering_criteria = { + "hosts": host_name, + "task_types": task_type, + "task_names": task_name, + "product_types": product_type, + "product_names": product_name, + } + profile = filter_profiles( + staging_dir_profiles, filtering_criteria, logger=log) + + if not profile or not profile["active"]: + return None + + if not anatomy: + anatomy = Anatomy(project_name) + + # get template from template name + template_name = profile["template_name"] + _validate_template_name(project_name, template_name, anatomy) + + template = anatomy.templates[STAGING_DIR_TEMPLATES][template_name] + + if not template: + # template should always be found either from anatomy or from profile + raise ValueError( + "Staging dir profile is misconfigured! " + "No template was found for profile! " + "Check your project settings at: " + "'ayon+settings://core/tools/publish/custom_staging_dir_profiles'" + ) + + data_persistence = profile["custom_staging_dir_persistent"] + + return {"template": template, "persistence": data_persistence} + + +def _validate_template_name(project_name, template_name, anatomy): + """Check that staging dir section with appropriate template exist. + + Raises: + ValueError - if misconfigured template + """ + # TODO: only for backward compatibility of anatomy for older projects + if STAGING_DIR_TEMPLATES not in anatomy.templates: + raise ValueError( + ( + 'Anatomy of project "{}" does not have set' ' "{}" template section!' + ).format(project_name, template_name) + ) + + if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: + raise ValueError( + ( + 'Anatomy of project "{}" does not have set' + ' "{}" template key at Staging Dir section!' + ).format(project_name, template_name) + ) + + +def get_staging_dir( + host_name, + project_entity, + folder_entity, + task_entity, + product_type, + product_name, + anatomy, + project_settings=None, + **kwargs +): + """Get staging dir data. + + If `force_temp` is set, staging dir will be created as tempdir. + If `always_get_some_dir` is set, staging dir will be created as tempdir if + no staging dir profile is found. + If `prefix` or `suffix` is not set, default values will be used. + + Arguments: + host_name (str): Name of host. + project_entity (Dict[str, Any]): Project entity. + folder_entity (Dict[str, Any]): Folder entity. + task_entity (Dict[str, Any]): Task entity. + product_type (str): Type of product. + product_name (str): Name of product. + anatomy (ayon_core.pipeline.Anatomy): Anatomy object. + project_settings (Optional[Dict[str, Any]]): Prepared project settings. + **kwargs: Arbitrary keyword arguments. See below. + + Keyword Arguments: + force_temp (bool): If True, staging dir will be created as tempdir. + always_return_path (bool): If True, staging dir will be created as + tempdir if no staging dir profile is found. + prefix (str): Prefix for staging dir. + suffix (str): Suffix for staging dir. + formatting_data (Dict[str, Any]): Data for formatting staging dir + template. + + Returns: + Dict[str, Any]: Staging dir data + """ + + log = kwargs.get("log") or Logger.get_logger("get_staging_dir") + always_return_path = kwargs.get("always_return_path") + + # make sure always_return_path is set to true by default + if always_return_path is None: + always_return_path = True + + if kwargs.get("force_temp"): + return get_temp_dir( + project_name=project_entity["name"], + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ) + + # making fewer queries to database + ctx_data = get_template_data( + project_entity, folder_entity, task_entity, host_name + ) + # add roots to ctx_data + ctx_data["root"] = anatomy.roots + + # add additional data + ctx_data.update({ + "product": { + "type": product_type, + "name": product_name + }, + "host": host_name, + }) + + # add additional data from kwargs + if kwargs.get("formatting_data"): + ctx_data.update(kwargs.get("formatting_data")) + + # get staging dir config + staging_dir_config = get_staging_dir_config( + host_name, + project_entity["name"], + task_entity["type"], + task_entity["name"], + product_type, + product_name, + project_settings=project_settings, + anatomy=anatomy, + log=log, + ) + + # if no preset matching and always_get_some_dir is set, return tempdir + if not staging_dir_config and always_return_path: + return { + "stagingDir": get_temp_dir( + project_name=project_name, + anatomy=anatomy, + prefix=kwargs.get("prefix"), + suffix=kwargs.get("suffix"), + ), + "stagingDir_persistent": False, + } + elif not staging_dir_config: + return None + + return { + "stagingDir": StringTemplate.format_template( + staging_dir_config["template"], ctx_data + ), + "stagingDir_persistent": staging_dir_config["persistence"], + } diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 29d4659393..a6328135ee 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -3,11 +3,80 @@ """ import os +import tempfile +from pathlib import Path from ayon_core.lib import StringTemplate from ayon_core.pipeline import Anatomy -def create_custom_tempdir(project_name, anatomy=None): +def get_temp_dir( + project_name=None, anatomy=None, prefix=None, suffix=None, make_local=False +): + """Get temporary dir path. + + If `make_local` is set, tempdir will be created in local tempdir. + If `anatomy` is not set, default anatomy will be used. + If `prefix` or `suffix` is not set, default values will be used. + + It also supports `OPENPYPE_TMPDIR`, so studio can define own temp + shared repository per project or even per more granular context. + Template formatting is supported also with optional keys. Folder is + created in case it doesn't exists. + + Available anatomy formatting keys: + - root[work | ] + - project[name | code] + + Note: + Staging dir does not have to be necessarily in tempdir so be careful + about its usage. + + Args: + project_name (str)[optional]: Name of project. + anatomy (openpype.pipeline.Anatomy)[optional]: Anatomy object. + make_local (bool)[optional]: If True, temp dir will be created in + local tempdir. + suffix (str)[optional]: Suffix for tempdir. + prefix (str)[optional]: Prefix for tempdir. + + Returns: + str: Path to staging dir of instance. + """ + prefix = prefix or "ay_tmp_" + suffix = suffix or "" + + if make_local: + return _create_local_staging_dir(prefix, suffix) + + # make sure anatomy is set + if not anatomy: + anatomy = Anatomy(project_name) + + # get customized tempdir path from `OPENPYPE_TMPDIR` env var + custom_temp_dir = _create_custom_tempdir(anatomy.project_name, anatomy) + + return _create_local_staging_dir(prefix, suffix, custom_temp_dir) + + +def _create_local_staging_dir(prefix, suffix, dir=None): + """Create local staging dir + + Args: + prefix (str): prefix for tempdir + suffix (str): suffix for tempdir + + Returns: + str: path to tempdir + """ + # use pathlib for creating tempdir + staging_dir = Path(tempfile.mkdtemp( + prefix=prefix, suffix=suffix, dir=dir + )) + + return staging_dir.as_posix() + + +def _create_custom_tempdir(project_name, anatomy=None): """ Create custom tempdir Template path formatting is supporting: @@ -38,7 +107,7 @@ def create_custom_tempdir(project_name, anatomy=None): if anatomy is None: anatomy = Anatomy(project_name) # create base formate data - data = { + template_formatting_data = { "root": anatomy.roots, "project": { "name": anatomy.project_name, @@ -47,19 +116,14 @@ def create_custom_tempdir(project_name, anatomy=None): } # path is anatomy template custom_tempdir = StringTemplate.format_template( - env_tmpdir, data).normalized() + env_tmpdir, template_formatting_data) + + custom_tempdir_path = Path(custom_tempdir) else: # path is absolute - custom_tempdir = env_tmpdir - - # create the dir path if it doesn't exists - if not os.path.exists(custom_tempdir): - try: - # create it if it doesn't exists - os.makedirs(custom_tempdir) - except IOError as error: - raise IOError( - "Path couldn't be created: {}".format(error)) - - return custom_tempdir + custom_tempdir_path = Path(env_tmpdir) + + custom_tempdir_path.mkdir(parents=True, exist_ok=True) + + return custom_tempdir_path.as_posix() diff --git a/client/ayon_core/plugins/publish/collect_custom_staging_dir.py b/client/ayon_core/plugins/publish/collect_custom_staging_dir.py deleted file mode 100644 index 49c3a98dd2..0000000000 --- a/client/ayon_core/plugins/publish/collect_custom_staging_dir.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -Requires: - anatomy - - -Provides: - instance.data -> stagingDir (folder path) - -> stagingDir_persistent (bool) -""" -import copy -import os.path - -import pyblish.api - -from ayon_core.pipeline.publish.lib import get_custom_staging_dir_info - - -class CollectCustomStagingDir(pyblish.api.InstancePlugin): - """Looks through profiles if stagingDir should be persistent and in special - location. - - Transient staging dir could be useful in specific use cases where is - desirable to have temporary renders in specific, persistent folders, could - be on disks optimized for speed for example. - - It is studio responsibility to clean up obsolete folders with data. - - Location of the folder is configured in `project_anatomy/templates/others`. - ('transient' key is expected, with 'folder' key) - - Which family/task type/product is applicable is configured in: - `project_settings/global/tools/publish/custom_staging_dir_profiles` - - """ - label = "Collect Custom Staging Directory" - order = pyblish.api.CollectorOrder + 0.4990 - - template_key = "transient" - - def process(self, instance): - product_type = instance.data["productType"] - product_name = instance.data["productName"] - host_name = instance.context.data["hostName"] - project_name = instance.context.data["projectName"] - project_settings = instance.context.data["project_settings"] - anatomy = instance.context.data["anatomy"] - task = instance.data["anatomyData"].get("task", {}) - - transient_tml, is_persistent = get_custom_staging_dir_info( - project_name, - host_name, - product_type, - product_name, - task.get("name"), - task.get("type"), - project_settings=project_settings, - anatomy=anatomy, - log=self.log) - - if transient_tml: - anatomy_data = copy.deepcopy(instance.data["anatomyData"]) - anatomy_data["root"] = anatomy.roots - scene_name = instance.context.data.get("currentFile") - if scene_name: - anatomy_data["scene_name"] = os.path.basename(scene_name) - transient_dir = transient_tml.format(**anatomy_data) - instance.data["stagingDir"] = transient_dir - - instance.data["stagingDir_persistent"] = is_persistent - result_str = "Adding '{}' as".format(transient_dir) - else: - result_str = "Not adding" - - self.log.debug("{} custom staging dir for instance with '{}'".format( - result_str, product_type - )) diff --git a/client/ayon_core/plugins/publish/extract_burnin.py b/client/ayon_core/plugins/publish/extract_burnin.py index 58a032a030..72578d9dc0 100644 --- a/client/ayon_core/plugins/publish/extract_burnin.py +++ b/client/ayon_core/plugins/publish/extract_burnin.py @@ -9,11 +9,13 @@ import pyblish.api from ayon_core import resources, AYON_CORE_ROOT -from ayon_core.pipeline import publish +from ayon_core.pipeline import ( + publish, + get_temp_dir +) from ayon_core.lib import ( run_ayon_launcher_process, - get_transcode_temp_directory, convert_input_paths_for_ffmpeg, should_convert_for_ffmpeg ) @@ -250,7 +252,10 @@ def main_process(self, instance): # - change staging dir of source representation # - must be set back after output definitions processing if do_convert: - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) repre["stagingDir"] = new_staging_dir convert_input_paths_for_ffmpeg( diff --git a/client/ayon_core/plugins/publish/extract_color_transcode.py b/client/ayon_core/plugins/publish/extract_color_transcode.py index a28a761e7e..ba173867f8 100644 --- a/client/ayon_core/plugins/publish/extract_color_transcode.py +++ b/client/ayon_core/plugins/publish/extract_color_transcode.py @@ -3,15 +3,15 @@ import clique import pyblish.api -from ayon_core.pipeline import publish +from ayon_core.pipeline import ( + publish, + get_temp_dir +) from ayon_core.lib import ( - is_oiio_supported, ) - from ayon_core.lib.transcoding import ( convert_colorspace, - get_transcode_temp_directory, ) from ayon_core.lib.profiles_filtering import filter_profiles @@ -104,7 +104,10 @@ def process(self, instance): new_repre = copy.deepcopy(repre) original_staging_dir = new_repre["stagingDir"] - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) new_repre["stagingDir"] = new_staging_dir if isinstance(new_repre["files"], list): @@ -254,7 +257,7 @@ def _translate_to_sequence(self, files_to_convert): (list) of [file.1001-1010#.exr] or [fileA.exr, fileB.exr] """ pattern = [clique.PATTERNS["frames"]] - collections, remainder = clique.assemble( + collections, _ = clique.assemble( files_to_convert, patterns=pattern, assume_padded_when_ambiguous=True) diff --git a/client/ayon_core/plugins/publish/extract_review.py b/client/ayon_core/plugins/publish/extract_review.py index 06b451bfbe..26cd2ef0b2 100644 --- a/client/ayon_core/plugins/publish/extract_review.py +++ b/client/ayon_core/plugins/publish/extract_review.py @@ -22,8 +22,8 @@ should_convert_for_ffmpeg, get_review_layer_name, convert_input_paths_for_ffmpeg, - get_transcode_temp_directory, ) +from ayon_core.pipeline import get_temp_dir from ayon_core.pipeline.publish import ( KnownPublishError, get_publish_instance_label, @@ -310,7 +310,10 @@ def main_process(self, instance): # - change staging dir of source representation # - must be set back after output definitions processing if do_convert: - new_staging_dir = get_transcode_temp_directory() + new_staging_dir = get_temp_dir( + project_name=instance.context.data["projectName"], + make_local=True, + ) repre["stagingDir"] = new_staging_dir convert_input_paths_for_ffmpeg( From 8b1674619ce7004069bed2fa10d8af39ffac3cb6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 12 Sep 2024 16:40:49 +0200 Subject: [PATCH 02/13] Add staging directory functionality and a new plugin for managing staging directories in the pipeline. - Added import statement for 'os' in creator_plugins.py - Implemented method 'apply_staging_dir' to apply staging directory with persistence to instance's transient data in creator_plugins.py - Updated comments and added TODOs related to staging directories in various files - Created a new plugin 'CollectManagedStagingDir' to manage staging directories in publish/lib.py --- .../pipeline/create/creator_plugins.py | 55 +++++++++++++++++++ client/ayon_core/pipeline/publish/lib.py | 2 +- .../publish/collect_managed_staging_dir.py | 43 +++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 client/ayon_core/plugins/publish/collect_managed_staging_dir.py diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 61c10ee736..1360a74519 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import os import copy import collections from typing import TYPE_CHECKING, Optional @@ -14,6 +15,7 @@ deregister_plugin, deregister_plugin_path ) +from ayon_core.pipeline import get_staging_dir from .constants import DEFAULT_VARIANT_VALUE from .product_name import get_product_name @@ -782,6 +784,59 @@ def get_pre_create_attr_defs(self): """ return self.pre_create_attr_defs + def apply_staging_dir(self, instance): + """Apply staging dir with persistence to instance's transient data. + + Method is called on instance creation and on instance update. + + Args: + instance (CreatedInstance): Instance for which should be staging + dir applied. + + Returns: + str: Path to staging dir. + """ + create_ctx = self.create_context + product_name = instance.get("productName") + product_type = instance.get("productType") + folder_path = instance.get("folderPath") + if not any([product_name, folder_path]): + return None + + version = instance.get("version") + if version is not None: + formatting_data = {"version": version} + + staging_dir_data = get_staging_dir( + create_ctx.host_name, + create_ctx.get_current_project_entity(), + create_ctx.get_current_folder_entity(), + create_ctx.get_current_task_entity(), + product_type, + product_name, + create_ctx.get_current_project_anatomy(), + create_ctx.get_current_project_settings(), + always_return_path=False, + log=self.log, + formatting_data=formatting_data, + ) + + if not staging_dir_data: + return None + + staging_dir_path = staging_dir_data["stagingDir"] + + # TODO: not sure if this is necessary + # path might be already created by get_staging_dir + if not os.path.exists(staging_dir_path): + os.makedirs(staging_dir_path) + + instance.transient_data.update(staging_dir_data) + + self.log.info(f"Applied staging dir to instance: {staging_dir_path}") + + return staging_dir_path + class HiddenCreator(BaseCreator): @abstractmethod diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 9cfcd3f71a..714794e8f8 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -628,7 +628,7 @@ def get_publish_repre_path(instance, repre, only_published=False): return None -# deprecated: backward compatibility only +# deprecated: backward compatibility only (2024-09-12) # TODO: remove in the future def get_custom_staging_dir_info( project_name, diff --git a/client/ayon_core/plugins/publish/collect_managed_staging_dir.py b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py new file mode 100644 index 0000000000..ca6d5161c1 --- /dev/null +++ b/client/ayon_core/plugins/publish/collect_managed_staging_dir.py @@ -0,0 +1,43 @@ +""" +Requires: + anatomy + + +Provides: + instance.data -> stagingDir (folder path) + -> stagingDir_persistent (bool) +""" + +import pyblish.api + +from ayon_core.pipeline.publish import get_instance_staging_dir + + +class CollectManagedStagingDir(pyblish.api.InstancePlugin): + """Apply matching Staging Dir profile to a instance. + + Apply Staging dir via profiles could be useful in specific use cases + where is desirable to have temporary renders in specific, + persistent folders, could be on disks optimized for speed for example. + + It is studio's responsibility to clean up obsolete folders with data. + + Location of the folder is configured in: + `ayon+anatomy://_/templates/staging`. + + Which family/task type/subset is applicable is configured in: + `ayon+settings://core/tools/publish/custom_staging_dir_profiles` + """ + + label = "Collect Managed Staging Directory" + order = pyblish.api.CollectorOrder + 0.4990 + + def process(self, instance): + + staging_dir_path = get_instance_staging_dir(instance) + persistance = instance.data.get("stagingDir_persistent", False) + + self.log.info(( + f"Instance staging dir was set to `{staging_dir_path}` " + f"and persistence is set to `{persistance}`" + )) From 9e57f74b5c354a3d864eed71f545a5a7f93cc001 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Fri, 13 Sep 2024 16:20:46 +0200 Subject: [PATCH 03/13] Update variable names for clarity and consistency. - Renamed variables for better understanding and uniformity - Improved readability by using more descriptive names --- client/ayon_core/pipeline/publish/lib.py | 6 +++--- client/ayon_core/pipeline/stagingdir.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 714794e8f8..fb4db6ddf1 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -696,7 +696,7 @@ def get_instance_staging_dir(instance): workfile_name, _ = os.path.splitext(workfile) formatting_data["workfile_name"] = workfile_name - dir_data = get_staging_dir( + staging_dir_data = get_staging_dir( host_name, project_entity, folder_entity, @@ -708,14 +708,14 @@ def get_instance_staging_dir(instance): formatting_data=formatting_data, ) - staging_dir_path = dir_data["stagingDir"] + staging_dir_path = staging_dir_data["stagingDir"] # TODO: not sure if this is necessary # path might be already created by get_staging_dir if not os.path.exists(staging_dir_path): os.makedirs(staging_dir_path) - instance.data.update(dir_data) + instance.data.update(staging_dir_data) return staging_dir_path diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index e8fa1c4853..5ab9596528 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -202,7 +202,7 @@ def get_staging_dir( if not staging_dir_config and always_return_path: return { "stagingDir": get_temp_dir( - project_name=project_name, + project_name=project_entity["name"], anatomy=anatomy, prefix=kwargs.get("prefix"), suffix=kwargs.get("suffix"), From 2b5ab5439aa7d33b5c70ca11ce476cadae81ba0f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:41:20 +0200 Subject: [PATCH 04/13] returning empty lines --- client/ayon_core/pipeline/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index d5c3140d37..505c847c36 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -120,12 +120,16 @@ "AYON_CONTAINER_ID", "AYON_INSTANCE_ID", "HOST_WORKFILE_EXTENSIONS", + # --- Anatomy --- "Anatomy", + # --- Temp dir --- "get_temp_dir", + # --- Staging dir --- "get_staging_dir", + # --- Create --- "BaseCreator", "Creator", @@ -134,6 +138,7 @@ "CreatedInstance", "CreatorError", "CreatorError", + # - legacy creation "LegacyCreator", "legacy_create", @@ -143,6 +148,7 @@ "deregister_creator_plugin", "register_creator_plugin_path", "deregister_creator_plugin_path", + # --- Load --- "HeroVersionType", "IncompatibleLoaderError", @@ -161,6 +167,7 @@ "get_representation_path", "get_representation_context", "get_repres_contexts", + # --- Publish --- "PublishValidationError", "PublishXmlValidationError", @@ -168,6 +175,7 @@ "AYONPyblishPluginMixin", "OpenPypePyblishPluginMixin", "OptionalPyblishPluginMixin", + # --- Actions --- "LauncherAction", "InventoryAction", @@ -179,6 +187,7 @@ "register_inventory_action_path", "deregister_inventory_action", "deregister_inventory_action_path", + # --- Process context --- "install_ayon_plugins", "install_openpype_plugins", @@ -197,12 +206,14 @@ "get_current_project_name", "get_current_folder_path", "get_current_task_name", + # Workfile templates "discover_workfile_build_plugins", "register_workfile_build_plugin", "deregister_workfile_build_plugin", "register_workfile_build_plugin_path", "deregister_workfile_build_plugin_path", + # Backwards compatible function names "install", "uninstall", From 2b765954a3451bc6cba358584b64fed9e8f633e6 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:54:05 +0200 Subject: [PATCH 05/13] returning empty lines --- client/ayon_core/pipeline/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index 505c847c36..ea8b1617c6 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -137,11 +137,13 @@ "HiddenCreator", "CreatedInstance", "CreatorError", + "CreatorError", # - legacy creation "LegacyCreator", "legacy_create", + "discover_creator_plugins", "discover_legacy_creator_plugins", "register_creator_plugin", @@ -154,15 +156,18 @@ "IncompatibleLoaderError", "LoaderPlugin", "ProductLoaderPlugin", + "discover_loader_plugins", "register_loader_plugin", "deregister_loader_plugin_path", "register_loader_plugin_path", "deregister_loader_plugin", + "load_container", "remove_container", "update_container", "switch_container", + "loaders_from_representation", "get_representation_path", "get_representation_context", @@ -179,9 +184,11 @@ # --- Actions --- "LauncherAction", "InventoryAction", + "discover_launcher_actions", "register_launcher_action", "register_launcher_action_path", + "discover_inventory_actions", "register_inventory_action", "register_inventory_action_path", @@ -194,12 +201,15 @@ "install_host", "uninstall_host", "is_installed", + "register_root", "registered_root", + "register_host", "registered_host", "deregister_host", "get_process_id", + "get_global_context", "get_current_context", "get_current_host_name", From 396af0cf8610c2d2991c8a7842f164dcc49d98f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:52:43 +0200 Subject: [PATCH 06/13] Update client/ayon_core/pipeline/create/creator_plugins.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/create/creator_plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 1360a74519..32ac2bd61f 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -800,7 +800,7 @@ def apply_staging_dir(self, instance): product_name = instance.get("productName") product_type = instance.get("productType") folder_path = instance.get("folderPath") - if not any([product_name, folder_path]): + if not product_name or not folder_path: return None version = instance.get("version") From 9a860785bbce917cc2064db74a018ab012922d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:52:57 +0200 Subject: [PATCH 07/13] Update client/ayon_core/pipeline/create/creator_plugins.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/create/creator_plugins.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 32ac2bd61f..0de7707e38 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -828,8 +828,7 @@ def apply_staging_dir(self, instance): # TODO: not sure if this is necessary # path might be already created by get_staging_dir - if not os.path.exists(staging_dir_path): - os.makedirs(staging_dir_path) + os.makedirs(staging_dir_path, exist_ok=True) instance.transient_data.update(staging_dir_data) From bd03634ed1a3237e49f1d3307a96722d0960b2a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 13:53:38 +0200 Subject: [PATCH 08/13] Update client/ayon_core/pipeline/publish/lib.py Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index fb4db6ddf1..ba56c38c78 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -642,7 +642,15 @@ def get_custom_staging_dir_info( log=None, ): from ayon_core.pipeline.stagingdir import get_staging_dir_config - + warnings.warn( + ( + "Function 'get_custom_staging_dir_info' in" + " 'ayon_core.pipeline.publish' is deprecated. Please use" + " 'get_custom_staging_dir_info'" + " in 'ayon_core.pipeline.stagingdir'." + ), + DeprecationWarning, + ) tr_data = get_staging_dir_config( host_name, project_name, From fedf8e60c7b33f5873538773561269e686ee81b4 Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 13:56:45 +0200 Subject: [PATCH 09/13] Add warnings module for future use Imported the 'warnings' module for potential future usage in the codebase. --- client/ayon_core/pipeline/publish/lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index ba56c38c78..657af9570b 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -2,6 +2,7 @@ import sys import inspect import copy +import warnings import xml.etree.ElementTree from typing import Optional, Union, List From ea23f355f6dcb24f2dcb2806878afa27ddd11907 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Je=C5=BEek?= Date: Thu, 24 Oct 2024 14:00:36 +0200 Subject: [PATCH 10/13] Apply suggestions from code review Co-authored-by: Jakub Trllo <43494761+iLLiCiTiT@users.noreply.github.com> --- client/ayon_core/pipeline/publish/lib.py | 3 +-- client/ayon_core/pipeline/stagingdir.py | 20 +++++++------------- client/ayon_core/pipeline/tempdir.py | 15 ++++++--------- 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 657af9570b..6a31da82b2 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -721,8 +721,7 @@ def get_instance_staging_dir(instance): # TODO: not sure if this is necessary # path might be already created by get_staging_dir - if not os.path.exists(staging_dir_path): - os.makedirs(staging_dir_path) + os.makedirs(staging_dir_path, exist_ok=True) instance.data.update(staging_dir_data) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index 5ab9596528..d0172c4848 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -1,9 +1,9 @@ from ayon_core.lib import Logger, filter_profiles, StringTemplate from ayon_core.settings import get_project_settings -from .anatomy import Anatomy -from .tempdir import get_temp_dir from ayon_core.pipeline.template_data import get_template_data +from .anatomy import Anatomy +from .tempdir import get_temp_dir STAGING_DIR_TEMPLATES = "staging" @@ -34,8 +34,10 @@ def get_staging_dir_config( Returns: Dict or None: Data with directory template and is_persistent or None + Raises: ValueError - if misconfigured template should be used + """ settings = project_settings or get_project_settings(project_name) @@ -91,14 +93,6 @@ def _validate_template_name(project_name, template_name, anatomy): Raises: ValueError - if misconfigured template """ - # TODO: only for backward compatibility of anatomy for older projects - if STAGING_DIR_TEMPLATES not in anatomy.templates: - raise ValueError( - ( - 'Anatomy of project "{}" does not have set' ' "{}" template section!' - ).format(project_name, template_name) - ) - if template_name not in anatomy.templates[STAGING_DIR_TEMPLATES]: raise ValueError( ( @@ -147,9 +141,9 @@ def get_staging_dir( template. Returns: - Dict[str, Any]: Staging dir data - """ + Optional[Dict[str, Any]]: Staging dir data + """ log = kwargs.get("log") or Logger.get_logger("get_staging_dir") always_return_path = kwargs.get("always_return_path") @@ -209,7 +203,7 @@ def get_staging_dir( ), "stagingDir_persistent": False, } - elif not staging_dir_config: + if not staging_dir_config: return None return { diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index a6328135ee..448e774e7c 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -23,24 +23,21 @@ def get_temp_dir( Template formatting is supported also with optional keys. Folder is created in case it doesn't exists. - Available anatomy formatting keys: - - root[work | ] - - project[name | code] - Note: Staging dir does not have to be necessarily in tempdir so be careful about its usage. Args: - project_name (str)[optional]: Name of project. - anatomy (openpype.pipeline.Anatomy)[optional]: Anatomy object. - make_local (bool)[optional]: If True, temp dir will be created in + project_name (str): Name of project. + anatomy (Optional[Anatomy]): Project Anatomy object. + suffix (Optional[str]): Suffix for tempdir. + prefix (Optional[str]): Prefix for tempdir. + make_local (Optional[bool]): If True, temp dir will be created in local tempdir. - suffix (str)[optional]: Suffix for tempdir. - prefix (str)[optional]: Prefix for tempdir. Returns: str: Path to staging dir of instance. + """ prefix = prefix or "ay_tmp_" suffix = suffix or "" From ea4ec677cac7a73f14225722f0dbc9c17328ff6a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 14:40:30 +0200 Subject: [PATCH 11/13] reviewer suggestions for changes - Renamed function `get_staging_dir` to `get_staging_dir_info` for clarity. - Updated variable names in multiple files to use `template_data` instead of `formatting_data`. - Adjusted function parameters in the staging directory module for consistency and added new optional parameters. - Improved logging by passing logger instances instead of creating new loggers within functions. --- client/ayon_core/pipeline/__init__.py | 4 +- .../pipeline/create/creator_plugins.py | 18 ++++----- client/ayon_core/pipeline/publish/lib.py | 17 ++++----- client/ayon_core/pipeline/stagingdir.py | 34 ++++++++++------- client/ayon_core/pipeline/tempdir.py | 37 ++++++++----------- 5 files changed, 54 insertions(+), 56 deletions(-) diff --git a/client/ayon_core/pipeline/__init__.py b/client/ayon_core/pipeline/__init__.py index ea8b1617c6..4060501a92 100644 --- a/client/ayon_core/pipeline/__init__.py +++ b/client/ayon_core/pipeline/__init__.py @@ -10,7 +10,7 @@ from .tempdir import get_temp_dir -from .stagingdir import get_staging_dir +from .stagingdir import get_staging_dir_info from .create import ( BaseCreator, @@ -128,7 +128,7 @@ "get_temp_dir", # --- Staging dir --- - "get_staging_dir", + "get_staging_dir_info", # --- Create --- "BaseCreator", diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 0de7707e38..124395ae16 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -15,7 +15,7 @@ deregister_plugin, deregister_plugin_path ) -from ayon_core.pipeline import get_staging_dir +from ayon_core.pipeline import get_staging_dir_info from .constants import DEFAULT_VARIANT_VALUE from .product_name import get_product_name @@ -805,9 +805,9 @@ def apply_staging_dir(self, instance): version = instance.get("version") if version is not None: - formatting_data = {"version": version} + template_data = {"version": version} - staging_dir_data = get_staging_dir( + staging_dir_info = get_staging_dir_info( create_ctx.host_name, create_ctx.get_current_project_entity(), create_ctx.get_current_folder_entity(), @@ -817,20 +817,20 @@ def apply_staging_dir(self, instance): create_ctx.get_current_project_anatomy(), create_ctx.get_current_project_settings(), always_return_path=False, - log=self.log, - formatting_data=formatting_data, + logger=self.log, + template_data=template_data, ) - if not staging_dir_data: + if not staging_dir_info: return None - staging_dir_path = staging_dir_data["stagingDir"] + staging_dir_path = staging_dir_info["stagingDir"] # TODO: not sure if this is necessary - # path might be already created by get_staging_dir + # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - instance.transient_data.update(staging_dir_data) + instance.transient_data.update(staging_dir_info) self.log.info(f"Applied staging dir to instance: {staging_dir_path}") diff --git a/client/ayon_core/pipeline/publish/lib.py b/client/ayon_core/pipeline/publish/lib.py index 6a31da82b2..0f3a7c1d45 100644 --- a/client/ayon_core/pipeline/publish/lib.py +++ b/client/ayon_core/pipeline/publish/lib.py @@ -18,7 +18,7 @@ ) from ayon_core.settings import get_project_settings from ayon_core.addon import AddonsManager -from ayon_core.pipeline import get_staging_dir +from ayon_core.pipeline import get_staging_dir_info from ayon_core.pipeline.plugin_discover import DiscoverResult from .constants import ( DEFAULT_PUBLISH_TEMPLATE, @@ -685,7 +685,7 @@ def get_instance_staging_dir(instance): return staging_dir anatomy_data = instance.data["anatomyData"] - formatting_data = copy.deepcopy(anatomy_data) + template_data = copy.deepcopy(anatomy_data) product_type = instance.data["productType"] product_name = instance.data["productName"] @@ -703,9 +703,9 @@ def get_instance_staging_dir(instance): if current_file: workfile = os.path.basename(current_file) workfile_name, _ = os.path.splitext(workfile) - formatting_data["workfile_name"] = workfile_name + template_data["workfile_name"] = workfile_name - staging_dir_data = get_staging_dir( + staging_dir_info = get_staging_dir_info( host_name, project_entity, folder_entity, @@ -714,16 +714,15 @@ def get_instance_staging_dir(instance): product_name, anatomy, project_settings=project_settings, - formatting_data=formatting_data, + template_data=template_data, ) - staging_dir_path = staging_dir_data["stagingDir"] + staging_dir_path = staging_dir_info["stagingDir"] - # TODO: not sure if this is necessary - # path might be already created by get_staging_dir + # path might be already created by get_staging_dir_info os.makedirs(staging_dir_path, exist_ok=True) - instance.data.update(staging_dir_data) + instance.data.update(staging_dir_info) return staging_dir_path diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index d0172c4848..e9d425cf28 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -102,7 +102,7 @@ def _validate_template_name(project_name, template_name, anatomy): ) -def get_staging_dir( +def get_staging_dir_info( host_name, project_entity, folder_entity, @@ -111,9 +111,13 @@ def get_staging_dir( product_name, anatomy, project_settings=None, + template_data=None, + always_return_path=None, + force_tmp_dir=None, + logger=None, **kwargs ): - """Get staging dir data. + """Get staging dir info data. If `force_temp` is set, staging dir will be created as tempdir. If `always_get_some_dir` is set, staging dir will be created as tempdir if @@ -129,29 +133,31 @@ def get_staging_dir( product_name (str): Name of product. anatomy (ayon_core.pipeline.Anatomy): Anatomy object. project_settings (Optional[Dict[str, Any]]): Prepared project settings. + template_data (Optional[Dict[str, Any]]): Data for formatting staging + dir template. + always_return_path (Optional[bool]): If True, staging dir will be + created as tempdir if no staging dir profile is found. Input value + False will return None if no staging dir profile is found. + force_tmp_dir (Optional[bool]): If True, staging dir will be created as + tempdir. + logger (Optional[logging.Logger]): Logger instance. **kwargs: Arbitrary keyword arguments. See below. Keyword Arguments: - force_temp (bool): If True, staging dir will be created as tempdir. - always_return_path (bool): If True, staging dir will be created as - tempdir if no staging dir profile is found. prefix (str): Prefix for staging dir. suffix (str): Suffix for staging dir. - formatting_data (Dict[str, Any]): Data for formatting staging dir - template. Returns: - Optional[Dict[str, Any]]: Staging dir data + Optional[Dict[str, Any]]: Staging dir info data """ - log = kwargs.get("log") or Logger.get_logger("get_staging_dir") - always_return_path = kwargs.get("always_return_path") + log = logger or Logger.get_logger("get_staging_dir_info") # make sure always_return_path is set to true by default if always_return_path is None: always_return_path = True - if kwargs.get("force_temp"): + if force_tmp_dir: return get_temp_dir( project_name=project_entity["name"], anatomy=anatomy, @@ -175,9 +181,9 @@ def get_staging_dir( "host": host_name, }) - # add additional data from kwargs - if kwargs.get("formatting_data"): - ctx_data.update(kwargs.get("formatting_data")) + # add additional template formatting data + if template_data: + ctx_data.update(template_data) # get staging dir config staging_dir_config = get_staging_dir_config( diff --git a/client/ayon_core/pipeline/tempdir.py b/client/ayon_core/pipeline/tempdir.py index 448e774e7c..440ed882aa 100644 --- a/client/ayon_core/pipeline/tempdir.py +++ b/client/ayon_core/pipeline/tempdir.py @@ -10,29 +10,25 @@ def get_temp_dir( - project_name=None, anatomy=None, prefix=None, suffix=None, make_local=False + project_name, anatomy=None, prefix=None, suffix=None, use_local_temp=False ): """Get temporary dir path. - If `make_local` is set, tempdir will be created in local tempdir. + If `use_local_temp` is set, tempdir will be created in local tempdir. If `anatomy` is not set, default anatomy will be used. If `prefix` or `suffix` is not set, default values will be used. - It also supports `OPENPYPE_TMPDIR`, so studio can define own temp + It also supports `AYON_TMPDIR`, so studio can define own temp shared repository per project or even per more granular context. Template formatting is supported also with optional keys. Folder is created in case it doesn't exists. - Note: - Staging dir does not have to be necessarily in tempdir so be careful - about its usage. - Args: project_name (str): Name of project. anatomy (Optional[Anatomy]): Project Anatomy object. suffix (Optional[str]): Suffix for tempdir. prefix (Optional[str]): Prefix for tempdir. - make_local (Optional[bool]): If True, temp dir will be created in + use_local_temp (Optional[bool]): If True, temp dir will be created in local tempdir. Returns: @@ -42,7 +38,7 @@ def get_temp_dir( prefix = prefix or "ay_tmp_" suffix = suffix or "" - if make_local: + if use_local_temp: return _create_local_staging_dir(prefix, suffix) # make sure anatomy is set @@ -55,19 +51,20 @@ def get_temp_dir( return _create_local_staging_dir(prefix, suffix, custom_temp_dir) -def _create_local_staging_dir(prefix, suffix, dir=None): +def _create_local_staging_dir(prefix, suffix, dirpath=None): """Create local staging dir Args: prefix (str): prefix for tempdir suffix (str): suffix for tempdir + dirpath (Optional[str]): path to tempdir Returns: str: path to tempdir """ # use pathlib for creating tempdir staging_dir = Path(tempfile.mkdtemp( - prefix=prefix, suffix=suffix, dir=dir + prefix=prefix, suffix=suffix, dir=dirpath )) return staging_dir.as_posix() @@ -89,31 +86,27 @@ def _create_custom_tempdir(project_name, anatomy=None): Returns: str | None: formatted path or None """ - env_tmpdir = os.getenv("AYON_TMPDIR") + env_tmpdir = os.getenv( + "AYON_TMPDIR", + ) if not env_tmpdir: - env_tmpdir = os.getenv("OPENPYPE_TMPDIR") - if not env_tmpdir: - return - print( - "DEPRECATION WARNING: Used 'OPENPYPE_TMPDIR' environment" - " variable. Please use 'AYON_TMPDIR' instead." - ) + return None custom_tempdir = None if "{" in env_tmpdir: if anatomy is None: anatomy = Anatomy(project_name) # create base formate data - template_formatting_data = { + template_data = { "root": anatomy.roots, "project": { "name": anatomy.project_name, "code": anatomy.project_code, - } + }, } # path is anatomy template custom_tempdir = StringTemplate.format_template( - env_tmpdir, template_formatting_data) + env_tmpdir, template_data) custom_tempdir_path = Path(custom_tempdir) From 1eb09045832f29024b0b2dba00adc6d71ace132f Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 14:48:47 +0200 Subject: [PATCH 12/13] Remove unnecessary root addition in get_staging_dir_info function. The code changes remove the unnecessary addition of roots to ctx_data in the get_staging_dir_info function. --- client/ayon_core/pipeline/stagingdir.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/stagingdir.py b/client/ayon_core/pipeline/stagingdir.py index e9d425cf28..818acef36a 100644 --- a/client/ayon_core/pipeline/stagingdir.py +++ b/client/ayon_core/pipeline/stagingdir.py @@ -169,8 +169,6 @@ def get_staging_dir_info( ctx_data = get_template_data( project_entity, folder_entity, task_entity, host_name ) - # add roots to ctx_data - ctx_data["root"] = anatomy.roots # add additional data ctx_data.update({ @@ -178,7 +176,7 @@ def get_staging_dir_info( "type": product_type, "name": product_name }, - "host": host_name, + "root": anatomy.roots }) # add additional template formatting data From fa9af2f8ded7e9c89519cfde2f01e04a3dd8b58a Mon Sep 17 00:00:00 2001 From: Jakub Jezek Date: Thu, 24 Oct 2024 15:06:11 +0200 Subject: [PATCH 13/13] Refactor Creator class method to handle missing info better. - Updated return type in docstring - Added a comment for clarity - Removed unnecessary return statements --- client/ayon_core/pipeline/create/creator_plugins.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/client/ayon_core/pipeline/create/creator_plugins.py b/client/ayon_core/pipeline/create/creator_plugins.py index 801feedd3d..4cbf432efd 100644 --- a/client/ayon_core/pipeline/create/creator_plugins.py +++ b/client/ayon_core/pipeline/create/creator_plugins.py @@ -843,14 +843,16 @@ def apply_staging_dir(self, instance): dir applied. Returns: - str: Path to staging dir. + Optional[str]: Staging dir path or None if not applied. """ create_ctx = self.create_context product_name = instance.get("productName") product_type = instance.get("productType") folder_path = instance.get("folderPath") + + # this can only work if product name and folder path are available if not product_name or not folder_path: - return None + return version = instance.get("version") if version is not None: @@ -871,7 +873,7 @@ def apply_staging_dir(self, instance): ) if not staging_dir_info: - return None + return staging_dir_path = staging_dir_info["stagingDir"]