From 36009c4c86376952c78aef80420c4ce18f4846fe Mon Sep 17 00:00:00 2001 From: GarethCabournDavies Date: Thu, 13 Jul 2023 05:57:50 -0700 Subject: [PATCH 1/5] Add ability to overwrite file retention level for specific instances --- pycbc/workflow/core.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index 46536b476e8..75575f504da 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -147,7 +147,12 @@ def __init__(self, cp, name, ifos=None, out_dir=None, tags=None, self.update_output_directory(out_dir=out_dir) # Determine the level at which output files should be kept - self.update_current_retention_level(self.current_retention_level) + if cp.has_option_tags('pegasus_profile-%s' % name, + 'pycbc|discard_output', tags): + # Flag to say not to keep file, whatever the retention level + self.update_current_retention_level(-1) + else: + self.update_current_retention_level(self.current_retention_level) # Should I reuse this executable? if reuse_executable: From d0ebda07f216df7422bf2a806ff4924dfe855544 Mon Sep 17 00:00:00 2001 From: GarethCabournDavies Date: Fri, 14 Jul 2023 02:21:44 -0700 Subject: [PATCH 2/5] Allow different retention levels rather than juts not keeping --- pycbc/workflow/core.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index 75575f504da..b5d733f65f3 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -47,6 +47,14 @@ from .configuration import WorkflowConfigParser, resolve_url from .pegasus_sites import make_catalog +# FIXME: Are these names suitably descriptive? +_retention_choices = { + 'do_not_keep' : 0, + 'all_files' : 1, + 'all_triggers' : 2, + 'merged_triggers' : 3, + 'results' : 4 +} def make_analysis_dir(path): """ @@ -148,9 +156,12 @@ def __init__(self, cp, name, ifos=None, out_dir=None, tags=None, # Determine the level at which output files should be kept if cp.has_option_tags('pegasus_profile-%s' % name, - 'pycbc|discard_output', tags): - # Flag to say not to keep file, whatever the retention level - self.update_current_retention_level(-1) + 'pycbc|retention_level', tags): + cfg_ret_level = cp.get_opt_tags('pegasus_profile-%s' % name, + 'pycbc|retention_level', tags) + self.update_current_retention_level( + _retention_choices[cfg_ret_level] + ) else: self.update_current_retention_level(self.current_retention_level) @@ -482,16 +493,9 @@ def update_current_retention_level(self, value): self.global_retention_threshold = 1 self.cp.set("workflow", "file-retention-level", "all_files") else: - # FIXME: Are these names suitably descriptive? - retention_choices = { - 'all_files' : 1, - 'all_triggers' : 2, - 'merged_triggers' : 3, - 'results' : 4 - } try: self.global_retention_threshold = \ - retention_choices[global_retention_level] + _retention_choices[global_retention_level] except KeyError: err_msg = "Cannot recognize the file-retention-level in the " err_msg += "[workflow] section of the ini file. " From 55d1431003ab51702128ba551fc66da37aa46873 Mon Sep 17 00:00:00 2001 From: GarethCabournDavies Date: Fri, 14 Jul 2023 03:13:30 -0700 Subject: [PATCH 3/5] CC --- pycbc/workflow/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index b5d733f65f3..3a6ae07af45 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -56,6 +56,7 @@ 'results' : 4 } + def make_analysis_dir(path): """ Make the analysis directory path, any parent directories that don't already From 8d77462b6edbe672b93ce534accc0d9ea4bdf89f Mon Sep 17 00:00:00 2001 From: GarethCabournDavies Date: Mon, 17 Jul 2023 07:06:23 -0700 Subject: [PATCH 4/5] Global and Executable retention levels are not the same --- pycbc/workflow/core.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index 3a6ae07af45..31104e21e20 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -47,15 +47,6 @@ from .configuration import WorkflowConfigParser, resolve_url from .pegasus_sites import make_catalog -# FIXME: Are these names suitably descriptive? -_retention_choices = { - 'do_not_keep' : 0, - 'all_files' : 1, - 'all_triggers' : 2, - 'merged_triggers' : 3, - 'results' : 4 -} - def make_analysis_dir(path): """ @@ -70,6 +61,7 @@ def make_analysis_dir(path): class Executable(pegasus_workflow.Executable): # These are the file retention levels + DO_NOT_KEEP = 0 INTERMEDIATE_PRODUCT = 1 ALL_TRIGGERS = 2 MERGED_TRIGGERS = 3 @@ -158,13 +150,14 @@ def __init__(self, cp, name, ifos=None, out_dir=None, tags=None, # Determine the level at which output files should be kept if cp.has_option_tags('pegasus_profile-%s' % name, 'pycbc|retention_level', tags): + # Get the retention_level from the config file + # This method allows us to use the retention levels + # defined above cfg_ret_level = cp.get_opt_tags('pegasus_profile-%s' % name, - 'pycbc|retention_level', tags) - self.update_current_retention_level( - _retention_choices[cfg_ret_level] - ) - else: - self.update_current_retention_level(self.current_retention_level) + 'pycbc|retention_level', tags) + self.current_retention_level = getattr(self, cfg_ret_level) + + self.update_current_retention_level(self.current_retention_level) # Should I reuse this executable? if reuse_executable: @@ -494,9 +487,17 @@ def update_current_retention_level(self, value): self.global_retention_threshold = 1 self.cp.set("workflow", "file-retention-level", "all_files") else: + # FIXME: Are these names suitably descriptive? + retention_choices = { + 'all_files' : 1, + 'all_triggers' : 2, + 'merged_triggers' : 3, + 'results' : 4 + } + try: self.global_retention_threshold = \ - _retention_choices[global_retention_level] + retention_choices[global_retention_level] except KeyError: err_msg = "Cannot recognize the file-retention-level in the " err_msg += "[workflow] section of the ini file. " From 1a83728fb28c6862a6ab55fccd96ba30f64138d7 Mon Sep 17 00:00:00 2001 From: GarethCabournDavies Date: Mon, 17 Jul 2023 07:08:57 -0700 Subject: [PATCH 5/5] Fix incorrect indent --- pycbc/workflow/core.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pycbc/workflow/core.py b/pycbc/workflow/core.py index 31104e21e20..692b04c6e60 100644 --- a/pycbc/workflow/core.py +++ b/pycbc/workflow/core.py @@ -493,8 +493,7 @@ def update_current_retention_level(self, value): 'all_triggers' : 2, 'merged_triggers' : 3, 'results' : 4 - } - + } try: self.global_retention_threshold = \ retention_choices[global_retention_level]