From 2f4459e5a3afea46f5b96216f025332a31061a1e Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Wed, 7 Feb 2024 12:31:51 +0100 Subject: [PATCH 01/17] Make link of the updated longterm datacheck file in a common directory --- src/osa/scripts/closer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index 03a36446..9f36ee77 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -520,8 +520,14 @@ def cherenkov_transparency(cmd: List[str]): log.info("Update longterm dl1 check file with cherenkov_transparency script.") log.debug(f"Executing {stringify(cmd)}") + nightdir = date_to_dir(options.date) + longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir + longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5" + linked_longterm_file = Path(cfg.get("LST1", "LONGTERM_DIR")) / f"night_wise/all/DL1_datacheck_{nightdir}.h5" + if not options.simulate and not options.test and shutil.which("sbatch") is not None: subprocess.run(cmd, check=True) + linked_longterm_file.symlink_to(longterm_datacheck_file) else: log.debug("Simulate launching scripts") From d46722153a8c1c2b170fb359a705099f1742513a Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Wed, 7 Feb 2024 16:01:55 +0100 Subject: [PATCH 02/17] Make the symlink to the created file only if it's the latest version --- src/osa/scripts/closer.py | 48 +++++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index 9f36ee77..d89e3962 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -8,6 +8,7 @@ import shutil import subprocess import sys +import glob from datetime import datetime, timedelta from pathlib import Path from typing import Tuple, Iterable, List @@ -18,7 +19,7 @@ from osa.job import are_all_jobs_correctly_finished, save_job_information from osa.nightsummary.extract import extract_runs, extract_sequences from osa.nightsummary.nightsummary import run_summary_table -from osa.paths import destination_dir +from osa.paths import destination_dir, get_major_version from osa.raw import is_raw_data_available from osa.report import start from osa.utils.cliopts import closercliparsing @@ -32,6 +33,7 @@ create_lock, gettag, date_to_iso, + get_lstchain_version, ) __all__ = [ @@ -158,6 +160,7 @@ def post_process(seq_tuple): list_job_id = merge_dl1_datacheck(seq_list) longterm_job_id = daily_datacheck(daily_longterm_cmd(list_job_id)) cherenkov_transparency(cherenkov_transparency_cmd(longterm_job_id)) + create_longterm_symlink() # Extract the provenance info extract_provenance(seq_list) @@ -520,17 +523,48 @@ def cherenkov_transparency(cmd: List[str]): log.info("Update longterm dl1 check file with cherenkov_transparency script.") log.debug(f"Executing {stringify(cmd)}") - nightdir = date_to_dir(options.date) - longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) / options.prod_id / nightdir - longterm_datacheck_file = longterm_dir / f"DL1_datacheck_{nightdir}.h5" - linked_longterm_file = Path(cfg.get("LST1", "LONGTERM_DIR")) / f"night_wise/all/DL1_datacheck_{nightdir}.h5" - if not options.simulate and not options.test and shutil.which("sbatch") is not None: subprocess.run(cmd, check=True) - linked_longterm_file.symlink_to(longterm_datacheck_file) else: log.debug("Simulate launching scripts") +def get_latest_version(longterm_files: List[str]) -> str: + """Get the latest version of the produced longterm DL1 datacheck files.""" + latest_version = 0 + for file in longterm_files: + idx1 = file.find("/v0.") + idx2 = file.find(f"/{date}") + version = file[idx1+1:idx2] + if int(version[3:])>latest_version: + latest_version = int(version[3:]) + + return "v0."+str(latest_version) + + +def create_longterm_symlink(): + """If the created longterm DL1 datacheck file corresponds to the latest + version available, make symlink to it in the "all" common directory.""" + nightdir = date_to_dir(options.date) + longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) + longterm_datacheck_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5" + linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5" + all_longterm_files = glob.glob(longterm_dir + f"/v*/{date}/DL1_datacheck_{date}.h5") + + if len(all_longterm_files) > 1: + latest_version = get_latest_version(all_longterm_files) + current_version = get_major_version(get_lstchain_version()) + if current_version == latest_version: + log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") + linked_longterm_file.symlink_to(longterm_datacheck_file) + else: + log.info("The created longterm DL1 datacheck file does not correspond to the \ + latest available version, so no symlink is made.") + continue + else: + log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") + linked_longterm_file.symlink_to(longterm_datacheck_file) + + if __name__ == "__main__": main() From c196668e3c83e5c59e30c69592cd97b06116e2bf Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Wed, 7 Feb 2024 16:04:55 +0100 Subject: [PATCH 03/17] Fix name of a variable --- src/osa/scripts/closer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index d89e3962..364c7fb5 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -531,10 +531,11 @@ def cherenkov_transparency(cmd: List[str]): def get_latest_version(longterm_files: List[str]) -> str: """Get the latest version of the produced longterm DL1 datacheck files.""" + nightdir = date_to_dir(options.date) latest_version = 0 for file in longterm_files: idx1 = file.find("/v0.") - idx2 = file.find(f"/{date}") + idx2 = file.find(f"/{nightdir}") version = file[idx1+1:idx2] if int(version[3:])>latest_version: latest_version = int(version[3:]) @@ -549,7 +550,7 @@ def create_longterm_symlink(): longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) longterm_datacheck_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5" linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5" - all_longterm_files = glob.glob(longterm_dir + f"/v*/{date}/DL1_datacheck_{date}.h5") + all_longterm_files = glob.glob(longterm_dir + f"/v*/{nightdir}/DL1_datacheck_{nightdir}.h5") if len(all_longterm_files) > 1: latest_version = get_latest_version(all_longterm_files) From 5aceb5a269092dbc3b2b0345d5c71f353892e860 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Wed, 7 Feb 2024 16:09:55 +0100 Subject: [PATCH 04/17] Fix mistake --- src/osa/scripts/closer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index 364c7fb5..7fa7595c 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -561,7 +561,7 @@ def create_longterm_symlink(): else: log.info("The created longterm DL1 datacheck file does not correspond to the \ latest available version, so no symlink is made.") - continue + return else: log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") linked_longterm_file.symlink_to(longterm_datacheck_file) From 7f9e0d3e376c26ae8f85feac0ae88280e4e03c2f Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Wed, 7 Feb 2024 16:34:31 +0100 Subject: [PATCH 05/17] Unlink already existing link file before creating new link --- src/osa/scripts/closer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index 7fa7595c..e47446a5 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -557,12 +557,13 @@ def create_longterm_symlink(): current_version = get_major_version(get_lstchain_version()) if current_version == latest_version: log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") + linked_longterm_file.unlink() linked_longterm_file.symlink_to(longterm_datacheck_file) else: log.info("The created longterm DL1 datacheck file does not correspond to the \ latest available version, so no symlink is made.") return - else: + else: log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") linked_longterm_file.symlink_to(longterm_datacheck_file) From 8648b654a41c8e7374171e405b44a40c7eb662cf Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 13 Feb 2024 11:09:07 +0100 Subject: [PATCH 06/17] Move get_latest_version and create_longterm_symlink functions to paths.py --- src/osa/paths.py | 42 ++++++++++++++++++++++++++++++++++++++- src/osa/scripts/closer.py | 38 +---------------------------------- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/osa/paths.py b/src/osa/paths.py index e4b907de..082ac88b 100644 --- a/src/osa/paths.py +++ b/src/osa/paths.py @@ -141,7 +141,8 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path: mongodb = cfg.get("database", "caco_db") try: # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB - options.filters = find_filter_wheels(int(run_id), mongodb) + #options.filters = find_filter_wheels(int(run_id), mongodb) + options.filters = search_filter(int(run_id), mongodb) except IOError: log.warning("No filter information found in database. Assuming positions 52.") options.filters = 52 @@ -346,3 +347,42 @@ def create_source_directories(source_list: list, cuts_dir: Path): if source is not None: source_dir = cuts_dir / source source_dir.mkdir(parents=True, exist_ok=True) + + +def get_latest_version(longterm_files: List[str]) -> str: + """Get the latest version of the produced longterm DL1 datacheck files.""" + nightdir = date_to_dir(options.date) + latest_version = 0 + for file in longterm_files: + idx1 = file.find("/v0.") + idx2 = file.find(f"/{nightdir}") + version = file[idx1+1:idx2] + if int(version[3:])>latest_version: + latest_version = int(version[3:]) + + return "v0."+str(latest_version) + + +def create_longterm_symlink(): + """If the created longterm DL1 datacheck file corresponds to the latest + version available, make symlink to it in the "all" common directory.""" + nightdir = date_to_dir(options.date) + longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) + longterm_datacheck_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5" + linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5" + all_longterm_files = glob.glob(longterm_dir + f"/v*/{nightdir}/DL1_datacheck_{nightdir}.h5") + + if len(all_longterm_files) > 1: + latest_version = get_latest_version(all_longterm_files) + current_version = get_major_version(get_lstchain_version()) + if current_version == latest_version: + log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") + linked_longterm_file.unlink() + linked_longterm_file.symlink_to(longterm_datacheck_file) + else: + log.info("The created longterm DL1 datacheck file does not correspond to the \ + latest available version, so no symlink is made.") + return + else: + log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") + linked_longterm_file.symlink_to(longterm_datacheck_file) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index e47446a5..977b0082 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -529,43 +529,7 @@ def cherenkov_transparency(cmd: List[str]): log.debug("Simulate launching scripts") -def get_latest_version(longterm_files: List[str]) -> str: - """Get the latest version of the produced longterm DL1 datacheck files.""" - nightdir = date_to_dir(options.date) - latest_version = 0 - for file in longterm_files: - idx1 = file.find("/v0.") - idx2 = file.find(f"/{nightdir}") - version = file[idx1+1:idx2] - if int(version[3:])>latest_version: - latest_version = int(version[3:]) - - return "v0."+str(latest_version) - - -def create_longterm_symlink(): - """If the created longterm DL1 datacheck file corresponds to the latest - version available, make symlink to it in the "all" common directory.""" - nightdir = date_to_dir(options.date) - longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) - longterm_datacheck_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5" - linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5" - all_longterm_files = glob.glob(longterm_dir + f"/v*/{nightdir}/DL1_datacheck_{nightdir}.h5") - - if len(all_longterm_files) > 1: - latest_version = get_latest_version(all_longterm_files) - current_version = get_major_version(get_lstchain_version()) - if current_version == latest_version: - log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") - linked_longterm_file.unlink() - linked_longterm_file.symlink_to(longterm_datacheck_file) - else: - log.info("The created longterm DL1 datacheck file does not correspond to the \ - latest available version, so no symlink is made.") - return - else: - log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") - linked_longterm_file.symlink_to(longterm_datacheck_file) + if __name__ == "__main__": From 7ea5a776d3ff9a653dded874dbc9f29cde6087d8 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 13 Feb 2024 11:56:41 +0100 Subject: [PATCH 07/17] Simplify get_latest_version function + return latest version file --- src/osa/paths.py | 40 +++++++++++---------------------------- src/osa/scripts/closer.py | 4 +--- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/src/osa/paths.py b/src/osa/paths.py index 082ac88b..9cf48bbc 100644 --- a/src/osa/paths.py +++ b/src/osa/paths.py @@ -17,6 +17,7 @@ from osa.configs.datamodel import Sequence from osa.utils import utils from osa.utils.logging import myLogger +from osa.utils.utils import date_to_dir log = myLogger(logging.getLogger(__name__)) @@ -349,18 +350,10 @@ def create_source_directories(source_list: list, cuts_dir: Path): source_dir.mkdir(parents=True, exist_ok=True) -def get_latest_version(longterm_files: List[str]) -> str: - """Get the latest version of the produced longterm DL1 datacheck files.""" - nightdir = date_to_dir(options.date) - latest_version = 0 - for file in longterm_files: - idx1 = file.find("/v0.") - idx2 = file.find(f"/{nightdir}") - version = file[idx1+1:idx2] - if int(version[3:])>latest_version: - latest_version = int(version[3:]) - - return "v0."+str(latest_version) +def get_latest_version_file(longterm_files: List[str]) -> Path: + """Get the latest version path of the produced longterm DL1 datacheck files for a given date.""" + latest_version_path = max(longterm_files, key=lambda path: int(path.parents[1].name.split(".")[1]) if path.parents[1].name.startswith("v") else "") + return latest_version_path def create_longterm_symlink(): @@ -368,21 +361,10 @@ def create_longterm_symlink(): version available, make symlink to it in the "all" common directory.""" nightdir = date_to_dir(options.date) longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) - longterm_datacheck_file = longterm_dir / options.prod_id / nightdir / f"DL1_datacheck_{nightdir}.h5" linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5" - all_longterm_files = glob.glob(longterm_dir + f"/v*/{nightdir}/DL1_datacheck_{nightdir}.h5") - - if len(all_longterm_files) > 1: - latest_version = get_latest_version(all_longterm_files) - current_version = get_major_version(get_lstchain_version()) - if current_version == latest_version: - log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") - linked_longterm_file.unlink() - linked_longterm_file.symlink_to(longterm_datacheck_file) - else: - log.info("The created longterm DL1 datacheck file does not correspond to the \ - latest available version, so no symlink is made.") - return - else: - log.info("Make symlink of the longterm DL1 datacheck file in the common directory.") - linked_longterm_file.symlink_to(longterm_datacheck_file) + all_longterm_files = sorted(longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")) + latest_version_file = get_latest_version_file(all_longterm_files) + + log.info("Make symlink of the latest version longterm DL1 datacheck file in the common directory.") + linked_longterm_file.unlink() + linked_longterm_file.symlink_to(latest_version_file) diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index 977b0082..c0edabce 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -8,7 +8,6 @@ import shutil import subprocess import sys -import glob from datetime import datetime, timedelta from pathlib import Path from typing import Tuple, Iterable, List @@ -19,7 +18,7 @@ from osa.job import are_all_jobs_correctly_finished, save_job_information from osa.nightsummary.extract import extract_runs, extract_sequences from osa.nightsummary.nightsummary import run_summary_table -from osa.paths import destination_dir, get_major_version +from osa.paths import destination_dir from osa.raw import is_raw_data_available from osa.report import start from osa.utils.cliopts import closercliparsing @@ -33,7 +32,6 @@ create_lock, gettag, date_to_iso, - get_lstchain_version, ) __all__ = [ From 44b70268b505af1051b65fc01afc49fe0b2d85b2 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 13 Feb 2024 12:03:17 +0100 Subject: [PATCH 08/17] Ensure the longterm directory exists for the closer tests --- src/osa/scripts/tests/test_osa_scripts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 7dcc7ee1..5cb423a3 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -179,6 +179,7 @@ def test_closer( drs4_time_calibration_files, systematic_correction_files, merged_run_summary, + longterm_dir, ): # First assure that the end of night flag is not set and remove it otherwise night_finished_flag = Path( @@ -198,6 +199,7 @@ def test_closer( for obs_file in test_observed_data: assert obs_file.exists() assert merged_run_summary.exists() + assert longterm_dir.exists() run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1") closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed" From 13682d0f35020ba07bee5630cf3e05a7824237fb Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 13 Feb 2024 12:32:25 +0100 Subject: [PATCH 09/17] Fix small issues --- src/osa/paths.py | 2 +- src/osa/scripts/closer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osa/paths.py b/src/osa/paths.py index 9cf48bbc..a07deb83 100644 --- a/src/osa/paths.py +++ b/src/osa/paths.py @@ -143,7 +143,7 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path: try: # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB #options.filters = find_filter_wheels(int(run_id), mongodb) - options.filters = search_filter(int(run_id), mongodb) + options.filters = find_filter_wheels(int(run_id), mongodb) except IOError: log.warning("No filter information found in database. Assuming positions 52.") options.filters = 52 diff --git a/src/osa/scripts/closer.py b/src/osa/scripts/closer.py index c0edabce..d54f0510 100644 --- a/src/osa/scripts/closer.py +++ b/src/osa/scripts/closer.py @@ -18,7 +18,7 @@ from osa.job import are_all_jobs_correctly_finished, save_job_information from osa.nightsummary.extract import extract_runs, extract_sequences from osa.nightsummary.nightsummary import run_summary_table -from osa.paths import destination_dir +from osa.paths import destination_dir, create_longterm_symlink from osa.raw import is_raw_data_available from osa.report import start from osa.utils.cliopts import closercliparsing From d75579c44bf36c8fe9e99d190584ac949d8eecb3 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 13 Feb 2024 12:59:54 +0100 Subject: [PATCH 10/17] Remove comment --- src/osa/paths.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/osa/paths.py b/src/osa/paths.py index a07deb83..dba2b544 100644 --- a/src/osa/paths.py +++ b/src/osa/paths.py @@ -142,7 +142,6 @@ def get_calibration_filename(run_id: int, prod_id: str) -> Path: mongodb = cfg.get("database", "caco_db") try: # Cast run_id to int to avoid problems with numpy int64 encoding in MongoDB - #options.filters = find_filter_wheels(int(run_id), mongodb) options.filters = find_filter_wheels(int(run_id), mongodb) except IOError: log.warning("No filter information found in database. Assuming positions 52.") From b1cbdfbdd7115d0dd164d656af2fd9bd40b21275 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 13 Feb 2024 14:10:26 +0100 Subject: [PATCH 11/17] Create longterm directories for the tests --- src/osa/conftest.py | 14 ++++++++++++++ src/osa/scripts/tests/test_osa_scripts.py | 1 + 2 files changed, 15 insertions(+) diff --git a/src/osa/conftest.py b/src/osa/conftest.py index d341436f..d10c89c8 100644 --- a/src/osa/conftest.py +++ b/src/osa/conftest.py @@ -555,3 +555,17 @@ def database(base_test_dir): ) cursor.connection.commit() yield cursor + + +@pytest.fixture(scope="session") +def longterm_link_latest_dir(base_test_dir): + directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all" + directory.mkdir(parents=True, exist_ok=True) + return directory + + +@pytest.fixture(scope="session") +def longterm_dir(base_test_dir): + directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date + directory.mkdir(parents=True, exist_ok=True) + return directory \ No newline at end of file diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 5cb423a3..858075f1 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -200,6 +200,7 @@ def test_closer( assert obs_file.exists() assert merged_run_summary.exists() assert longterm_dir.exists() + assert longterm_link_latest_dir.exists() run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1") closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed" From 5eb337da4917d53c13eefd08838f317297f68f55 Mon Sep 17 00:00:00 2001 From: Maria Lainez <98marialainez@gmail.com> Date: Tue, 13 Feb 2024 14:22:25 +0100 Subject: [PATCH 12/17] Remove duplicated fixture + add necessary argument --- src/osa/conftest.py | 23 ++++++++--------------- src/osa/scripts/tests/test_osa_scripts.py | 1 + 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/osa/conftest.py b/src/osa/conftest.py index d10c89c8..773f77e5 100644 --- a/src/osa/conftest.py +++ b/src/osa/conftest.py @@ -482,7 +482,14 @@ def datacheck_dl1_files(base_test_dir): @pytest.fixture(scope="session") def longterm_dir(base_test_dir): - directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "v0.1.0" / "20200117" + directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date + directory.mkdir(parents=True, exist_ok=True) + return directory + + +@pytest.fixture(scope="session") +def longterm_link_latest_dir(base_test_dir): + directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all" directory.mkdir(parents=True, exist_ok=True) return directory @@ -555,17 +562,3 @@ def database(base_test_dir): ) cursor.connection.commit() yield cursor - - -@pytest.fixture(scope="session") -def longterm_link_latest_dir(base_test_dir): - directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / "night_wise" / "all" - directory.mkdir(parents=True, exist_ok=True) - return directory - - -@pytest.fixture(scope="session") -def longterm_dir(base_test_dir): - directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date - directory.mkdir(parents=True, exist_ok=True) - return directory \ No newline at end of file diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 858075f1..54c45759 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -180,6 +180,7 @@ def test_closer( systematic_correction_files, merged_run_summary, longterm_dir, + longterm_link_latest_dir, ): # First assure that the end of night flag is not set and remove it otherwise night_finished_flag = Path( From 9851c28fdc28b4591a1017bdc47d71f507e66a6d Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 13 Feb 2024 19:02:36 +0100 Subject: [PATCH 13/17] Update src/osa/scripts/tests/test_osa_scripts.py --- src/osa/scripts/tests/test_osa_scripts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 54c45759..4b1471ca 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -181,6 +181,7 @@ def test_closer( merged_run_summary, longterm_dir, longterm_link_latest_dir, + daily_datacheck_dl1_files, ): # First assure that the end of night flag is not set and remove it otherwise night_finished_flag = Path( From fbb19d388112d90f8a3cedaf7deec723fbb81bf6 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 13 Feb 2024 19:02:43 +0100 Subject: [PATCH 14/17] Update src/osa/scripts/tests/test_osa_scripts.py --- src/osa/scripts/tests/test_osa_scripts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 4b1471ca..1aee924f 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -203,6 +203,8 @@ def test_closer( assert merged_run_summary.exists() assert longterm_dir.exists() assert longterm_link_latest_dir.exists() + for check_file in daily_datacheck_dl1_files: + assert check_file.exists() run_program("closer", "-y", "-v", "-t", "-d", "2020-01-17", "LST1") closed_seq_file = running_analysis_dir / "sequence_LST1_01809.closed" From 4b704015701d2bd8571a46a6b1cd01fb44512bb2 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 13 Feb 2024 19:02:52 +0100 Subject: [PATCH 15/17] Update src/osa/paths.py --- src/osa/paths.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/osa/paths.py b/src/osa/paths.py index dba2b544..f7e4a478 100644 --- a/src/osa/paths.py +++ b/src/osa/paths.py @@ -365,5 +365,4 @@ def create_longterm_symlink(): latest_version_file = get_latest_version_file(all_longterm_files) log.info("Make symlink of the latest version longterm DL1 datacheck file in the common directory.") - linked_longterm_file.unlink() linked_longterm_file.symlink_to(latest_version_file) From 9b6bad926a569f5f24a6aa3f82f0f4824a3d82a1 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 13 Feb 2024 19:03:00 +0100 Subject: [PATCH 16/17] Update src/osa/conftest.py --- src/osa/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/conftest.py b/src/osa/conftest.py index 773f77e5..d1bdc52c 100644 --- a/src/osa/conftest.py +++ b/src/osa/conftest.py @@ -482,7 +482,7 @@ def datacheck_dl1_files(base_test_dir): @pytest.fixture(scope="session") def longterm_dir(base_test_dir): - directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date + directory = base_test_dir / "OSA" / "DL1DataCheck_LongTerm" / prod_id / date_to_dir(date) directory.mkdir(parents=True, exist_ok=True) return directory From de414165f554cc77e674069a434e9d658e758d7b Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Wed, 14 Feb 2024 11:15:03 +0100 Subject: [PATCH 17/17] avoid circular import between paths and utils --- src/osa/paths.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/osa/paths.py b/src/osa/paths.py index f7e4a478..b84b736e 100644 --- a/src/osa/paths.py +++ b/src/osa/paths.py @@ -17,7 +17,7 @@ from osa.configs.datamodel import Sequence from osa.utils import utils from osa.utils.logging import myLogger -from osa.utils.utils import date_to_dir + log = myLogger(logging.getLogger(__name__)) @@ -351,18 +351,22 @@ def create_source_directories(source_list: list, cuts_dir: Path): def get_latest_version_file(longterm_files: List[str]) -> Path: """Get the latest version path of the produced longterm DL1 datacheck files for a given date.""" - latest_version_path = max(longterm_files, key=lambda path: int(path.parents[1].name.split(".")[1]) if path.parents[1].name.startswith("v") else "") - return latest_version_path + return max( + longterm_files, + key=lambda path: int(path.parents[1].name.split(".")[1]) + if path.parents[1].name.startswith("v") + else "", + ) def create_longterm_symlink(): """If the created longterm DL1 datacheck file corresponds to the latest version available, make symlink to it in the "all" common directory.""" - nightdir = date_to_dir(options.date) + nightdir = utils.date_to_dir(options.date) longterm_dir = Path(cfg.get("LST1", "LONGTERM_DIR")) linked_longterm_file = longterm_dir / f"night_wise/all/DL1_datacheck_{nightdir}.h5" - all_longterm_files = sorted(longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5")) + all_longterm_files = longterm_dir.rglob(f"v*/{nightdir}/DL1_datacheck_{nightdir}.h5") latest_version_file = get_latest_version_file(all_longterm_files) - log.info("Make symlink of the latest version longterm DL1 datacheck file in the common directory.") + log.info("Symlink the latest version longterm DL1 datacheck file in the common directory.") linked_longterm_file.symlink_to(latest_version_file)