From 711b319ab9385507951f8748bbb59ebe27e29a51 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 1 Jul 2024 12:30:12 +0200 Subject: [PATCH 01/49] Create gainSel_webmaker.py --- src/osa/scripts/gainSel_webmaker.py | 168 ++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100644 src/osa/scripts/gainSel_webmaker.py diff --git a/src/osa/scripts/gainSel_webmaker.py b/src/osa/scripts/gainSel_webmaker.py new file mode 100644 index 00000000..e41958aa --- /dev/null +++ b/src/osa/scripts/gainSel_webmaker.py @@ -0,0 +1,168 @@ +"""Produce the HTML file with the processing status from the sequencer report.""" + + +import logging +import subprocess as sp +import sys +from datetime import datetime, timedelta +from pathlib import Path +from textwrap import dedent +from typing import Iterable + +import pandas as pd + +from osa.configs import options +from osa.configs.config import cfg +from osa.utils.cliopts import sequencer_webmaker_argparser +from osa.utils.logging import myLogger +from osa.utils.utils import is_day_closed, date_to_iso, date_to_dir + +log = myLogger(logging.getLogger()) + + +def html_content(body: str, date: str) -> str: + """Build the HTML content. + + Parameters + ---------- + body : str + Table with the sequencer status report. + date : str + Date of the processing YYYY-MM-DD. + + Returns + ------- + str + HTML content. + """ + time_update = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + + return dedent( + f""" + + + + OSA Sequencer status + + +

OSA processing status

+

Processing data from: {date}. Last updated: {time_update} UTC

+ {body} + + """ + ) + + +def get_sequencer_output(date: str, config: str, test=False, no_gainsel=False) -> list: + """Call sequencer to get table with the sequencer status report. + + Parameters + ---------- + date : str + Date of the processing YYYY-MM-DD. + config : str + OSA configuration file to use. + test : bool + + Returns + ------- + list + Lines of the sequencer output. + """ + log.info("Calling sequencer...") + commandargs = [ + "sequencer", + "-c", + config, + "-s", + "-d", + date, + options.tel_id, + ] + + if no_gainsel: + commandargs.insert(1, "--no-gainsel") + + if test: + commandargs.insert(-1, "-t") + + try: + output = sp.run(commandargs, stdout=sp.PIPE, stderr=sp.STDOUT, encoding="utf-8", check=True) + except sp.CalledProcessError as error: + log.error(f"Command {commandargs} failed, {error.returncode}") + sys.exit(1) + else: + # Strip newlines and fit it into a table: + return output.stdout.splitlines() + + +def lines_to_matrix(lines: Iterable) -> list: + """Build the matrix from the sequencer output lines.""" + matrix = [] + for line in lines: + l_fields = line.split() + if len(l_fields) == 18: + matrix.append(l_fields) + return matrix + + +def matrix_to_html(matrix: list) -> str: + """Build the html table with the sequencer status report.""" + log.info("Building the html table from sequencer output") + if len(matrix) < 2: + return "

No data found

" + df = pd.DataFrame(matrix[1:], columns=matrix[0]) + return df.to_html(index=False) + + +def main(): + """Produce the html file with the processing status from the sequencer report.""" + + log.setLevel(logging.INFO) + + args = sequencer_webmaker_argparser().parse_args() + + if args.date: + flat_date = date_to_dir(args.date) + options.date = args.date + + else: + # yesterday by default + yesterday = datetime.now() - timedelta(days=1) + options.date = yesterday + flat_date = date_to_dir(yesterday) + + date = date_to_iso(options.date) + + if is_day_closed(): + log.info(f"Date {date} is already closed for {options.tel_id}") + sys.exit(1) + + run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR")) + run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv" + if not run_summary_file.is_file(): + log.error(f"No RunSummary file found for {date}") + sys.exit(1) + + # Get the table with the sequencer status report: + lines = get_sequencer_output(date, args.config, test=args.test, no_gainsel=args.no_gainsel) + + # Build the html sequencer table that will be place in the body of the HTML file + matrix = lines_to_matrix(lines) + html_table = matrix_to_html(matrix) + + # Save the HTML file + log.info("Saving the HTML file") + directory = Path(cfg.get("LST1", "SEQUENCER_WEB_DIR")) + directory.mkdir(parents=True, exist_ok=True) + + html_file = directory / Path(f"osa_status_{flat_date}.html") + html_file.write_text(html_content(html_table, date), encoding="utf-8") + + log.info("Done") + + +if __name__ == "__main__": + main() From e9c4e8b2922e866853ba35e066ac6723a7f6b777 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 11:09:29 +0200 Subject: [PATCH 02/49] Delete src/osa/scripts/gainSel_webmaker.py --- src/osa/scripts/gainSel_webmaker.py | 168 ---------------------------- 1 file changed, 168 deletions(-) delete mode 100644 src/osa/scripts/gainSel_webmaker.py diff --git a/src/osa/scripts/gainSel_webmaker.py b/src/osa/scripts/gainSel_webmaker.py deleted file mode 100644 index e41958aa..00000000 --- a/src/osa/scripts/gainSel_webmaker.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Produce the HTML file with the processing status from the sequencer report.""" - - -import logging -import subprocess as sp -import sys -from datetime import datetime, timedelta -from pathlib import Path -from textwrap import dedent -from typing import Iterable - -import pandas as pd - -from osa.configs import options -from osa.configs.config import cfg -from osa.utils.cliopts import sequencer_webmaker_argparser -from osa.utils.logging import myLogger -from osa.utils.utils import is_day_closed, date_to_iso, date_to_dir - -log = myLogger(logging.getLogger()) - - -def html_content(body: str, date: str) -> str: - """Build the HTML content. - - Parameters - ---------- - body : str - Table with the sequencer status report. - date : str - Date of the processing YYYY-MM-DD. - - Returns - ------- - str - HTML content. - """ - time_update = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") - - return dedent( - f""" - - - - OSA Sequencer status - - -

OSA processing status

-

Processing data from: {date}. Last updated: {time_update} UTC

- {body} - - """ - ) - - -def get_sequencer_output(date: str, config: str, test=False, no_gainsel=False) -> list: - """Call sequencer to get table with the sequencer status report. - - Parameters - ---------- - date : str - Date of the processing YYYY-MM-DD. - config : str - OSA configuration file to use. - test : bool - - Returns - ------- - list - Lines of the sequencer output. - """ - log.info("Calling sequencer...") - commandargs = [ - "sequencer", - "-c", - config, - "-s", - "-d", - date, - options.tel_id, - ] - - if no_gainsel: - commandargs.insert(1, "--no-gainsel") - - if test: - commandargs.insert(-1, "-t") - - try: - output = sp.run(commandargs, stdout=sp.PIPE, stderr=sp.STDOUT, encoding="utf-8", check=True) - except sp.CalledProcessError as error: - log.error(f"Command {commandargs} failed, {error.returncode}") - sys.exit(1) - else: - # Strip newlines and fit it into a table: - return output.stdout.splitlines() - - -def lines_to_matrix(lines: Iterable) -> list: - """Build the matrix from the sequencer output lines.""" - matrix = [] - for line in lines: - l_fields = line.split() - if len(l_fields) == 18: - matrix.append(l_fields) - return matrix - - -def matrix_to_html(matrix: list) -> str: - """Build the html table with the sequencer status report.""" - log.info("Building the html table from sequencer output") - if len(matrix) < 2: - return "

No data found

" - df = pd.DataFrame(matrix[1:], columns=matrix[0]) - return df.to_html(index=False) - - -def main(): - """Produce the html file with the processing status from the sequencer report.""" - - log.setLevel(logging.INFO) - - args = sequencer_webmaker_argparser().parse_args() - - if args.date: - flat_date = date_to_dir(args.date) - options.date = args.date - - else: - # yesterday by default - yesterday = datetime.now() - timedelta(days=1) - options.date = yesterday - flat_date = date_to_dir(yesterday) - - date = date_to_iso(options.date) - - if is_day_closed(): - log.info(f"Date {date} is already closed for {options.tel_id}") - sys.exit(1) - - run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR")) - run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv" - if not run_summary_file.is_file(): - log.error(f"No RunSummary file found for {date}") - sys.exit(1) - - # Get the table with the sequencer status report: - lines = get_sequencer_output(date, args.config, test=args.test, no_gainsel=args.no_gainsel) - - # Build the html sequencer table that will be place in the body of the HTML file - matrix = lines_to_matrix(lines) - html_table = matrix_to_html(matrix) - - # Save the HTML file - log.info("Saving the HTML file") - directory = Path(cfg.get("LST1", "SEQUENCER_WEB_DIR")) - directory.mkdir(parents=True, exist_ok=True) - - html_file = directory / Path(f"osa_status_{flat_date}.html") - html_file.write_text(html_content(html_table, date), encoding="utf-8") - - log.info("Done") - - -if __name__ == "__main__": - main() From ff207d297b9e889da6eab6cc11bcff449c29ad31 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 11:09:45 +0200 Subject: [PATCH 03/49] Create gainSelWebmaker.py --- src/osa/scripts/gainSelWebmaker.py | 178 +++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 src/osa/scripts/gainSelWebmaker.py diff --git a/src/osa/scripts/gainSelWebmaker.py b/src/osa/scripts/gainSelWebmaker.py new file mode 100644 index 00000000..1dd1059b --- /dev/null +++ b/src/osa/scripts/gainSelWebmaker.py @@ -0,0 +1,178 @@ + +import logging +import re +import glob +from pathlib import Path +import argparse +import sys +from textwrap import dedent + +from astropy.table import Table +from datetime import datetime, timedelta +from osa.utils.utils import date_to_dir, date_to_iso +from osa.configs.config import cfg +from osa.paths import DEFAULT_CFG +from osa.nightsummary.nightsummary import run_summary_table +import numpy as np +import pandas as pd +from typing import Iterable +from argparse import ArgumentParser +from osa.configs import options + +def valid_date(string): + """Check if the string is a valid date and return a datetime object.""" + return datetime.strptime(string, "%Y%m%d") + +common_parser = ArgumentParser(add_help=False) +common_parser.add_argument( + "-c", + "--config", + type=Path, + default=DEFAULT_CFG, + help="Use specific config file [default configs/sequencer.cfg]", +) +common_parser.add_argument( + "-d", + "--date", + help="Date (YYYYMMDD) of the start of the night", + type=valid_date, +) + + + +def html_content(body: str, date: str) -> str: + """Build the HTML content. + + Parameters + ---------- + body : str + Table with the sequencer status report. + date : str + Date of the processing YYYY-MM-DD. + + Returns + ------- + str + HTML content. + """ + time_update = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + + return dedent( + f""" + + + + OSA Gain Selection status + + +

OSA Gain Selection processing status

+

Processing data from: {date}. Last updated: {time_update} UTC

+ {body} + + """ + ) + + +def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: + """Search for failed jobs in the log directory.""" + base_dir = Path("/fefs/aswg/data/real") #Path(cfg.get("LST1", "BASE")) + log_dir = base_dir / f"R0G/log/{date}" + history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history") + + success_subruns = 0 + failed_subruns = 0 + pending_subruns = 0 + + for file in history_files: + match = re.search(f"gain_selection_{run_id:05d}.(\d+).history", str(file)) + subrun = match.group(1) + if file.read_text() != "": + gainsel_rc = file.read_text().splitlines()[-1][-1] + + if gainsel_rc == "1": + failed_subruns = failed_subruns+1 + + elif gainsel_rc == "0": + success_subruns = success_subruns+1 + else: + pending_subruns = pending_subruns+1 + return [pending_subruns, success_subruns, failed_subruns] + + + +def check_failed_jobs(date: str): + """Search for failed jobs in the log directory.""" + summary_table = run_summary_table(datetime.fromisoformat(date)) + data_runs = summary_table[summary_table["run_type"] == "DATA"] + + gainsel_summary = [] + for run in data_runs: + run_id = run["run_id"] + checkgainsel = check_gainsel_jobs_runwise(date.replace('-',''), run_id) + gainsel_summary.append([run_id, checkgainsel[0], checkgainsel[1], checkgainsel[2]]) + + gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) + gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, 'FAILED', np.where(gainsel_df['pending'] != 0, 'PENDING', 'COMPLETED')) + gainsel_df['GainSel%'] = round(gainsel_df['success']*100/(gainsel_df['pending']+gainsel_df['failed']+gainsel_df['success']) +,1) + runs = summary_table["run_id"] + summary_table = summary_table.to_pandas() + final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] + + return final_table + +def main(): + """Produce the html file with the processing OSA Gain Selection status.""" + args = ArgumentParser( + description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] + ).parse_args() + + + html_table = '' + + if args.date: + flat_date = date_to_dir(args.date) + options.date = args.date + + else: + # yesterday by default + yesterday = datetime.now() - timedelta(days=1) + options.date = yesterday + flat_date = date_to_dir(yesterday) + + date = date_to_iso(options.date) + + run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR")) + run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv" + + if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0: + + html_table = "

No data found

" + # Save the HTML file + directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR"$ + directory.mkdir(parents=True, exist_ok=True) + html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") + html_file.write_text(html_content(html_table, date), encoding="utf-8") + + else: + # Get the table with the sequencer status report: + lines = check_failed_jobs(date) + + + lines.reset_index(drop=True, inplace=True) + if html_table == '': + html_table = lines.to_html() + + # Save the HTML file + directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR")) + + directory.mkdir(parents=True, exist_ok=True) + + html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") + html_file.write_text(html_content(html_table, date), encoding="utf-8") + +if __name__ == "__main__": + main() + From 73b3bf9f993c7597a0f8c65db5f7e09eeee94382 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 11:12:37 +0200 Subject: [PATCH 04/49] Create gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 177 ++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 src/osa/scripts/gainsel_webmaker.py diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py new file mode 100644 index 00000000..9eaf7866 --- /dev/null +++ b/src/osa/scripts/gainsel_webmaker.py @@ -0,0 +1,177 @@ +import logging +import re +import glob +from pathlib import Path +import argparse +import sys +from textwrap import dedent + +from astropy.table import Table +from datetime import datetime, timedelta + +from osa.utils.utils import date_to_dir, date_to_iso +from osa.configs.config import cfg +from osa.paths import DEFAULT_CFG +from osa.nightsummary.nightsummary import run_summary_table +import numpy as np +import pandas as pd +from typing import Iterable +from argparse import ArgumentParser +from osa.configs import options + +def valid_date(string): + """Check if the string is a valid date and return a datetime object.""" + return datetime.strptime(string, "%Y%m%d") + +common_parser = ArgumentParser(add_help=False) +common_parser.add_argument( + "-c", + "--config", + type=Path, + default=DEFAULT_CFG, + help="Use specific config file [default configs/sequencer.cfg]", +) +common_parser.add_argument( + "-d", + "--date", + help="Date (YYYYMMDD) of the start of the night", + type=valid_date, +) + + + +def html_content(body: str, date: str) -> str: + """Build the HTML content. + + Parameters + ---------- + body : str + Table with the sequencer status report. + date : str + Date of the processing YYYY-MM-DD. + + Returns + ------- + str + HTML content. + """ + time_update = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + + return dedent( + f""" + + + + OSA Gain Selection status + + +

OSA Gain Selection processing status

+

Processing data from: {date}. Last updated: {time_update} UTC

+ {body} + + """ + ) + + +def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: + """Search for failed jobs in the log directory.""" + base_dir = Path("/fefs/aswg/data/real") #Path(cfg.get("LST1", "BASE")) + log_dir = base_dir / f"R0G/log/{date}" + history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history") + + success_subruns = 0 + failed_subruns = 0 + pending_subruns = 0 + + for file in history_files: + match = re.search(f"gain_selection_{run_id:05d}.(\d+).history", str(file)) + subrun = match.group(1) + if file.read_text() != "": + gainsel_rc = file.read_text().splitlines()[-1][-1] + + if gainsel_rc == "1": + failed_subruns = failed_subruns+1 + + elif gainsel_rc == "0": + success_subruns = success_subruns+1 + else: + pending_subruns = pending_subruns+1 + return [pending_subruns, success_subruns, failed_subruns] + + + +def check_failed_jobs(date: str): + """Search for failed jobs in the log directory.""" + summary_table = run_summary_table(datetime.fromisoformat(date)) + data_runs = summary_table[summary_table["run_type"] == "DATA"] + + gainsel_summary = [] + for run in data_runs: + run_id = run["run_id"] + checkgainsel = check_gainsel_jobs_runwise(date.replace('-',''), run_id) + gainsel_summary.append([run_id, checkgainsel[0], checkgainsel[1], checkgainsel[2]]) + + gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) + gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, 'FAILED', np.where(gainsel_df['pending'] != 0, 'PENDING', 'COMPLETED')) + gainsel_df['GainSel%'] = round(gainsel_df['success']*100/(gainsel_df['pending']+gainsel_df['failed']+gainsel_df['success']) +,1) + runs = summary_table["run_id"] + summary_table = summary_table.to_pandas() + final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] + + return final_table + +def main(): + """Produce the html file with the processing OSA Gain Selection status.""" + args = ArgumentParser( + description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] + ).parse_args() + + + html_table = '' + + if args.date: + flat_date = date_to_dir(args.date) + options.date = args.date + + else: + # yesterday by default + yesterday = datetime.now() - timedelta(days=1) + options.date = yesterday + flat_date = date_to_dir(yesterday) + + date = date_to_iso(options.date) + + run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR")) + run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv" + + if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0: + + html_table = "

No data found

" + # Save the HTML file + directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR"$ + directory.mkdir(parents=True, exist_ok=True) + html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") + html_file.write_text(html_content(html_table, date), encoding="utf-8") + + else: + # Get the table with the sequencer status report: + lines = check_failed_jobs(date) + + + lines.reset_index(drop=True, inplace=True) + if html_table == '': + html_table = lines.to_html() + + # Save the HTML file + directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR")) + + directory.mkdir(parents=True, exist_ok=True) + + html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") + html_file.write_text(html_content(html_table, date), encoding="utf-8") + +if __name__ == "__main__": + main() From d2c31a07e17c1ee535027301960d4c9d51c8018c Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 11:13:29 +0200 Subject: [PATCH 05/49] Delete src/osa/scripts/gainSelWebmaker.py --- src/osa/scripts/gainSelWebmaker.py | 178 ----------------------------- 1 file changed, 178 deletions(-) delete mode 100644 src/osa/scripts/gainSelWebmaker.py diff --git a/src/osa/scripts/gainSelWebmaker.py b/src/osa/scripts/gainSelWebmaker.py deleted file mode 100644 index 1dd1059b..00000000 --- a/src/osa/scripts/gainSelWebmaker.py +++ /dev/null @@ -1,178 +0,0 @@ - -import logging -import re -import glob -from pathlib import Path -import argparse -import sys -from textwrap import dedent - -from astropy.table import Table -from datetime import datetime, timedelta -from osa.utils.utils import date_to_dir, date_to_iso -from osa.configs.config import cfg -from osa.paths import DEFAULT_CFG -from osa.nightsummary.nightsummary import run_summary_table -import numpy as np -import pandas as pd -from typing import Iterable -from argparse import ArgumentParser -from osa.configs import options - -def valid_date(string): - """Check if the string is a valid date and return a datetime object.""" - return datetime.strptime(string, "%Y%m%d") - -common_parser = ArgumentParser(add_help=False) -common_parser.add_argument( - "-c", - "--config", - type=Path, - default=DEFAULT_CFG, - help="Use specific config file [default configs/sequencer.cfg]", -) -common_parser.add_argument( - "-d", - "--date", - help="Date (YYYYMMDD) of the start of the night", - type=valid_date, -) - - - -def html_content(body: str, date: str) -> str: - """Build the HTML content. - - Parameters - ---------- - body : str - Table with the sequencer status report. - date : str - Date of the processing YYYY-MM-DD. - - Returns - ------- - str - HTML content. - """ - time_update = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") - - return dedent( - f""" - - - - OSA Gain Selection status - - -

OSA Gain Selection processing status

-

Processing data from: {date}. Last updated: {time_update} UTC

- {body} - - """ - ) - - -def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: - """Search for failed jobs in the log directory.""" - base_dir = Path("/fefs/aswg/data/real") #Path(cfg.get("LST1", "BASE")) - log_dir = base_dir / f"R0G/log/{date}" - history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history") - - success_subruns = 0 - failed_subruns = 0 - pending_subruns = 0 - - for file in history_files: - match = re.search(f"gain_selection_{run_id:05d}.(\d+).history", str(file)) - subrun = match.group(1) - if file.read_text() != "": - gainsel_rc = file.read_text().splitlines()[-1][-1] - - if gainsel_rc == "1": - failed_subruns = failed_subruns+1 - - elif gainsel_rc == "0": - success_subruns = success_subruns+1 - else: - pending_subruns = pending_subruns+1 - return [pending_subruns, success_subruns, failed_subruns] - - - -def check_failed_jobs(date: str): - """Search for failed jobs in the log directory.""" - summary_table = run_summary_table(datetime.fromisoformat(date)) - data_runs = summary_table[summary_table["run_type"] == "DATA"] - - gainsel_summary = [] - for run in data_runs: - run_id = run["run_id"] - checkgainsel = check_gainsel_jobs_runwise(date.replace('-',''), run_id) - gainsel_summary.append([run_id, checkgainsel[0], checkgainsel[1], checkgainsel[2]]) - - gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) - gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, 'FAILED', np.where(gainsel_df['pending'] != 0, 'PENDING', 'COMPLETED')) - gainsel_df['GainSel%'] = round(gainsel_df['success']*100/(gainsel_df['pending']+gainsel_df['failed']+gainsel_df['success']) -,1) - runs = summary_table["run_id"] - summary_table = summary_table.to_pandas() - final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] - - return final_table - -def main(): - """Produce the html file with the processing OSA Gain Selection status.""" - args = ArgumentParser( - description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] - ).parse_args() - - - html_table = '' - - if args.date: - flat_date = date_to_dir(args.date) - options.date = args.date - - else: - # yesterday by default - yesterday = datetime.now() - timedelta(days=1) - options.date = yesterday - flat_date = date_to_dir(yesterday) - - date = date_to_iso(options.date) - - run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR")) - run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv" - - if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0: - - html_table = "

No data found

" - # Save the HTML file - directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR"$ - directory.mkdir(parents=True, exist_ok=True) - html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") - html_file.write_text(html_content(html_table, date), encoding="utf-8") - - else: - # Get the table with the sequencer status report: - lines = check_failed_jobs(date) - - - lines.reset_index(drop=True, inplace=True) - if html_table == '': - html_table = lines.to_html() - - # Save the HTML file - directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR")) - - directory.mkdir(parents=True, exist_ok=True) - - html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") - html_file.write_text(html_content(html_table, date), encoding="utf-8") - -if __name__ == "__main__": - main() - From 14f9c57aa4a54f0000a6bf182c38521a3238f286 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 13:45:25 +0200 Subject: [PATCH 06/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 9eaf7866..a0f68185 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -1,9 +1,6 @@ -import logging import re -import glob +from glob import glob from pathlib import Path -import argparse -import sys from textwrap import dedent from astropy.table import Table @@ -15,7 +12,6 @@ from osa.nightsummary.nightsummary import run_summary_table import numpy as np import pandas as pd -from typing import Iterable from argparse import ArgumentParser from osa.configs import options @@ -87,7 +83,6 @@ def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: for file in history_files: match = re.search(f"gain_selection_{run_id:05d}.(\d+).history", str(file)) - subrun = match.group(1) if file.read_text() != "": gainsel_rc = file.read_text().splitlines()[-1][-1] @@ -117,7 +112,6 @@ def check_failed_jobs(date: str): gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, 'FAILED', np.where(gainsel_df['pending'] != 0, 'PENDING', 'COMPLETED')) gainsel_df['GainSel%'] = round(gainsel_df['success']*100/(gainsel_df['pending']+gainsel_df['failed']+gainsel_df['success']) ,1) - runs = summary_table["run_id"] summary_table = summary_table.to_pandas() final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] From 17fd64011e0202542cf93b2766c1e5952c8a6ee5 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 13:51:54 +0200 Subject: [PATCH 07/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index a0f68185..f93cef1c 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -1,5 +1,4 @@ import re -from glob import glob from pathlib import Path from textwrap import dedent @@ -82,7 +81,6 @@ def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: pending_subruns = 0 for file in history_files: - match = re.search(f"gain_selection_{run_id:05d}.(\d+).history", str(file)) if file.read_text() != "": gainsel_rc = file.read_text().splitlines()[-1][-1] From eba84c65adf6f03cd5f2a2ce016540a6257c9839 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 13:53:24 +0200 Subject: [PATCH 08/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index f93cef1c..95b2d055 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -1,4 +1,3 @@ -import re from pathlib import Path from textwrap import dedent From f3f228f649c1f7dc536e20962cfdc9b759c34ece Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 14:43:57 +0200 Subject: [PATCH 09/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 95b2d055..6a244bd6 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -1,9 +1,7 @@ from pathlib import Path from textwrap import dedent - from astropy.table import Table from datetime import datetime, timedelta - from osa.utils.utils import date_to_dir, date_to_iso from osa.configs.config import cfg from osa.paths import DEFAULT_CFG @@ -16,7 +14,8 @@ def valid_date(string): """Check if the string is a valid date and return a datetime object.""" return datetime.strptime(string, "%Y%m%d") - + + common_parser = ArgumentParser(add_help=False) common_parser.add_argument( "-c", @@ -32,8 +31,6 @@ def valid_date(string): type=valid_date, ) - - def html_content(body: str, date: str) -> str: """Build the HTML content. From 6d8988b38411150ebbe199068d9a62fee83f9e17 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 14:46:42 +0200 Subject: [PATCH 10/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 6a244bd6..9fb04e21 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -65,7 +65,6 @@ def html_content(body: str, date: str) -> str: """ ) - def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: """Search for failed jobs in the log directory.""" base_dir = Path("/fefs/aswg/data/real") #Path(cfg.get("LST1", "BASE")) @@ -110,13 +109,12 @@ def check_failed_jobs(date: str): final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] return final_table - + def main(): """Produce the html file with the processing OSA Gain Selection status.""" args = ArgumentParser( description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] ).parse_args() - html_table = '' From 886f9e7d62ebb7973eab4571f7242d5ec395194e Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 14:51:09 +0200 Subject: [PATCH 11/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 31 +++++++++++++---------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 9fb04e21..c606626f 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -74,11 +74,11 @@ def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: success_subruns = 0 failed_subruns = 0 pending_subruns = 0 - + for file in history_files: if file.read_text() != "": gainsel_rc = file.read_text().splitlines()[-1][-1] - + if gainsel_rc == "1": failed_subruns = failed_subruns+1 @@ -88,8 +88,6 @@ def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: pending_subruns = pending_subruns+1 return [pending_subruns, success_subruns, failed_subruns] - - def check_failed_jobs(date: str): """Search for failed jobs in the log directory.""" summary_table = run_summary_table(datetime.fromisoformat(date)) @@ -100,7 +98,7 @@ def check_failed_jobs(date: str): run_id = run["run_id"] checkgainsel = check_gainsel_jobs_runwise(date.replace('-',''), run_id) gainsel_summary.append([run_id, checkgainsel[0], checkgainsel[1], checkgainsel[2]]) - + gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, 'FAILED', np.where(gainsel_df['pending'] != 0, 'PENDING', 'COMPLETED')) gainsel_df['GainSel%'] = round(gainsel_df['success']*100/(gainsel_df['pending']+gainsel_df['failed']+gainsel_df['success']) @@ -109,19 +107,19 @@ def check_failed_jobs(date: str): final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] return final_table - + def main(): """Produce the html file with the processing OSA Gain Selection status.""" args = ArgumentParser( description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] ).parse_args() - + html_table = '' - + if args.date: flat_date = date_to_dir(args.date) options.date = args.date - + else: # yesterday by default yesterday = datetime.now() - timedelta(days=1) @@ -129,35 +127,34 @@ def main(): flat_date = date_to_dir(yesterday) date = date_to_iso(options.date) - + run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR")) run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv" - + if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0: - + html_table = "

No data found

" # Save the HTML file directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR"$ directory.mkdir(parents=True, exist_ok=True) html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") html_file.write_text(html_content(html_table, date), encoding="utf-8") - + else: # Get the table with the sequencer status report: lines = check_failed_jobs(date) - lines.reset_index(drop=True, inplace=True) if html_table == '': html_table = lines.to_html() - + # Save the HTML file directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR")) - + directory.mkdir(parents=True, exist_ok=True) html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") html_file.write_text(html_content(html_table, date), encoding="utf-8") - + if __name__ == "__main__": main() From 29a35422240b3ae7b4952593ab640fc3b0455d5a Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 14:53:43 +0200 Subject: [PATCH 12/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index c606626f..ad0b3deb 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -105,15 +105,15 @@ def check_failed_jobs(date: str): ,1) summary_table = summary_table.to_pandas() final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] - - return final_table + return final_table + def main(): """Produce the html file with the processing OSA Gain Selection status.""" args = ArgumentParser( description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] ).parse_args() - + html_table = '' if args.date: From 421c7ed68faa82818643a04d75889026d6ef180d Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 14:56:38 +0200 Subject: [PATCH 13/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index ad0b3deb..1af24d4a 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -107,19 +107,19 @@ def check_failed_jobs(date: str): final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] return final_table - + def main(): """Produce the html file with the processing OSA Gain Selection status.""" args = ArgumentParser( description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] ).parse_args() - + html_table = '' if args.date: flat_date = date_to_dir(args.date) options.date = args.date - + else: # yesterday by default yesterday = datetime.now() - timedelta(days=1) From f3ae27a3a4df6c0266f853951c89790806764bf8 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 16:36:18 +0200 Subject: [PATCH 14/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 1af24d4a..256d9f88 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -67,7 +67,7 @@ def html_content(body: str, date: str) -> str: def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: """Search for failed jobs in the log directory.""" - base_dir = Path("/fefs/aswg/data/real") #Path(cfg.get("LST1", "BASE")) + base_dir = Path(cfg.get("LST1", "BASE")) log_dir = base_dir / f"R0G/log/{date}" history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history") @@ -135,7 +135,7 @@ def main(): html_table = "

No data found

" # Save the HTML file - directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR"$ + directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) directory.mkdir(parents=True, exist_ok=True) html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") html_file.write_text(html_content(html_table, date), encoding="utf-8") From 74b4488c5da23fa88ec07fe084f4384bb2ba87ab Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 16:41:31 +0200 Subject: [PATCH 15/49] Update sequencer_webmaker.py --- src/osa/scripts/sequencer_webmaker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/osa/scripts/sequencer_webmaker.py b/src/osa/scripts/sequencer_webmaker.py index e41958aa..54b123ba 100644 --- a/src/osa/scripts/sequencer_webmaker.py +++ b/src/osa/scripts/sequencer_webmaker.py @@ -20,7 +20,7 @@ log = myLogger(logging.getLogger()) -def html_content(body: str, date: str) -> str: +def html_content(body: str, date: str, title: str) -> str: """Build the HTML content. Parameters @@ -43,11 +43,11 @@ def html_content(body: str, date: str) -> str: - OSA Sequencer status{title} status -

OSA processing status

+

{title} processing status

Processing data from: {date}. Last updated: {time_update} UTC

{body} @@ -159,7 +159,7 @@ def main(): directory.mkdir(parents=True, exist_ok=True) html_file = directory / Path(f"osa_status_{flat_date}.html") - html_file.write_text(html_content(html_table, date), encoding="utf-8") + html_file.write_text(html_content(html_table, date, "OSA Sequencer"), encoding="utf-8") log.info("Done") From dc8ff36949b1f868580e2e3347722ff026139135 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 16:43:17 +0200 Subject: [PATCH 16/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 40 +++-------------------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 256d9f88..35c1326d 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -10,6 +10,7 @@ import pandas as pd from argparse import ArgumentParser from osa.configs import options +from osa.scripts.sequencer_webmaker import html_content def valid_date(string): """Check if the string is a valid date and return a datetime object.""" @@ -31,39 +32,6 @@ def valid_date(string): type=valid_date, ) -def html_content(body: str, date: str) -> str: - """Build the HTML content. - - Parameters - ---------- - body : str - Table with the sequencer status report. - date : str - Date of the processing YYYY-MM-DD. - - Returns - ------- - str - HTML content. - """ - time_update = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") - - return dedent( - f""" - - - - OSA Gain Selection status - - -

OSA Gain Selection processing status

-

Processing data from: {date}. Last updated: {time_update} UTC

- {body} - - """ - ) def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: """Search for failed jobs in the log directory.""" @@ -138,7 +106,7 @@ def main(): directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) directory.mkdir(parents=True, exist_ok=True) html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") - html_file.write_text(html_content(html_table, date), encoding="utf-8") + html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") else: # Get the table with the sequencer status report: @@ -149,12 +117,12 @@ def main(): html_table = lines.to_html() # Save the HTML file - directory = Path("/fefs/aswg/data/real/OSA/GainSelWeb")#Path(cfg.get("LST1", "SEQUENCER_WEB_DIR")) + directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) directory.mkdir(parents=True, exist_ok=True) html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") - html_file.write_text(html_content(html_table, date), encoding="utf-8") + html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") if __name__ == "__main__": main() From e1e085f76f7743f5085473c561366621ac3d9b48 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Wed, 3 Jul 2024 16:45:01 +0200 Subject: [PATCH 17/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 35c1326d..5604e2ea 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -1,5 +1,4 @@ from pathlib import Path -from textwrap import dedent from astropy.table import Table from datetime import datetime, timedelta from osa.utils.utils import date_to_dir, date_to_iso From b21201a4098618caa7df08e4f67108d80c30506d Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Thu, 4 Jul 2024 10:44:28 +0200 Subject: [PATCH 18/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 5604e2ea..d26a165e 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -104,7 +104,7 @@ def main(): # Save the HTML file directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) directory.mkdir(parents=True, exist_ok=True) - html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") + html_file = directory / f"osa_gainsel_status_{flat_date}.html" html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") else: From a5da4c8656b5602cd5b263a6950367ab92c36f3a Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Thu, 4 Jul 2024 10:44:40 +0200 Subject: [PATCH 19/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index d26a165e..42b10fa3 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -108,7 +108,7 @@ def main(): html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") else: - # Get the table with the sequencer status report: + # Get the table with the gain selection check report: lines = check_failed_jobs(date) lines.reset_index(drop=True, inplace=True) From 9f12973117ffc85edba824af912f734c7ceadc3e Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Thu, 4 Jul 2024 10:45:43 +0200 Subject: [PATCH 20/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 42b10fa3..8c66df82 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -120,7 +120,7 @@ def main(): directory.mkdir(parents=True, exist_ok=True) - html_file = directory / Path(f"osa_gainsel_status_{flat_date}.html") + html_file = directory / f"osa_gainsel_status_{flat_date}.html" html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") if __name__ == "__main__": From 5f45df38346c5c57778c4cc8fce5116412a66d50 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Thu, 4 Jul 2024 10:49:28 +0200 Subject: [PATCH 21/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 8c66df82..26a6b621 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -63,8 +63,8 @@ def check_failed_jobs(date: str): gainsel_summary = [] for run in data_runs: run_id = run["run_id"] - checkgainsel = check_gainsel_jobs_runwise(date.replace('-',''), run_id) - gainsel_summary.append([run_id, checkgainsel[0], checkgainsel[1], checkgainsel[2]]) + gainsel_job_status = check_gainsel_jobs_runwise(date.replace('-',''), run_id) + gainsel_summary.append([run_id, gainsel_job_status]) gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, 'FAILED', np.where(gainsel_df['pending'] != 0, 'PENDING', 'COMPLETED')) @@ -81,8 +81,6 @@ def main(): description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] ).parse_args() - html_table = '' - if args.date: flat_date = date_to_dir(args.date) options.date = args.date @@ -109,11 +107,10 @@ def main(): else: # Get the table with the gain selection check report: - lines = check_failed_jobs(date) + table_gain_selection_jobs = check_failed_jobs(date) - lines.reset_index(drop=True, inplace=True) - if html_table == '': - html_table = lines.to_html() + table_gain_selection_jobs.reset_index(drop=True, inplace=True) + html_table = table_gain_selection_jobs.to_html() # Save the HTML file directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) From e92e8efa3634d647ed698a26f78a5331811b21f3 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Thu, 4 Jul 2024 12:59:45 +0200 Subject: [PATCH 22/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 26a6b621..d170c98a 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -67,11 +67,25 @@ def check_failed_jobs(date: str): gainsel_summary.append([run_id, gainsel_job_status]) gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) - gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, 'FAILED', np.where(gainsel_df['pending'] != 0, 'PENDING', 'COMPLETED')) + gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, + 'FAILED', + np.where(gainsel_df['pending'] != 0, + 'PENDING', + 'COMPLETED')) + gainsel_df['GainSel%'] = round(gainsel_df['success']*100/(gainsel_df['pending']+gainsel_df['failed']+gainsel_df['success']) ,1) + summary_table = summary_table.to_pandas() - final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id','n_subruns','run_type','pending','success','failed','GainSelStatus', 'GainSel%']] + + final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id', + 'n_subruns', + 'run_type', + 'pending', + 'success', + 'failed', + 'GainSelStatus', + 'GainSel%']] return final_table From 13096ede99d346e1168baf20ee4efa754f5d34a4 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 10:16:12 +0200 Subject: [PATCH 23/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index d170c98a..b9aaffb0 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -73,8 +73,8 @@ def check_failed_jobs(date: str): 'PENDING', 'COMPLETED')) - gainsel_df['GainSel%'] = round(gainsel_df['success']*100/(gainsel_df['pending']+gainsel_df['failed']+gainsel_df['success']) -,1) + total_job_number = gainsel_df['pending'] + gainsel_df['failed'] + gainsel_df['success'] + gainsel_df['GainSel%'] = round(gainsel_df['success'] * 100 / total_job_number, 1) summary_table = summary_table.to_pandas() From fa825dfc6286b265de1da7e4694ef3c6ed306d26 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 10:16:26 +0200 Subject: [PATCH 24/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index b9aaffb0..4851101a 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -63,7 +63,7 @@ def check_failed_jobs(date: str): gainsel_summary = [] for run in data_runs: run_id = run["run_id"] - gainsel_job_status = check_gainsel_jobs_runwise(date.replace('-',''), run_id) + gainsel_job_status = check_gainsel_jobs_runwise(date, run_id) gainsel_summary.append([run_id, gainsel_job_status]) gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) From afdbba0ff789bb10f15f6536855500c3c7776cc4 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 10:16:34 +0200 Subject: [PATCH 25/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 4851101a..a7def68a 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -57,7 +57,7 @@ def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: def check_failed_jobs(date: str): """Search for failed jobs in the log directory.""" - summary_table = run_summary_table(datetime.fromisoformat(date)) + summary_table = run_summary_table(date) data_runs = summary_table[summary_table["run_type"] == "DATA"] gainsel_summary = [] From 085dcaf788d739d20e648c0c2fe7addda4c491d8 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 10:45:42 +0200 Subject: [PATCH 26/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index a7def68a..633f092d 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -32,7 +32,7 @@ def valid_date(string): ) -def check_gainsel_jobs_runwise(date: str, run_id: int) -> bool: +def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool: """Search for failed jobs in the log directory.""" base_dir = Path(cfg.get("LST1", "BASE")) log_dir = base_dir / f"R0G/log/{date}" From dad866d732e2e011886e57127b014171e403087e Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 10:45:53 +0200 Subject: [PATCH 27/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 633f092d..e91e5879 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -35,6 +35,7 @@ def valid_date(string): def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool: """Search for failed jobs in the log directory.""" base_dir = Path(cfg.get("LST1", "BASE")) + flat_date = date_to_dir(date) log_dir = base_dir / f"R0G/log/{date}" history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history") From 43ab3795481b4ee0529426a5e310b5662397fc90 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 10:46:09 +0200 Subject: [PATCH 28/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index e91e5879..9b8dc86d 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -56,7 +56,7 @@ def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool: pending_subruns = pending_subruns+1 return [pending_subruns, success_subruns, failed_subruns] -def check_failed_jobs(date: str): +def check_failed_jobs(date: datetime): """Search for failed jobs in the log directory.""" summary_table = run_summary_table(date) data_runs = summary_table[summary_table["run_type"] == "DATA"] From 596a033a2b8faae87b0feaaf0753913e992ef49b Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 10:46:19 +0200 Subject: [PATCH 29/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 9b8dc86d..138f824d 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -36,7 +36,7 @@ def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool: """Search for failed jobs in the log directory.""" base_dir = Path(cfg.get("LST1", "BASE")) flat_date = date_to_dir(date) - log_dir = base_dir / f"R0G/log/{date}" + log_dir = base_dir / f"R0G/log/{flat_date}" history_files = log_dir.glob(f"gain_selection_{run_id:05d}.????.history") success_subruns = 0 From ae1f5061475060462b3b986dd9bbf899dce97752 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 11:14:07 +0200 Subject: [PATCH 30/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 138f824d..0097fc92 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -13,7 +13,7 @@ def valid_date(string): """Check if the string is a valid date and return a datetime object.""" - return datetime.strptime(string, "%Y%m%d") + return datetime.strptime(string, "%Y-%m-%d") common_parser = ArgumentParser(add_help=False) @@ -65,7 +65,7 @@ def check_failed_jobs(date: datetime): for run in data_runs: run_id = run["run_id"] gainsel_job_status = check_gainsel_jobs_runwise(date, run_id) - gainsel_summary.append([run_id, gainsel_job_status]) + gainsel_summary.append([run_id, gainsel_job_status[0], gainsel_job_status[1], gainsel_job_status[2]]) gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, @@ -102,12 +102,11 @@ def main(): else: # yesterday by default - yesterday = datetime.now() - timedelta(days=1) - options.date = yesterday - flat_date = date_to_dir(yesterday) + yesterday = datetime.now() - timedelta(days=1) + options.date = yesterday + flat_date = date_to_dir(yesterday) date = date_to_iso(options.date) - run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR")) run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv" @@ -122,7 +121,7 @@ def main(): else: # Get the table with the gain selection check report: - table_gain_selection_jobs = check_failed_jobs(date) + table_gain_selection_jobs = check_failed_jobs(datetime.strptime(date, "%Y-%m-%d")) table_gain_selection_jobs.reset_index(drop=True, inplace=True) html_table = table_gain_selection_jobs.to_html() From 00e82f6cb0b5bf307de2555eef620c9755dd2948 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 11:14:54 +0200 Subject: [PATCH 31/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 0097fc92..90f91e70 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -132,7 +132,7 @@ def main(): directory.mkdir(parents=True, exist_ok=True) html_file = directory / f"osa_gainsel_status_{flat_date}.html" - html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") + html_file.write_text(html_content(html_table, flat_date, "OSA Gain Selection"), encoding="utf-8") if __name__ == "__main__": main() From 4b64f2f2a3ab85839db7a6de1be4bc10d09f8345 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 11:52:44 +0200 Subject: [PATCH 32/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 90f91e70..75d3dcc5 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -130,9 +130,10 @@ def main(): directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) directory.mkdir(parents=True, exist_ok=True) - - html_file = directory / f"osa_gainsel_status_{flat_date}.html" - html_file.write_text(html_content(html_table, flat_date, "OSA Gain Selection"), encoding="utf-8") + + html_file =open( directory / f"osa_gainsel_status_{date}.html", "w") + html_file.write(html_content(html_table, date, "OSA Gain Selection")) + html_file.close() if __name__ == "__main__": main() From 0f93dea7cedb115e6cccc4250c650ef52a837585 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Fri, 5 Jul 2024 11:58:11 +0200 Subject: [PATCH 33/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 75d3dcc5..5a0481d1 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -130,7 +130,7 @@ def main(): directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) directory.mkdir(parents=True, exist_ok=True) - + html_file =open( directory / f"osa_gainsel_status_{date}.html", "w") html_file.write(html_content(html_table, date, "OSA Gain Selection")) html_file.close() From 4590557fe227c67d6457fe0fa48c3dbccd394475 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 10:21:09 +0200 Subject: [PATCH 34/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 5a0481d1..b05df56e 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -114,7 +114,7 @@ def main(): html_table = "

No data found

" # Save the HTML file - directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) + directory = Path(cfg.get("LST1", "GAIN_SELECTION_WEB_DIR")) directory.mkdir(parents=True, exist_ok=True) html_file = directory / f"osa_gainsel_status_{flat_date}.html" html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") @@ -127,7 +127,7 @@ def main(): html_table = table_gain_selection_jobs.to_html() # Save the HTML file - directory = Path(cfg.get("LST1", "GAIN_SELECTION_FLAG_DIR")) + directory = Path(cfg.get("LST1", "GAIN_SELECTION_WEB_DIR")) directory.mkdir(parents=True, exist_ok=True) From bc8aa4cabe74da500e40752394eb4b22d16dd7fe Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 10:55:53 +0200 Subject: [PATCH 35/49] Update pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 59bd222a..684a5bc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ reprocessing = "osa.scripts.reprocessing:main" reprocess_longterm = "osa.scripts.reprocess_longterm:main" gain_selection = "osa.scripts.gain_selection:main" update_source_catalog = "osa.scripts.update_source_catalog:main" +gainsel_webmaker = "osa.scripts.gainsel_webmaker:main" [tool.setuptools.packages.find] where = ["src"] From e105d111b8d84bf37d7d75789f134b57457840f5 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 11:05:27 +0200 Subject: [PATCH 36/49] Create test_gainsel_webmaker.py --- src/osa/tests/test_gainsel_webmaker.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 src/osa/tests/test_gainsel_webmaker.py diff --git a/src/osa/tests/test_gainsel_webmaker.py b/src/osa/tests/test_gainsel_webmaker.py new file mode 100644 index 00000000..d4e603fa --- /dev/null +++ b/src/osa/tests/test_gainsel_webmaker.py @@ -0,0 +1,16 @@ +def test_gain_selection_webmaker( + run_summary, + merged_run_summary, + drs4_time_calibration_files, + systematic_correction_files, + base_test_dir, +): + + + output = sp.run( + ["gainsel_webmaker", "--test", "-d", "2020-01-17"], + text=True, + stdout=sp.PIPE, + stderr=sp.PIPE, + ) + assert output.returncode != 0 From 0df83893fe14be46dbaa0a78e54e38ba5f6e8818 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 11:09:34 +0200 Subject: [PATCH 37/49] Update sequencer.cfg --- src/osa/configs/sequencer.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/src/osa/configs/sequencer.cfg b/src/osa/configs/sequencer.cfg index 6deeae3d..da9dc1fe 100644 --- a/src/osa/configs/sequencer.cfg +++ b/src/osa/configs/sequencer.cfg @@ -29,6 +29,7 @@ LONGTERM_DIR: %(OSA_DIR)s/DL1DataCheck_LongTerm MERGED_SUMMARY: %(OSA_DIR)s/Catalog/merged_RunSummary.ecsv SEQUENCER_WEB_DIR: %(OSA_DIR)s/SequencerWeb GAIN_SELECTION_FLAG_DIR: %(OSA_DIR)s/GainSel +GAIN_SELECTION_WEB_DIR: %(OSA_DIR)s/GainSelWeb # To be set by the user. Using PROD-ID will overcome the automatic # fetching of lstchain version. Otherwise leave it empty (and without the colon symbol). From abf6ba19d6b590d346ae881bdf92df5bfb8540e7 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 11:20:00 +0200 Subject: [PATCH 38/49] Delete src/osa/tests/test_gainsel_webmaker.py --- src/osa/tests/test_gainsel_webmaker.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 src/osa/tests/test_gainsel_webmaker.py diff --git a/src/osa/tests/test_gainsel_webmaker.py b/src/osa/tests/test_gainsel_webmaker.py deleted file mode 100644 index d4e603fa..00000000 --- a/src/osa/tests/test_gainsel_webmaker.py +++ /dev/null @@ -1,16 +0,0 @@ -def test_gain_selection_webmaker( - run_summary, - merged_run_summary, - drs4_time_calibration_files, - systematic_correction_files, - base_test_dir, -): - - - output = sp.run( - ["gainsel_webmaker", "--test", "-d", "2020-01-17"], - text=True, - stdout=sp.PIPE, - stderr=sp.PIPE, - ) - assert output.returncode != 0 From 399b6b5973e7d4df98cdbcadc2d2deea42ac60e8 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 15:33:38 +0200 Subject: [PATCH 39/49] Update test_osa_scripts.py --- src/osa/scripts/tests/test_osa_scripts.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index bdfa153f..5c5aa7d5 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -23,6 +23,7 @@ "theta2_significance", "source_coordinates", "sequencer_webmaker", + "gainsel_webmaker", ] options.date = datetime.datetime.fromisoformat("2020-01-17") @@ -397,3 +398,15 @@ def test_sequencer_webmaker( # Running without test option will make the script fail output = sp.run(["sequencer_webmaker", "-d", "2020-01-17"]) assert output.returncode != 0 + + +def test_gainsel_webmaker( + base_test_dir, +): + + output = sp.run(["gainsel_webmaker", "-d", "20200117"]) + assert output.returncode == 0 + directory = base_test_dir / "OSA" / "GainSelWeb" + directory.mkdir(parents=True, exist_ok=True) + expected_file = directory / "osa_gainsel_status_20200117.html" + assert expected_file.exists() From 85542aaa2013554aafb46f9add4310320b1732c9 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 16:04:10 +0200 Subject: [PATCH 40/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index b05df56e..876db2d1 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -121,7 +121,7 @@ def main(): else: # Get the table with the gain selection check report: - table_gain_selection_jobs = check_failed_jobs(datetime.strptime(date, "%Y-%m-%d")) + table_gain_selection_jobs = check_failed_jobs(options.date) table_gain_selection_jobs.reset_index(drop=True, inplace=True) html_table = table_gain_selection_jobs.to_html() From 4f194074a7a613cb98a04a6fdf1524c79c566ac8 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 16:04:29 +0200 Subject: [PATCH 41/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 876db2d1..20bdb61a 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -91,7 +91,10 @@ def check_failed_jobs(date: datetime): return final_table def main(): - """Produce the html file with the processing OSA Gain Selection status.""" + """Produce the html file with the processing OSA Gain Selection status. + + It creates an HTML file osa_gainsel_status_YYYYMMDD.html + """ args = ArgumentParser( description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] ).parse_args() From a9b6f5bd7b490ff3bc55ea28900adc12f6a84e83 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 16:05:38 +0200 Subject: [PATCH 42/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 20bdb61a..81eed9ff 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -96,7 +96,8 @@ def main(): It creates an HTML file osa_gainsel_status_YYYYMMDD.html """ args = ArgumentParser( - description="Script to make an xhtml from LSTOSA sequencer output", parents=[common_parser] + description="Script to create an HTML file for gain selection status (osa_gainsel_status_YYYYMMDD.html)", + parents=[common_parser] ).parse_args() if args.date: From 025156bcad762973cfe2604279b7c3f31b8d1862 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 16:05:45 +0200 Subject: [PATCH 43/49] Update src/osa/scripts/gainsel_webmaker.py Co-authored-by: Daniel Morcuende --- src/osa/scripts/gainsel_webmaker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 81eed9ff..a8545d53 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -135,7 +135,7 @@ def main(): directory.mkdir(parents=True, exist_ok=True) - html_file =open( directory / f"osa_gainsel_status_{date}.html", "w") + html_file =open( directory / f"osa_gainsel_status_{flat_date}.html", "w") html_file.write(html_content(html_table, date, "OSA Gain Selection")) html_file.close() From 9c9f8163c9922b1d6e195ac107d5fe5ba74ba054 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Mon, 8 Jul 2024 16:06:29 +0200 Subject: [PATCH 44/49] Update test_osa_scripts.py --- src/osa/scripts/tests/test_osa_scripts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 5c5aa7d5..d1bd5e7f 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -407,6 +407,5 @@ def test_gainsel_webmaker( output = sp.run(["gainsel_webmaker", "-d", "20200117"]) assert output.returncode == 0 directory = base_test_dir / "OSA" / "GainSelWeb" - directory.mkdir(parents=True, exist_ok=True) expected_file = directory / "osa_gainsel_status_20200117.html" assert expected_file.exists() From 0e8ff4367ceac9c07241d3280a2330c38e0c80fb Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Tue, 9 Jul 2024 10:38:21 +0200 Subject: [PATCH 45/49] Update test_osa_scripts.py --- src/osa/scripts/tests/test_osa_scripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index d1bd5e7f..0e4220ac 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -404,7 +404,7 @@ def test_gainsel_webmaker( base_test_dir, ): - output = sp.run(["gainsel_webmaker", "-d", "20200117"]) + output = sp.run(["gainsel_webmaker", "-d", "2020-01-17"]) assert output.returncode == 0 directory = base_test_dir / "OSA" / "GainSelWeb" expected_file = directory / "osa_gainsel_status_20200117.html" From e982daf27e3799b4fee393f04f0a27460046d1f3 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Tue, 9 Jul 2024 11:29:56 +0200 Subject: [PATCH 46/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index a8545d53..fc4ddf5c 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -68,11 +68,13 @@ def check_failed_jobs(date: datetime): gainsel_summary.append([run_id, gainsel_job_status[0], gainsel_job_status[1], gainsel_job_status[2]]) gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) - gainsel_df['GainSelStatus'] = np.where(gainsel_df['failed'] != 0, + gainsel_df['GainSelStatus'] = np.where((gainsel_df['failed'] == 0) & (gainsel_df['pending'] == 0) & (gainsel_df['success'] == 0), + 'NOT STARTED', + np.where(gainsel_df['failed'] != 0, 'FAILED', np.where(gainsel_df['pending'] != 0, 'PENDING', - 'COMPLETED')) + 'COMPLETED'))) total_job_number = gainsel_df['pending'] + gainsel_df['failed'] + gainsel_df['success'] gainsel_df['GainSel%'] = round(gainsel_df['success'] * 100 / total_job_number, 1) From 3ad9629d04c3c0d5cbe12682efdc7550c2b53e29 Mon Sep 17 00:00:00 2001 From: rcervinoucm Date: Tue, 9 Jul 2024 11:52:16 +0200 Subject: [PATCH 47/49] Update gainsel_webmaker.py --- src/osa/scripts/gainsel_webmaker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index fc4ddf5c..6c87ca38 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -122,7 +122,7 @@ def main(): # Save the HTML file directory = Path(cfg.get("LST1", "GAIN_SELECTION_WEB_DIR")) directory.mkdir(parents=True, exist_ok=True) - html_file = directory / f"osa_gainsel_status_{flat_date}.html" + html_file = directory / f"osa_gainsel_status_{date}.html" html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") else: @@ -137,7 +137,7 @@ def main(): directory.mkdir(parents=True, exist_ok=True) - html_file =open( directory / f"osa_gainsel_status_{flat_date}.html", "w") + html_file =open( directory / f"osa_gainsel_status_{date}.html", "w") html_file.write(html_content(html_table, date, "OSA Gain Selection")) html_file.close() From 1de1b19cea0f306ad9d0da32292d016493d9872c Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 9 Jul 2024 12:20:44 +0200 Subject: [PATCH 48/49] some simplifications and cleanup --- src/osa/scripts/gainsel_webmaker.py | 125 ++++++++++++---------- src/osa/scripts/tests/test_osa_scripts.py | 15 +++ 2 files changed, 83 insertions(+), 57 deletions(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index 6c87ca38..dfac07f4 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -1,15 +1,20 @@ +import logging +from argparse import ArgumentParser +from datetime import datetime, timedelta from pathlib import Path + +import pandas as pd from astropy.table import Table -from datetime import datetime, timedelta -from osa.utils.utils import date_to_dir, date_to_iso + +from osa.configs import options from osa.configs.config import cfg -from osa.paths import DEFAULT_CFG from osa.nightsummary.nightsummary import run_summary_table -import numpy as np -import pandas as pd -from argparse import ArgumentParser -from osa.configs import options +from osa.paths import DEFAULT_CFG from osa.scripts.sequencer_webmaker import html_content +from osa.utils.utils import date_to_dir, date_to_iso + +log = logging.getLogger(__name__) + def valid_date(string): """Check if the string is a valid date and return a datetime object.""" @@ -27,7 +32,7 @@ def valid_date(string): common_parser.add_argument( "-d", "--date", - help="Date (YYYYMMDD) of the start of the night", + help="Date of the start of the night in ISO format (YYYY-MM-DD). Defaults to yesterday", type=valid_date, ) @@ -48,58 +53,70 @@ def check_gainsel_jobs_runwise(date: datetime, run_id: int) -> bool: gainsel_rc = file.read_text().splitlines()[-1][-1] if gainsel_rc == "1": - failed_subruns = failed_subruns+1 + failed_subruns += 1 elif gainsel_rc == "0": - success_subruns = success_subruns+1 + success_subruns += 1 + else: - pending_subruns = pending_subruns+1 - return [pending_subruns, success_subruns, failed_subruns] + pending_subruns += 1 + + return {"pending": pending_subruns, "success": success_subruns, "failed": failed_subruns} + -def check_failed_jobs(date: datetime): +def check_failed_jobs(date: datetime) -> pd.DataFrame: """Search for failed jobs in the log directory.""" summary_table = run_summary_table(date) data_runs = summary_table[summary_table["run_type"] == "DATA"] - gainsel_summary = [] + gainsel_status_dict = {} for run in data_runs: run_id = run["run_id"] gainsel_job_status = check_gainsel_jobs_runwise(date, run_id) - gainsel_summary.append([run_id, gainsel_job_status[0], gainsel_job_status[1], gainsel_job_status[2]]) - - gainsel_df = pd.DataFrame(gainsel_summary, columns=['run_id', 'pending','success','failed']) - gainsel_df['GainSelStatus'] = np.where((gainsel_df['failed'] == 0) & (gainsel_df['pending'] == 0) & (gainsel_df['success'] == 0), - 'NOT STARTED', - np.where(gainsel_df['failed'] != 0, - 'FAILED', - np.where(gainsel_df['pending'] != 0, - 'PENDING', - 'COMPLETED'))) - - total_job_number = gainsel_df['pending'] + gainsel_df['failed'] + gainsel_df['success'] - gainsel_df['GainSel%'] = round(gainsel_df['success'] * 100 / total_job_number, 1) + gainsel_status_dict[run_id] = gainsel_job_status + gainsel_df = pd.DataFrame(gainsel_status_dict.values(), index=gainsel_status_dict.keys()) + gainsel_df.reset_index(inplace=True) + gainsel_df.rename(columns={"index": "run_id"}, inplace=True) summary_table = summary_table.to_pandas() - final_table = pd.merge(summary_table, gainsel_df, on="run_id")[['run_id', - 'n_subruns', - 'run_type', - 'pending', - 'success', - 'failed', - 'GainSelStatus', - 'GainSel%']] + final_table = pd.merge(summary_table, gainsel_df, on="run_id")[ + [ + "run_id", + "n_subruns", + "pending", + "success", + "failed", + ] + ] + + def determine_status(row): + if row["failed"] > 0: + return "FAILED" + elif row["pending"] > 0: + return "PENDING" + elif row["success"] == row["n_subruns"]: + return "COMPLETED" + else: + return "NOT STARTED" + + final_table["GainSel%"] = round(final_table["success"] * 100 / final_table["n_subruns"]) + final_table["GainSelStatus"] = final_table.apply(determine_status, axis=1) return final_table + def main(): """Produce the html file with the processing OSA Gain Selection status. - + It creates an HTML file osa_gainsel_status_YYYYMMDD.html """ args = ArgumentParser( - description="Script to create an HTML file for gain selection status (osa_gainsel_status_YYYYMMDD.html)", - parents=[common_parser] + description=( + "Script to create an HTML file with the gain selection status " + "(osa_gainsel_status_YYYYMMDD.html)" + ), + parents=[common_parser], ).parse_args() if args.date: @@ -107,7 +124,7 @@ def main(): options.date = args.date else: - # yesterday by default + # yesterday by default yesterday = datetime.now() - timedelta(days=1) options.date = yesterday flat_date = date_to_dir(yesterday) @@ -116,30 +133,24 @@ def main(): run_summary_directory = Path(cfg.get("LST1", "RUN_SUMMARY_DIR")) run_summary_file = run_summary_directory / f"RunSummary_{flat_date}.ecsv" - if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0: + gain_selection_web_directory = Path(cfg.get("LST1", "GAIN_SELECTION_WEB_DIR")) + gain_selection_web_directory.mkdir(parents=True, exist_ok=True) + html_file = gain_selection_web_directory / f"osa_gainsel_status_{flat_date}.html" - html_table = "

No data found

" - # Save the HTML file - directory = Path(cfg.get("LST1", "GAIN_SELECTION_WEB_DIR")) - directory.mkdir(parents=True, exist_ok=True) - html_file = directory / f"osa_gainsel_status_{date}.html" - html_file.write_text(html_content(html_table, date, "OSA Gain Selection"), encoding="utf-8") + # Create and save the HTML file + if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0: + content = "

No data found

" + log.warning(f"No data found for date {date}, creating an empty HTML file.") + html_file.write_text(html_content(content, date, "OSA Gain Selection"), encoding="utf-8") else: - # Get the table with the gain selection check report: + # Get the table with the gain selection check report in HTML format: table_gain_selection_jobs = check_failed_jobs(options.date) + content = table_gain_selection_jobs.to_html() - table_gain_selection_jobs.reset_index(drop=True, inplace=True) - html_table = table_gain_selection_jobs.to_html() - - # Save the HTML file - directory = Path(cfg.get("LST1", "GAIN_SELECTION_WEB_DIR")) - - directory.mkdir(parents=True, exist_ok=True) + html_file.write_text(html_content(content, date, "OSA Gain Selection")) + log.info(f"Created HTML file {html_file}") - html_file =open( directory / f"osa_gainsel_status_{date}.html", "w") - html_file.write(html_content(html_table, date, "OSA Gain Selection")) - html_file.close() if __name__ == "__main__": main() diff --git a/src/osa/scripts/tests/test_osa_scripts.py b/src/osa/scripts/tests/test_osa_scripts.py index 0e4220ac..f8344769 100644 --- a/src/osa/scripts/tests/test_osa_scripts.py +++ b/src/osa/scripts/tests/test_osa_scripts.py @@ -409,3 +409,18 @@ def test_gainsel_webmaker( directory = base_test_dir / "OSA" / "GainSelWeb" expected_file = directory / "osa_gainsel_status_20200117.html" assert expected_file.exists() + + # Test a date with non-existing run summary + output = sp.run(["gainsel_webmaker", "-d", "2024-01-12"]) + assert output.returncode == 0 + directory = base_test_dir / "OSA" / "GainSelWeb" + expected_file = directory / "osa_gainsel_status_20240112.html" + assert expected_file.exists() + + +def test_gainsel_web_content(): + from osa.scripts.gainsel_webmaker import check_failed_jobs + + table = check_failed_jobs(options.date) + assert table["GainSelStatus"][0] == "NOT STARTED" + assert table["GainSel%"][0] == 0.0 From 107d558d7e2f7e0e810cb407e9253578402278c8 Mon Sep 17 00:00:00 2001 From: Daniel Morcuende Date: Tue, 9 Jul 2024 12:53:09 +0200 Subject: [PATCH 49/49] remove redundant line --- src/osa/scripts/gainsel_webmaker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/osa/scripts/gainsel_webmaker.py b/src/osa/scripts/gainsel_webmaker.py index dfac07f4..e6611548 100644 --- a/src/osa/scripts/gainsel_webmaker.py +++ b/src/osa/scripts/gainsel_webmaker.py @@ -141,7 +141,6 @@ def main(): if not run_summary_file.is_file() or len(Table.read(run_summary_file)["run_id"]) == 0: content = "

No data found

" log.warning(f"No data found for date {date}, creating an empty HTML file.") - html_file.write_text(html_content(content, date, "OSA Gain Selection"), encoding="utf-8") else: # Get the table with the gain selection check report in HTML format: