From 22f060723e0a5922080fb4794b8cf292e9214164 Mon Sep 17 00:00:00 2001 From: Nikki Tebaldi <17799906+nikki-t@users.noreply.github.com> Date: Fri, 27 Oct 2023 20:28:51 +0000 Subject: [PATCH] Print final log statement - Use temporary TXT file to track data - Print final statement via Python script --- create_generic_download_list.py | 4 ++ download_list_creator_lambda.py | 39 ++++++++++++++++++- .../startup_generic_download_list_creator.csh | 1 + write_final_log.py | 14 +++++++ 4 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 write_final_log.py diff --git a/create_generic_download_list.py b/create_generic_download_list.py index 998cdac..9145a3a 100644 --- a/create_generic_download_list.py +++ b/create_generic_download_list.py @@ -48,6 +48,7 @@ #import urllib.request, urllib.error, urllib.parse from generic_split_search_dates_into_months import generic_split_search_dates_into_months; +from write_final_log import write_final_log # Make a query to OBPG to fetch a list of filename and checksum. # @@ -1200,6 +1201,7 @@ def create_generic_download_list(search_dtype, # L2 output_file_pointer.write(new_line + '\n'); found_names += 1; # Keep track of how many names we have written to this new file. print(f"{g_module_name} - INFO: Processed: {pathlib.Path(output_file_name).name} | {new_line}") + write_final_log(f"processed: {new_line}") if (g_trace_flag): print(g_module_name + "TRACE:file_state_status",file_state_status,"new_line",new_line) else: @@ -1254,6 +1256,7 @@ def create_generic_download_list(search_dtype, # L2 print(g_module_name + "INFO:NUM_EXISTING_NAMES_SAME_CHECKSUM_IN_STATE",g_num_existing_names_same_checksum_in_state); print(g_module_name + "INFO:NUM_EXISTING_NAMES_DIFFERENT_CHECKSUM_IN_STATE",g_num_existing_names_different_checksum_in_state); print(f"{g_module_name} - INFO: Number of downloads: {all_names_found_in_execution}") + write_final_log(f"number_downloads: {all_names_found_in_execution}") except: print(g_module_name + "ERROR:len(g_state_for_saving_dictionary)",len(g_state_for_saving_dictionary)); print(g_module_name + "ERROR:Had issues writing content of g_state_for_saving_dictionary to state file " + default_state_filename); @@ -1285,6 +1288,7 @@ def create_generic_download_list(search_dtype, # L2 # Add a few more debug prints so the user know why zero files are returned. print("regular_expression_to_check [", regular_expression_to_check, "]"); print("CRAWLER_SEARCH_FILE_PATTERN[",os.getenv('CRAWLER_SEARCH_FILE_PATTERN',''),"]"); + write_final_log("number_downloads: 0") return(o_encountered_error_flag); diff --git a/download_list_creator_lambda.py b/download_list_creator_lambda.py index 56ad6db..26aeb30 100644 --- a/download_list_creator_lambda.py +++ b/download_list_creator_lambda.py @@ -30,6 +30,7 @@ # Local imports from notify import notify +from write_final_log import write_final_log # Constants LOG_PREFIX = { @@ -68,17 +69,23 @@ def event_handler(event, context): naming_pattern_indicator = "" txt_file_list = pathlib.Path(f"/tmp/txt_file_list_{processing_type}.txt") + # Set final log file environment variable + os.environ["FINAL_LOG_MESSAGE"] = f"/tmp/final_log_message_{UNIQUE_ID}.txt" + # Create required directories if not output_directory.is_dir(): output_directory.mkdir(parents=True, exist_ok=True) if not state_file_name.parent.is_dir(): state_file_name.parent.mkdir(parents=True, exist_ok=True) + # Get logger + logger = get_logger() + logger.info(f"Unique identifier: {UNIQUE_ID}") + write_final_log(f"unique_id: {UNIQUE_ID}") + # Check if state file exists and pull from S3 to /tmp if it does s3_client = boto3.client("s3") bucket = f"{event['prefix']}" - logger = get_logger() - logger.info(f"Unique identifier: {UNIQUE_ID}") get_s3_state_file(s3_client, bucket, state_file_name, logger) # Execute shell script @@ -130,6 +137,9 @@ def event_handler(event, context): else: logger.info("No new downloads were found.") + # Print final log message + print_final_log(logger) + # Delete logger for handler in logger.handlers: logger.removeHandler(handler) @@ -269,6 +279,31 @@ def delete_files(txt_list, state_file_name, txt_file_list, logger): txt_file_list.unlink() logger.info(f"Deleted file from /tmp: {txt_file_list.name}") + +def print_final_log(logger): + """Print final log message.""" + + # Open file used to track data + log_file = pathlib.Path(os.environ.get("FINAL_LOG_MESSAGE")) + with open(log_file) as fh: + log_data = fh.read().splitlines() + + # Organize file data into a string + execution_data = "" + processed = [] + for line in log_data: + if "unique_id" in line: execution_data += line + if "dataset" in line: execution_data += f" - {line}" + if "number_downloads" in line: execution_data += f" - {line}" + if "processed" in line: processed.append(line.split("processed: ")[-1]) + + final_log_message = "" + if execution_data: final_log_message += execution_data + if len(processed) > 0: final_log_message += f" - processed: {', '.join(processed)}" + + # Print final log message and remove temp log file + logger.info(final_log_message) + log_file.unlink() def handle_error(sigevent_description, sigevent_data, logger): """Handle errors by logging them and sending out a notification.""" diff --git a/shell/startup_generic_download_list_creator.csh b/shell/startup_generic_download_list_creator.csh index 075a955..79d4925 100755 --- a/shell/startup_generic_download_list_creator.csh +++ b/shell/startup_generic_download_list_creator.csh @@ -217,6 +217,7 @@ else set dataset = $processing_type endif echo "startup_generic_download_list_creator.csh - INFO: Dataset:" $dataset +echo "dataset: $dataset" >> $FINAL_LOG_MESSAGE # Create the $HOME/logs directory if it does not exist yet set logging_dir = `printenv | grep OBPG_DOWNLOAD_LIST_CREATOR_LOGGING | awk -F= '{print $2}'` diff --git a/write_final_log.py b/write_final_log.py new file mode 100644 index 0000000..8bb4298 --- /dev/null +++ b/write_final_log.py @@ -0,0 +1,14 @@ +# +# Function to write a message to the final log file which will later be +# retrieved and logged to support Cloud Metrics. +# + +# Standard imports +import os + +def write_final_log(message): + """Write message to final log file.""" + + final_log = os.environ.get("FINAL_LOG_MESSAGE") + with open(final_log, 'a') as fh: + fh.write(f"{message}\n") \ No newline at end of file