Print final log statement

- Use temporary TXT file to track data
- Print final statement via Python script

create_generic_download_list.py

@@ -48,6 +48,7 @@
 #import urllib.request, urllib.error, urllib.parse
 
 from generic_split_search_dates_into_months import generic_split_search_dates_into_months;
+from write_final_log import write_final_log
 
 # Make a query to OBPG to fetch a list of filename and checksum.
 #
@@ -1200,6 +1201,7 @@ def create_generic_download_list(search_dtype,       # L2
                         output_file_pointer.write(new_line + '\n');
                         found_names += 1; # Keep track of how many names we have written to this new file.
                         print(f"{g_module_name} - INFO: Processed: {pathlib.Path(output_file_name).name} | {new_line}")
+                        write_final_log(f"processed: {new_line}")
                         if (g_trace_flag):
                             print(g_module_name + "TRACE:file_state_status",file_state_status,"new_line",new_line)
                     else:
@@ -1254,6 +1256,7 @@ def create_generic_download_list(search_dtype,       # L2
                 print(g_module_name + "INFO:NUM_EXISTING_NAMES_SAME_CHECKSUM_IN_STATE",g_num_existing_names_same_checksum_in_state);
                 print(g_module_name + "INFO:NUM_EXISTING_NAMES_DIFFERENT_CHECKSUM_IN_STATE",g_num_existing_names_different_checksum_in_state);
                 print(f"{g_module_name} - INFO: Number of downloads: {all_names_found_in_execution}")
+                write_final_log(f"number_downloads: {all_names_found_in_execution}")
               except:
                 print(g_module_name + "ERROR:len(g_state_for_saving_dictionary)",len(g_state_for_saving_dictionary));
                 print(g_module_name + "ERROR:Had issues writing content of g_state_for_saving_dictionary to state file " + default_state_filename);
@@ -1285,6 +1288,7 @@ def create_generic_download_list(search_dtype,       # L2
         # Add a few more debug prints so the user know why zero files are returned.
         print("regular_expression_to_check [", regular_expression_to_check, "]");
         print("CRAWLER_SEARCH_FILE_PATTERN[",os.getenv('CRAWLER_SEARCH_FILE_PATTERN',''),"]");
+        write_final_log("number_downloads: 0")
 
     return(o_encountered_error_flag);
 

download_list_creator_lambda.py

@@ -30,6 +30,7 @@
 
 # Local imports
 from notify import notify
+from write_final_log import write_final_log
 
 # Constants
 LOG_PREFIX = {
@@ -68,17 +69,23 @@ def event_handler(event, context):
         naming_pattern_indicator = ""
     txt_file_list = pathlib.Path(f"/tmp/txt_file_list_{processing_type}.txt")
 
+    # Set final log file environment variable
+    os.environ["FINAL_LOG_MESSAGE"] = f"/tmp/final_log_message_{UNIQUE_ID}.txt"
+
     # Create required directories
     if not output_directory.is_dir():
         output_directory.mkdir(parents=True, exist_ok=True)
     if not state_file_name.parent.is_dir():
         state_file_name.parent.mkdir(parents=True, exist_ok=True)
 
+    # Get logger
+    logger = get_logger()
+    logger.info(f"Unique identifier: {UNIQUE_ID}")
+    write_final_log(f"unique_id: {UNIQUE_ID}")
+
     # Check if state file exists and pull from S3 to /tmp if it does
     s3_client = boto3.client("s3")
     bucket = f"{event['prefix']}"
-    logger = get_logger()
-    logger.info(f"Unique identifier: {UNIQUE_ID}")
     get_s3_state_file(s3_client, bucket, state_file_name, logger)
 
     # Execute shell script
@@ -130,6 +137,9 @@ def event_handler(event, context):
     else:
         logger.info("No new downloads were found.")
 
+    # Print final log message
+    print_final_log(logger)
+
     # Delete logger    
     for handler in logger.handlers:
         logger.removeHandler(handler) 
@@ -269,6 +279,31 @@ def delete_files(txt_list, state_file_name, txt_file_list, logger):
 
     txt_file_list.unlink()
     logger.info(f"Deleted file from /tmp: {txt_file_list.name}")
+
+def print_final_log(logger):
+    """Print final log message."""
+
+    # Open file used to track data
+    log_file = pathlib.Path(os.environ.get("FINAL_LOG_MESSAGE"))
+    with open(log_file) as fh:
+        log_data = fh.read().splitlines()
+
+    # Organize file data into a string
+    execution_data = ""
+    processed = []
+    for line in log_data:
+        if "unique_id" in line: execution_data += line
+        if "dataset" in line: execution_data += f" - {line}"
+        if "number_downloads" in line: execution_data += f" - {line}"
+        if "processed" in line: processed.append(line.split("processed: ")[-1])
+
+    final_log_message = ""
+    if execution_data: final_log_message += execution_data
+    if len(processed) > 0: final_log_message += f" - processed: {', '.join(processed)}"
+
+    # Print final log message and remove temp log file
+    logger.info(final_log_message)
+    log_file.unlink()
 
 def handle_error(sigevent_description, sigevent_data, logger):
     """Handle errors by logging them and sending out a notification."""

shell/startup_generic_download_list_creator.csh

@@ -217,6 +217,7 @@ else
     set dataset = $processing_type
 endif
 echo "startup_generic_download_list_creator.csh - INFO: Dataset:" $dataset
+echo "dataset: $dataset" >> $FINAL_LOG_MESSAGE
 
 # Create the $HOME/logs directory if it does not exist yet
 set logging_dir = `printenv | grep OBPG_DOWNLOAD_LIST_CREATOR_LOGGING | awk -F= '{print $2}'`

write_final_log.py

@@ -0,0 +1,14 @@
+#
+# Function to write a message to the final log file which will later be 
+# retrieved and logged to support Cloud Metrics.
+#
+
+# Standard imports
+import os
+
+def write_final_log(message):
+    """Write message to final log file."""
+
+    final_log = os.environ.get("FINAL_LOG_MESSAGE")
+    with open(final_log, 'a') as fh:
+        fh.write(f"{message}\n")