From aa82c8cc0347923dafef8b27ea6175a51b3d1ecc Mon Sep 17 00:00:00 2001
From: Mike McCann <MBARIMike@gmail.com>
Date: Wed, 5 Jun 2024 21:59:36 +0000
Subject: [PATCH 1/2] Write consolidated exclude_list.txt file in same place as
 load log file, trap exception seen on MacOS

---
 .vscode/launch.json  |  2 +-
 smdb/scripts/load.py | 29 +++++++++++++++++++++++++----
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/.vscode/launch.json b/.vscode/launch.json
index 2b01b289..8ed3d2f8 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -83,7 +83,7 @@
       //"args": ["--compilation", "-v", "1", "--skipuntil", "2022/AxialSeamount/FiguresCaress", "--limit", "10", "--log_file", "compilation.txt"],
       //"args": ["--compilation", "-v", "1"],
       //"args": ["--spreadsheets", "-v", "1"],
-      "args": ["-v", "1", "--last_n_days", "0.5"],
+      "args": ["-v", "2", "--last_n_days", "0.5"],
       //"args": ["-v", "1", "--spreadsheets", "--parent_dir", "2024", "--append_to_log_file"],
       //"args": ["-v", "1", "--spreadsheets"],
       //"args": ["-v", "1", "--compilation", "--last_n_days", "30"],
diff --git a/smdb/scripts/load.py b/smdb/scripts/load.py
index 1e6c8b59..74da9c76 100755
--- a/smdb/scripts/load.py
+++ b/smdb/scripts/load.py
@@ -90,6 +90,7 @@ class BaseLoader:
     LOG_FILE = "load.txt"
     LOCAL_LOG_FILE = f"/etc/smdb/{LOG_FILE}"
     MEDIA_LOG_FILE = f"logs/{LOG_FILE}"
+    MEDIA_EXCLUDE_LIST_FILE = "logs/exclude_list.txt"
     LOCATE_DB = "/etc/smdb/SeafloorMapping.db"
 
     def __init__(self):
@@ -1191,10 +1192,18 @@ def load_from_grds(self):
                     exclude_count += 1
                     continue
                 if self.args.last_n_days:
-                    if os.path.getmtime(fp) < time() - self.args.last_n_days * 86400:
-                        self.logger.debug(
-                            f"Skipping file {fp} older than {self.args.last_n_days = }"
-                        )
+                    try:
+                        if (
+                            os.path.getmtime(fp)
+                            < time() - self.args.last_n_days * 86400
+                        ):
+                            self.logger.debug(
+                                f"Skipping file {fp} older than {self.args.last_n_days = }"
+                            )
+                            continue
+                    except FileNotFoundError:
+                        # Error seen with smb mount on MacOS but not on Linux
+                        self.logger.warning(f"File not found: {fp}")
                         continue
                 miss_count += 1
                 self.logger.info(
@@ -1807,6 +1816,17 @@ def write_exclude_path_csvs(self) -> None:
                     fh.write(f"{path}\n")
             self.logger.info(f"Wrote {len(paths)} paths to {csv_file}")
 
+    def write_consolidated_exclude_list(self) -> None:
+        """Write the consolidated exclude_paths to <parent_dir>/SMDB/exlude_list.csv file"""
+        ds = DefaultStorage()
+        ds.delete(self.MEDIA_EXCLUDE_LIST_FILE)
+        with ds.open(self.MEDIA_EXCLUDE_LIST_FILE, "w") as fh:
+            for path in sorted(self.exclude_paths):
+                fh.write(f"{path}\n")
+        self.logger.info(
+            f"Wrote {len(self.exclude_paths)} paths to {self.MEDIA_EXCLUDE_LIST_FILE}"
+        )
+
 
 def run(*args):
     # Possible use: https://django-extensions.readthedocs.io/en/latest/runscript.html
@@ -1852,6 +1872,7 @@ def exclude_file_load():
     ef.read_config_exclude_list()
     ef.read_exclude_path_xlsxs()
     ef.write_exclude_path_csvs()
+    ef.write_consolidated_exclude_list()
 
 
 def bootstrap_load() -> list:

From 2cff44fa153ec66dec041c8c8f7d357da4268769 Mon Sep 17 00:00:00 2001
From: Mike McCann <MBARIMike@gmail.com>
Date: Wed, 5 Jun 2024 22:18:26 +0000
Subject: [PATCH 2/2] Correct doc string for write_consolidated_exclude_list()

---
 smdb/scripts/load.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/smdb/scripts/load.py b/smdb/scripts/load.py
index 74da9c76..3215d442 100755
--- a/smdb/scripts/load.py
+++ b/smdb/scripts/load.py
@@ -1817,7 +1817,7 @@ def write_exclude_path_csvs(self) -> None:
             self.logger.info(f"Wrote {len(paths)} paths to {csv_file}")
 
     def write_consolidated_exclude_list(self) -> None:
-        """Write the consolidated exclude_paths to <parent_dir>/SMDB/exlude_list.csv file"""
+        """Write the consolidated exclude paths to exclude_list.txt next to the load log file"""
         ds = DefaultStorage()
         ds.delete(self.MEDIA_EXCLUDE_LIST_FILE)
         with ds.open(self.MEDIA_EXCLUDE_LIST_FILE, "w") as fh: