From 40e7d3265f31a2be024e68127666c0f656c65460 Mon Sep 17 00:00:00 2001
From: RayPlante <raymond.plante@nist.gov>
Date: Wed, 27 Mar 2024 09:52:23 -0400
Subject: [PATCH] Attempt to address non-notifying preservation failures:   *
 ensure that MIDASSIP.revdatadir is never None   * reinstate use of presvpod
 in POD processing queue   * add some more log message

---
 python/nistoar/pdr/preserv/bagger/midas3.py  | 36 ++++++++++++++------
 python/nistoar/pdr/publish/midas3/service.py | 11 +++++-
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/python/nistoar/pdr/preserv/bagger/midas3.py b/python/nistoar/pdr/preserv/bagger/midas3.py
index 9e67c31c..7b1455e7 100644
--- a/python/nistoar/pdr/preserv/bagger/midas3.py
+++ b/python/nistoar/pdr/preserv/bagger/midas3.py
@@ -201,27 +201,39 @@ def __init__(self, midasid, reviewdir, uploaddir=None, podrec=None, nerdrec=None
                                   the POD JSON file
         :param nerdrec str|dict:  either the NERDm record data (as a dict) or a filepath to 
                                   the NERDm JSON file
+        :raises SIPDirectoryError:  if either of the given directories are not readable or 
+                                  if neither exist 
         """
         self.midasid = midasid
+        self.revdatadir = reviewdir
+        self.upldatadir = uploaddir
 
         # ensure we have at least one readable input directory
-        self.revdatadir = self._check_input_datadir(reviewdir)
-        self.upldatadir = self._check_input_datadir(uploaddir)
-
         self._indirs = []
-        if self.revdatadir:
-            self._indirs.append(self.revdatadir)
-        if self.upldatadir:
-            self._indirs.append(self.upldatadir)
+        self.refresh_indirs()  # may raise SIPDirectoryNotFound, SIPDirectoryError
+
+        self.nerd = nerdrec
+        self.pod  = podrec
+
+    def refresh_indirs(self):
+        """
+        check and update the directories where this bagger will look for data
+        """
+        indirs = []
+        indir = self._check_input_datadir(self.revdatadir)
+        if indir:
+            indirs.append(indir)
+        indir = self._check_input_datadir(self.upldatadir)
+        if indir:
+            indirs.append(indir)
+        self._indirs = indirs
 
         if not self._indirs:
             if log.isEnabledFor(logging.DEBUG):
-                log.warn("No input directories available for midasid=%s", midasid)
+                log.warn("No input directories available for midasid=%s", self.midasid)
                 log.debug("Input dirs:\n  %s\n  %s", str(self.revdatadir), str(self.upldatadir))
             raise SIPDirectoryNotFound(msg="No input directories available", sys=self)
-
-        self.nerd = nerdrec
-        self.pod  = podrec
+        
 
     @property
     def input_dirs(self):
@@ -885,6 +897,8 @@ def apply_pod(self, pod, validate=True, force=False, lock=True):
     def _apply_pod(self, pod, validate=True, force=False):
         if not isinstance(pod, (str, unicode, Mapping)):
             raise NERDTypeError("dict", type(pod), "POD Dataset")
+        self.log.debug("%s: refreshing available input directories", self.midasid)
+        self.sip.refresh_indirs()
         self.ensure_base_bag()
 
         log.debug("BagBuilder log has %s formatters:\n%s", len(self.bagbldr.log.handlers),
diff --git a/python/nistoar/pdr/publish/midas3/service.py b/python/nistoar/pdr/publish/midas3/service.py
index 2fa1e4f9..dd6d6511 100644
--- a/python/nistoar/pdr/publish/midas3/service.py
+++ b/python/nistoar/pdr/publish/midas3/service.py
@@ -908,6 +908,9 @@ def preserve_new(self, pod, async=None):
                                          + halted)
 
         pod['_preserve'] = 'new'
+        if worker.is_working():
+            self.log.info("Note: POD processing is currently in progress; final processing for "+
+                          "publication will follow.")
         self._apply_pod_async(pod, async)
         if async and worker.is_working():
             # wait around for a little while to see if finishes quickly
@@ -930,6 +933,8 @@ def _check_pod_for_preservation(self, pod):
         if pod.get('accessLevel') == 'non-public':
             raise ejs.ValidationError("Unacceptable accessLevel property value for preservation: " +
                                       str(pod.get('accessLevel')))
+        if pod.get("_preserve"):
+            del pod['_preserve']
 
     def preserve_update(self, pod, async=None):
         """
@@ -1063,7 +1068,11 @@ def launch(self):
         def queue_POD(self, pod):
             self.ensure_qlock()
             with self.qlock:
-                write_json(pod, self.next_pod)
+                if pod.get('_preserve'):
+                    self.log.debug("%s: Queuing POD for preservation", self.id)
+                    write_json(pod, self.presv_pod)
+                else:
+                    write_json(pod, self.next_pod)
 
         def mark_for_preservation(self, asupdate=False):
             # NOTE: use of this function is DEPRECATED