Merge pull request #336 from usnistgov/fix/pres-flow-bug

Fix preservation workflow bugs

python/nistoar/pdr/preserv/bagger/midas3.py

@@ -201,27 +201,39 @@ def __init__(self, midasid, reviewdir, uploaddir=None, podrec=None, nerdrec=None
                                   the POD JSON file
         :param nerdrec str|dict:  either the NERDm record data (as a dict) or a filepath to 
                                   the NERDm JSON file
+        :raises SIPDirectoryError:  if either of the given directories are not readable or 
+                                  if neither exist 
         """
         self.midasid = midasid
+        self.revdatadir = reviewdir
+        self.upldatadir = uploaddir
 
         # ensure we have at least one readable input directory
-        self.revdatadir = self._check_input_datadir(reviewdir)
-        self.upldatadir = self._check_input_datadir(uploaddir)
-
         self._indirs = []
-        if self.revdatadir:
-            self._indirs.append(self.revdatadir)
-        if self.upldatadir:
-            self._indirs.append(self.upldatadir)
+        self.refresh_indirs()  # may raise SIPDirectoryNotFound, SIPDirectoryError
+
+        self.nerd = nerdrec
+        self.pod  = podrec
+
+    def refresh_indirs(self):
+        """
+        check and update the directories where this bagger will look for data
+        """
+        indirs = []
+        indir = self._check_input_datadir(self.revdatadir)
+        if indir:
+            indirs.append(indir)
+        indir = self._check_input_datadir(self.upldatadir)
+        if indir:
+            indirs.append(indir)
+        self._indirs = indirs
 
         if not self._indirs:
             if log.isEnabledFor(logging.DEBUG):
-                log.warn("No input directories available for midasid=%s", midasid)
+                log.warn("No input directories available for midasid=%s", self.midasid)
                 log.debug("Input dirs:\n  %s\n  %s", str(self.revdatadir), str(self.upldatadir))
             raise SIPDirectoryNotFound(msg="No input directories available", sys=self)
-
-        self.nerd = nerdrec
-        self.pod  = podrec
+
 
     @property
     def input_dirs(self):
@@ -885,6 +897,8 @@ def apply_pod(self, pod, validate=True, force=False, lock=True):
     def _apply_pod(self, pod, validate=True, force=False):
         if not isinstance(pod, (str, unicode, Mapping)):
             raise NERDTypeError("dict", type(pod), "POD Dataset")
+        self.log.debug("%s: refreshing available input directories", self.midasid)
+        self.sip.refresh_indirs()
         self.ensure_base_bag()
 
         log.debug("BagBuilder log has %s formatters:\n%s", len(self.bagbldr.log.handlers),

python/nistoar/pdr/publish/midas3/service.py

@@ -908,6 +908,9 @@ def preserve_new(self, pod, async=None):
                                          + halted)
 
         pod['_preserve'] = 'new'
+        if worker.is_working():
+            self.log.info("Note: POD processing is currently in progress; final processing for "+
+                          "publication will follow.")
         self._apply_pod_async(pod, async)
         if async and worker.is_working():
             # wait around for a little while to see if finishes quickly
@@ -930,6 +933,8 @@ def _check_pod_for_preservation(self, pod):
         if pod.get('accessLevel') == 'non-public':
             raise ejs.ValidationError("Unacceptable accessLevel property value for preservation: " +
                                       str(pod.get('accessLevel')))
+        if pod.get("_preserve"):
+            del pod['_preserve']
 
     def preserve_update(self, pod, async=None):
         """
@@ -1063,7 +1068,11 @@ def launch(self):
         def queue_POD(self, pod):
             self.ensure_qlock()
             with self.qlock:
-                write_json(pod, self.next_pod)
+                if pod.get('_preserve'):
+                    self.log.debug("%s: Queuing POD for preservation", self.id)
+                    write_json(pod, self.presv_pod)
+                else:
+                    write_json(pod, self.next_pod)
 
         def mark_for_preservation(self, asupdate=False):
             # NOTE: use of this function is DEPRECATED