issue 55: test and (#59)

* move some code around

Part of the problem with run ids is that they are controlled from two locations. The simplest control is in dawgie.pl.farm and the other part is on dawgie.pl.scheduler. The first step is to pull the run ID bits from dawgie.pl.farm.dispatch() into their own functions and then test them to make sure they work as expected.

* update testing

test_06.Schedule.test_update() now checks the result of the update and makes sure everyone has the correct run ID. The test, however, is failing because not all of the children are being scheduled. This would explain why tasks have the wrong run IDs. The more complete verification is now that all children are scheduled and that they have the correct run ID.

* actual fixes

Updated dawgie.pl.schedule._priors(). Seems it was checking for dawgie.Task when it should have been dawgie.Algorithm. Fixed this and the test workds just fine.

Python/dawgie/pl/farm.py

@@ -216,13 +216,8 @@ def delta_time_to_string(diff):
 
 insights = {}
 def dispatch():
-    # pylint: disable=too-many-branches,too-many-statements
-    if dawgie.pl.start.sdp.waiting_on_crew() and not _agency:
-        log.info("farm.dispatch: Waiting for crew to finish...")
-        return
-    if not dawgie.pl.start.sdp.is_pipeline_active():
-        log.info("Pipeline is not active. Returning from farm.dispatch().")
-        return
+    # pylint: disable=too-many-branches
+    if not something_to_do(): return
 
     jobs = dawgie.pl.schedule.next_job_batch()
 
@@ -231,15 +226,7 @@ def dispatch():
         pass
 
     for j in jobs:
-        runid = j.get ('runid', None)
-
-        if not runid:
-            runid = dawgie.db.next()
-            log.critical (('New run ID ({0:d}) for algorithm {1:s} ' +
-                           'trigger by the event: ').format (runid, j.tag) +
-                          j.get ('event', 'Not Specified'))
-            pass
-
+        runid = rerunid (j)
         j.set ('status', dawgie.pl.schedule.State.running)
         fm = j.get ('factory').__module__
         fn = j.get ('factory').__name__
@@ -305,3 +292,24 @@ def plow():
                                        Foreman(), dawgie.context.worker_backlog)
     twisted.internet.task.LoopingCall(dispatch).start(5).addErrback (dawgie.pl.LogDeferredException(None, 'dispatching scheduled jobs to workers on the farm').log)
     return
+
+def rerunid (job):
+    runid = job.get ('runid', None)
+
+    if runid is None:
+        runid = dawgie.db.next()
+        log.critical (('New run ID ({0:d}) for algorithm {1:s} ' +
+                       'trigger by the event: ').format (runid, job.tag) +
+                      job.get ('event', 'Not Specified'))
+        pass
+    return runid
+
+def something_to_do():
+    # pylint: disable=too-many-branches,too-many-statements
+    if dawgie.pl.start.sdp.waiting_on_crew() and not _agency:
+        log.info("farm.dispatch: Waiting for crew to finish...")
+        return False
+    if not dawgie.pl.start.sdp.is_pipeline_active():
+        log.info("Pipeline is not active. Returning from farm.dispatch().")
+        return False
+    return True

Python/dawgie/pl/schedule.py

@@ -118,9 +118,9 @@ def _is_asp (n:dawgie.pl.dag.Node)->bool:
 
 def _priors (node):
     result = []
+    if isinstance (node, dawgie.Algorithm): result = node.previous()
     if isinstance (node, dawgie.Analyzer): result = node.traits()
     if isinstance (node, dawgie.Regression): result = node.variables()
-    if isinstance (node, dawgie.Task): result = node.previous()
     return result
 
 def _to_name (m):

Test/test_06.py

@@ -185,6 +185,15 @@ def test_tasks(self):
     def test_update(self):
         root = [n for n in filter (lambda n:n.tag.startswith ('network.'),
                                    dawgie.pl.schedule.ae.at)][0]
-        dawgie.pl.schedule.update (['a.b.c.d.e'], root, 12)
+        root.get ('doing').add ('fred')
+        for c in root:
+            if c.tag == 'disk.engine': c.get ('doing').add ('fred')
+            pass
+        dawgie.pl.schedule.que.clear()
+        dawgie.pl.schedule.update (['12.fred.network.analyzer.test.image'],
+                                   root, 12)
+        self.assertEqual (len (root), len (dawgie.pl.schedule.que))
+        for n in dawgie.pl.schedule.que:\
+            self.assertEqual (12, n.get ('runid'), n.tag)
         return
     pass

Test/test_10.py

@@ -0,0 +1,66 @@
+'''
+
+COPYRIGHT:
+Copyright (c) 2015-2019, California Institute of Technology ("Caltech").
+U.S. Government sponsorship acknowledged.
+
+All rights reserved.
+
+LICENSE:
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+- Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+- Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+- Neither the name of Caltech nor its operating division, the Jet
+Propulsion Laboratory, nor the names of its contributors may be used to
+endorse or promote products derived from this software without specific prior
+written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+NTR:
+'''
+
+import dawgie.pl.dag
+import dawgie.pl.farm
+import unittest
+
+class Farm(unittest.TestCase):
+    def test_rerunid (self):
+        n = dawgie.pl.dag.Node('a')
+        r = dawgie.pl.farm.rerunid (n)
+        self.assertEqual (2, r)
+        n.set ('runid', 17)
+        r = dawgie.pl.farm.rerunid (n)
+        self.assertEqual (17, r)
+        n.set ('runid', 0)
+        r = dawgie.pl.farm.rerunid (n)
+        self.assertEqual (0, r)
+        return
+
+    def something_to_do():
+        self.assertFalse (dawgie.pl.farm.something_to_do())
+        dawgie.pl.farm._agency = True
+        self.assertFalse (dawgie.pl.farm.something_to_do())
+        dawgie.pl.start.sdp.wait_on_crew.clear()
+        self.assertFalse (dawgie.pl.farm.something_to_do())
+        dawgie.pl.start.sdp.start == 'running'
+        self.assertTrue (dawgie.pl.farm.something_to_do())
+        return
+    pass