Merge pull request #318 from NOAA-GFDL/317.fre-pp-run-double-check

Updates to 'fre pp run' to account for delay in starting Cylc scheduler

fre/pp/run_script.py

@@ -1,6 +1,7 @@
 ''' fre pp run '''
 
 import subprocess
+import time
 import click
 
 def pp_run_subtool(experiment, platform, target):
@@ -9,10 +10,25 @@ def pp_run_subtool(experiment, platform, target):
     <experiment>__<platform>__<target>
     """
 
+    # Check to see if the workflow is already running
     name = experiment + '__' + platform + '__' + target
+    result = subprocess.run(['cylc', 'scan', '--name', f"^{name}$"], capture_output=True).stdout.decode('utf-8')
+    if len(result):
+        print("Workflow already running!")
+        return
+
+    # If not running, start it
     cmd = f"cylc play {name}"
     subprocess.run(cmd, shell=True, check=True)
 
+    # wait 30 seconds for the scheduler to come up; then confirm
+    print("Workflow started; waiting 30 seconds to confirm")
+    time.sleep(30)
+    result = subprocess.run(['cylc', 'scan', '--name', f"^{name}$"], capture_output=True).stdout.decode('utf-8')
+
+    if not len(result):
+        raise Exception('Cylc scheduler was started without error but is not running after 30 seconds')
+
 @click.command()
 def _pp_run_subtool(experiment, platform, target):
     ''' entry point to run for click '''