Merging latest changes for running on Compute Canada clusters

encode-wrapper/backend.conf

@@ -23,6 +23,8 @@ backend {
         submit = """
           ls ${singularity_container} $(echo ${singularity_bindpath} | tr , ' ') 1>/dev/null && (sbatch \
           --export=ALL \
+          --mail-type=END,FAIL --mail-user=$JOB_MAIL \
+          -A $RAP_ID \
           -J ${job_name} \
           -D ${cwd} \
           -o ${out} \
@@ -36,52 +38,42 @@ backend {
           ${"--account " + slurm_account} \
           ${"--gres gpu:" + gpu} \
           ${slurm_extra_param} \
-          --wrap "chmod u+x ${script} && SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1),${singularity_bindpath} singularity exec --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} ${script}")
+          --wrap "chmod u+x ${script} && SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1),${singularity_bindpath} singularity exec --cleanenv --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} ${script}")
         """
         kill = "scancel ${job_id}"
         check-alive = "squeue -j ${job_id}"
         job-id-regex = "Submitted batch job (\\d+).*"
       }
     }
 
-    sge_singularity {
+    pbs_singularity {
       actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
       config {
-        script-epilogue = "sleep 30 && sync"
+        script-epilogue = "sleep 30"
         concurrent-job-limit = 50
         runtime-attributes = """
-          String sge_pe = "shm"
           Int cpu = 1
           Int? gpu
-          Int? time
-          Int? memory_mb
-          String? sge_queue
-          String? sge_extra_param
+          Int time = 1
+          Int memory_mb = 1024
           String singularity_container
           String? singularity_bindpath
         """
         submit = """
-          ls ${singularity_container} $(echo ${singularity_bindpath} | tr , ' ') 1>/dev/null && (echo "chmod u+x ${script} && SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1),${singularity_bindpath} singularity exec --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} ${script}" | qsub \
-          -S /bin/sh \
-          -terse \
-          -b n \
+          ls ${singularity_container} $(echo ${singularity_bindpath} | tr , ' ') 1>/dev/null && (echo "chmod u+x ${script} && SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1),${singularity_bindpath} singularity exec --cleanenv --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} ${script}" | qsub \
           -N ${job_name} \
-          -wd ${cwd} \
           -o ${out} \
           -e ${err} \
-          ${if cpu>1 then "-pe " + sge_pe + " " + cpu else " "} \
-          ${"-l h_vmem=" + memory_mb/cpu + "m"} \
-          ${"-l s_vmem=" + memory_mb/cpu + "m"} \
-          ${"-l h_rt=" + time*3600} \
-          ${"-l s_rt=" + time*3600} \
-          ${"-q " + sge_queue} \
-          ${"-l gpu=" + gpu} \
-          ${sge_extra_param} \
-          -V)
+          -l nodes=1:ppn=${cpu} \
+          -l mem=${memory_mb}MB \
+          -l walltime=${time}:0:0 \
+          ${if gpu>1 then "-lngpus=" + gpu else ""} \
+          -V
+          )
         """
         kill = "qdel ${job_id}"
         check-alive = "qstat -j ${job_id}"
-        job-id-regex = "(\\d+)"
+        job-id-regex = "(\\d+).*"
       }
     }
 
@@ -96,7 +88,7 @@ backend {
           String? singularity_bindpath
         """
         submit = """
-          ls ${singularity_container} $(echo ${singularity_bindpath} | tr , ' ') 1>/dev/null && (chmod u+x ${script} && SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1),${singularity_bindpath} singularity exec --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} ${script} & echo $! && disown)
+          ls ${singularity_container} $(echo ${singularity_bindpath} | tr , ' ') 1>/dev/null && (chmod u+x ${script} && SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1),${singularity_bindpath} singularity exec --cleanenv --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} ${script} & echo $! && disown)
         """
         job-id-regex = "(\\d+)"
         check-alive = "ps -ef | grep -v grep | grep ${job_id}" 
@@ -110,112 +102,6 @@ backend {
         concurrent-job-limit = 10
       }
     }
-
-    sge {
-      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
-      config {
-        script-epilogue = "sleep 30 && sync"
-        concurrent-job-limit = 50
-        runtime-attributes = """
-        String sge_pe = "shm"
-        Int cpu = 1
-        Int? gpu
-        Int? time
-        Int? memory_mb
-        String? sge_queue
-        String? sge_extra_param
-        """
-        submit = """
-        qsub \
-        -S /bin/sh \
-        -terse \
-        -b n \
-        -N ${job_name} \
-        -wd ${cwd} \
-        -o ${out} \
-        -e ${err} \
-        ${if cpu>1 then "-pe " + sge_pe + " " + cpu else " "} \
-        ${"-l h_vmem=" + memory_mb/cpu + "m"} \
-        ${"-l s_vmem=" + memory_mb/cpu + "m"} \
-        ${"-l h_rt=" + time*3600} \
-        ${"-l s_rt=" + time*3600} \
-        ${"-q " + sge_queue} \
-        ${"-l gpu=" + gpu} \
-        ${sge_extra_param} \
-        -V \
-        ${script}
-        """
-        kill = "qdel ${job_id}"
-        check-alive = "qstat -j ${job_id}"
-        job-id-regex = "(\\d+)"
-      }
-    }
-
-    slurm {
-      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
-      config {
-        script-epilogue = "sleep 30"
-        concurrent-job-limit = 50        
-        runtime-attributes = """
-        Int cpu = 1
-        Int? gpu
-        Int? time
-        Int? memory_mb
-        String? slurm_partition
-        String? slurm_account
-        String? slurm_extra_param
-        """
-        submit = """
-        sbatch \
-        --export=ALL \
-        -J ${job_name} \
-        -D ${cwd} \
-        -o ${out} \
-        -e ${err} \
-        ${"-t " + time*60} \
-        -n 1 \
-        --ntasks-per-node=1 \
-        ${"--cpus-per-task=" + cpu} \
-        ${"--mem=" + memory_mb} \
-        ${"-p " + slurm_partition} \
-        ${"--account " + slurm_account} \
-        ${"--gres gpu:" + gpu} \
-        ${slurm_extra_param} \
-        --wrap "/bin/bash ${script}"
-        """
-        kill = "scancel ${job_id}"
-        check-alive = "squeue -j ${job_id}"
-        job-id-regex = "Submitted batch job (\\d+).*"
-      }
-    }
-
-    google {
-      actor-factory = "cromwell.backend.impl.jes.JesBackendLifecycleActorFactory"
-      config {
-        # Google project
-        project = "your-project-name"
-
-        # Base bucket for workflow executions
-        root = "gs://your-bucket-name"
-
-        concurrent-job-limit = 1000
-        genomics-api-queries-per-100-seconds = 1000
-        maximum-polling-interval = 600
-
-        genomics {
-          auth = "application-default"
-          compute-service-account = "default"
-          endpoint-url = "https://genomics.googleapis.com/"
-          restrict-metadata-access = false
-        }
-
-        filesystems {
-          gcs {
-            auth = "application-default"
-          }
-        }
-      }
-    }
   }
 }
 
@@ -238,13 +124,3 @@ call-caching {
   enabled = false
   invalidate-bad-cache-results = true
 }
-
-google {
-  application-name = "cromwell"
-  auths = [
-    {
-      name = "application-default"
-      scheme = "application_default"
-    }
-  ]
-}

encode-wrapper/backend_ihec_pbs_singularity.conf

@@ -0,0 +1,65 @@
+include required(classpath("application"))
+
+backend {
+  default = "Local"
+  providers {
+
+    pbs_singularity {
+      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+      config {
+        script-epilogue = "sleep 30"
+        concurrent-job-limit = 50
+        runtime-attributes = """
+          Int cpu = 1
+          Int? gpu
+          Int time = 1
+          Int memory_mb = 1024
+          String singularity_container
+          String? singularity_bindpath
+        """
+        submit = """
+          ls ${singularity_container} $(echo ${singularity_bindpath} | tr , ' ') 1>/dev/null && (echo "chmod u+x ${script} && SINGULARITY_BINDPATH=$(echo ${cwd} | sed 's/cromwell-executions/\n/g' | head -n1),${singularity_bindpath} singularity exec --cleanenv --home ${cwd} ${if defined(gpu) then '--nv' else ''} ${singularity_container} ${script}" | qsub \
+          -N ${job_name} \
+          -o ${out} \
+          -e ${err} \
+          -l nodes=1:ppn=${cpu} \
+          -l mem=${memory_mb}MB \
+          -l walltime=${time}:0:0 \
+          ${if gpu>1 then "-lngpus=" + gpu else ""} \
+          -V
+          )
+        """
+        kill = "qdel ${job_id}"
+        check-alive = "qstat -j ${job_id}"
+        job-id-regex = "(\\d+).*"
+      }
+    }
+
+    Local {
+      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
+      config {
+        concurrent-job-limit = 10
+      }
+    }
+  }
+}
+
+services {
+  LoadController {
+    class = "cromwell.services.loadcontroller.impl.LoadControllerServiceActor"
+    config {      
+      # disable it (for login nodes on Stanford SCG, Sherlock)
+      control-frequency = 21474834 seconds
+    }
+  }
+}
+
+system {
+  abort-jobs-on-terminate = true
+  graceful-server-shutdown = true
+}
+
+call-caching {
+  enabled = false
+  invalidate-bad-cache-results = true
+}