Merge branch 'master' into relax-unique-lower-threshold

README.md

@@ -36,6 +36,7 @@ if you prefer to install verkko in a separate environment. Alternatively, you ca
 To install Verkko from github (for developers only) run:
 
     git clone https://github.com/marbl/verkko.git
+    git checkout <desired branch> (optional if you want to use a branch for development/compilation and not master)
     cd verkko/src
     git submodule init && git submodule update
     make -j32

src/Snakefiles/c4-findErrors.sm

@@ -24,6 +24,8 @@ checkpoint configureFindErrors:
         finished          = '0-correction/configureFindErrors.finished'
     log:
         err               = '0-correction/configureFindErrors.err'
+    params:
+        batchsize         = int(config['cor_batch_size']) * 1000
     threads:
         1
     resources:
@@ -41,22 +43,28 @@ set -e
 
 mkdir -p find-errors-jobs
 
-bat=1
-bgn=1
-len=19999
-tot=\$({VERKKO}/bin/sqStoreDumpMetaData -S ../{input.seqstore} -stats | grep total-reads | awk '{{ print \$1 }}')
-
-while [ \$bgn -lt \$tot ] ; do
-  end=\`expr \$bgn + \$len\`
-  if [ \$end -gt \$tot ] ; then
-    end=\$tot
-  fi
-
-  echo > \`printf find-errors-jobs/%04d.range \$bat\` \$bgn \$end
-
-  bat=\`expr \$bat + 1\`
-  bgn=\`expr \$end + 1\`   #  OEA expects inclusive ranges.
-done
+{VERKKO}/bin/sqStoreDumpMetaData -S ../{input.seqstore} -reads \\\\
+| \\\\
+awk -v lim={params.batchsize} \\\\
+'function output(bat, bgn, end, file) {{
+   file=sprintf("find-errors-jobs/%04d.range", bat)
+   print bgn, end > file
+   close(file)
+ }}
+ BEGIN {{ bat=1; bgn=1; end=1; len=0; }}
+ /^[ \\t]*[0-9]+/ {{
+   if (len + \$3 > lim) {{
+     output(bat, bgn, end)
+     bat++
+     bgn = \$1  #  OEA expects inclusive ranges
+     len =  0
+   }}
+   end  = \$1
+   len += \$3
+ }}
+ END {{
+   output(bat, bgn, end)
+ }}'
 
 touch ../{output.finished}
 EOF

src/verkko.sh

@@ -124,9 +124,7 @@ mer_window=75
 
 cor_min_read=4000
 cor_min_overlap=1000
-
-cor_index_batches=16
-cor_overlap_batches=32
+cor_batch_size=30000
 
 #  buildGraph, parameters for MBG
 mbg_baseK=1001
@@ -410,6 +408,7 @@ while [ $# -gt 0 ] ; do
     elif [ "$opt" = "--correct-k-mer-window" ] ;       then mer_window=$arg;       shift
     elif [ "$opt" = "--correct-min-read-length" ] ;    then cor_min_read=$arg;     shift
     elif [ "$opt" = "--correct-min-overlap-length" ] ; then cor_min_overlap=$arg;  shift
+    elif [ "$opt" = "--correct-batch-size" ] ;         then cor_batch_size=$arg;  shift
 
     elif [ "$opt" = "--correct-index-batches" ] ;      then cor_index_batches=$arg;    shift
     elif [ "$opt" = "--correct-overlap-batches" ] ;    then cor_overlap_batches=$arg;  shift
@@ -517,7 +516,7 @@ if [ "x$mbg" = "x" ] ; then    #  Not set by an option,
   mbg=${verkko}/bin/MBG        #  Set it to our bin/ directory.
 fi
 if [ ! -e $mbg ] ; then        #  Not in the bin directory,
-  mbg=$(which MBG)             #  Set it to whatever is in the PATH.
+  mbg=$(which MBG 2>/dev/null) #  Set it to whatever is in the PATH.
 fi
 if [ "x$mbg" != "x" ]; then
   mbg=$(fullpath $mbg)
@@ -527,17 +526,17 @@ if [ "x$graphaligner" = "x" ] ; then
   graphaligner=${verkko}/bin/GraphAligner
 fi
 if [ ! -e $graphaligner ] ; then
-  graphaligner=$(which GraphAligner)
+  graphaligner=$(which GraphAligner 2>/dev/null)
 fi
 if [ "x$graphaligner" != "x" ]; then
-  graphaligner=$(fullpath $graphaligner)
+  graphaligner=$(fullpath $graphaligner 2>/dev/null)
 fi
 
 if [ "x$mashmap" = "x" ] ; then
   mashmap=${verkko}/bin/mashmap
 fi
 if [ ! -e $mashmap ] ; then
-  mashmap=$(which mashmap)
+  mashmap=$(which mashmap 2>/dev/null)
 fi
 if [ "x$mashmap" != "x" ]; then
   mashmap=$(fullpath $mashmap)
@@ -547,7 +546,7 @@ if [ "x$winnowmap" = "x" ] ; then
   winnowmap=${verkko}/bin/winnowmap
 fi
 if [ ! -e $winnowmap ] ; then
-  winnowmap=$(which winnowmap)
+  winnowmap=$(which winnowmap 2>/dev/null)
 fi
 if [ "x$winnowmap" != "x" ]; then
   winnowmap=$(fullpath $winnowmap)
@@ -557,7 +556,7 @@ if [ "x$bwa" = "x" ] ; then
   bwa=${verkko}/bin/bwa
 fi
 if [ ! -e $bwa ] ; then
-  bwa=$(which bwa)
+  bwa=$(which bwa 2>/dev/null)
 fi
 if [ "x$bwa" != "x" ]; then
   bwa=$(fullpath $bwa)
@@ -567,7 +566,7 @@ if [ "x$samtools" = "x" ] ; then
   samtools=${verkko}/bin/samtools
 fi
 if [ ! -e $samtools ] ; then
-  samtools=$(which samtools)
+  samtools=$(which samtools 2>/dev/null)
 fi
 if [ "x$samtools" != "x" ]; then
   samtools=$(fullpath $samtools)
@@ -809,10 +808,13 @@ if [ "x$help" = "xhelp" -o "x$errors" != "x" ] ; then
     echo ""
     echo "ADVANCED MODULE PARAMETERS (expert users):"
     echo "HiFi read correction:"
-    echo "    --correct-k-mer-size"
-    echo "    --correct-k-mer-window"
-    echo "    --correct-min-read-length"
-    echo "    --correct-min-overlap-length"
+    echo "    --correct-k-mer-size           Set the k-mer size to use for finding overlaps (201)"
+    echo "    --correct-k-mer-window         Set the window size for sketching reads when finding overlaps (75)"
+    echo "    --correct-min-read-length      Set the overall minimum read length (4000)"
+    echo "    --correct-min-overlap-length   Set the minimum overlap length (1000)"
+    echo "    --correct-hash-bits            Set the overlapper table size (25)"
+    echo "    --correct-batch-size           Set the RED batch size, in Mbp (30000)"
+    echo "                                   (might also need to adjust --red-run)"
     echo "    "
     echo "MBG:"
     echo "    --base-k"
@@ -912,6 +914,8 @@ echo >> ${outd}/verkko.yml ""
 echo >> ${outd}/verkko.yml "cor_index_batches:   '${cor_index_batches}'"
 echo >> ${outd}/verkko.yml "cor_overlap_batches: '${cor_overlap_batches}'"
 echo >> ${outd}/verkko.yml ""
+echo >> ${outd}/verkko.yml "cor_batch_size:      '${cor_batch_size}'"
+echo >> ${outd}/verkko.yml ""
 echo >> ${outd}/verkko.yml "#  build-graph, MBG"
 echo >> ${outd}/verkko.yml "mbg_baseK:           '${mbg_baseK}'"
 echo >> ${outd}/verkko.yml "mbg_maxK:            '${mbg_maxK}'"