dib-lab · ctb · Aug 20, 2023 · Aug 20, 2023 · Aug 21, 2023 · Aug 21, 2023
diff --git a/01.basic-benchmarks.ipynb b/01.basic-benchmarks.ipynb
diff --git a/02.basic-benchmarks-seaborn.ipynb b/02.basic-benchmarks-seaborn.ipynb
diff --git a/README.md b/README.md
@@ -1,4 +1,26 @@
-## command to get file sizes
+# Benchmarks for branchwater paper
+
+This repo contains the benchmarking code for
+
+[Sourmash Branchwater Enables Lightweight Petabyte-Scale Sequence Search](https://dib-lab.github.io/2022-paper-branchwater-software/) ([github repo](https://github.com/dib-lab/2022-paper-branchwater-software/)), Irber et al., 2022.
+
+[doi: 10.1101/2022.11.02.514947](https://www.biorxiv.org/content/10.1101/2022.11.02.514947v1.full)
+
+## Running
+
+Allocate 64 threads and 64 GB:
+```!
+srun -p high2 --time=72:00:00 --nodes=1 --cpus-per-task 64 --mem 80GB --pty bash
+```
+
+then run the snakemake workflow:
+```
+snakemake -c 64 -p -s benchmarking.snakefile  --resources only_one_job=1 -k
+```
+
+## appendix
+
+### command to get file sizes
 
 ```
 find . -type f -printf "%k %p\n" | tee ~/scratch/magsearch/ls-wort.txt

diff --git a/benchmarking.snakefile b/benchmarking.snakefile
@@ -1,172 +1,184 @@
-sra_search_bin="/home/ctbrown/scratch/magsearch/bin/searcher"
+#sra_search_bin="/home/ctbrown/scratch/magsearch/bin/searcher"
+manysearch_cmd = "sourmash scripts manysearch "
+#manysearch_cmd = "sourmash scripts manysearch --ignore-abundance"
 
 rule all:
     input:
         expand("benchmarks/{x}_vs_{y}.txt", x=['a'],
                y=['a', 'b', 'c', 'd', 'e']),
+        expand("outputs/output_{x}_vs_{y}.csv", x=['a'],
+               y=['a', 'b', 'c', 'd', 'e']),
+
+rule big:
+    input:
+        expand("benchmarks/a_vs_a_1000_t{t}.txt", t=[4,8,16]),
         "outputs/output_a_vs_largest_10k.csv",
+        expand("benchmarks/a_vs_catalog.txt"),
         expand("benchmarks/a_{n}_vs_a.txt", n=range(100, 1000, 100)),
         expand("benchmarks/a_vs_a_{n}.txt", n=range(1000, 10000, 1000)),
-        expand("benchmarks/a_vs_a_1000_t{t}.txt", t=[4,8,16]),
-        expand("benchmarks/a_vs_catalog.txt"),
+
+rule threads:
+    input:
+        expand("benchmarks/a_vs_a_1000_t{t}.txt", t=[4,8,16,24,32,40,48,56,64]),
+
 
 rule a_vs_a:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-a.txt",
     output:
         csv="outputs/output_a_vs_a.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_a.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_sub_vs_a:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a-{n}.sigs.txt",
         against="data/wort-list-a.txt",
     output:
         csv="outputs/output_a_{n}_vs_a.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_{n}_vs_a.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_a_sub:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-a-{n}.txt",
     output:
         csv="outputs/output_a_vs_a_{n}.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_a_{n}.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_b:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-b.txt",
     output:
         csv="outputs/output_a_vs_b.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_b.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_c:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-c.txt",
     output:
         csv="outputs/output_a_vs_c.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_c.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_d:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
-        against="data/wort-list-d.txt",
+        against="wort-list.d.mf.csv",
     output:
         csv="outputs/output_a_vs_d.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_d.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_e:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-e.txt",
     output:
         csv="outputs/output_a_vs_e.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_e.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_largest:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-largest-10k.txt",
     output:
         csv="outputs/output_a_vs_largest_10k.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_largest_10k.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads}
     """
 
 rule a_vs_a_sub_threads:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/wort-list-a-{n}.txt",
     output:
         csv="outputs/output_a_vs_a_{n}_t{thr}.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_a_{n}_t{thr}.txt"
-    threads: 64
+    threads: 64                 # max out threads so nothing else running
     shell: """
-        export RAYON_NUM_THREADS={wildcards.thr}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against}
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {wildcards.thr}
     """
 
 
 rule a_vs_catalog:
     input:
-        bin=sra_search_bin,
         queries="data/gtdb-list-a.sigs.txt",
         against="data/metagenomes-catalog.txt",
     output:
         csv="outputs/output_a_vs_catalog.csv",
+    resources:
+        only_one_job=1,
     benchmark:
         "benchmarks/a_vs_catalog.txt"
     threads: 32
     shell: """
-        export RAYON_NUM_THREADS={threads}
-        {input.bin} -k 31 --scaled=1000 -o {output.csv} \
-            {input.queries} {input.against} || true
+        {manysearch_cmd} -k 31 --scaled=1000 -o {output.csv} \
+            {input.queries} {input.against} -c {threads} || true
     """
diff --git a/benchmarks/a_100_vs_a.txt b/benchmarks/a_100_vs_a.txt
diff --git a/benchmarks/a_200_vs_a.txt b/benchmarks/a_200_vs_a.txt
diff --git a/benchmarks/a_300_vs_a.txt b/benchmarks/a_300_vs_a.txt
diff --git a/benchmarks/a_400_vs_a.txt b/benchmarks/a_400_vs_a.txt
diff --git a/benchmarks/a_500_vs_a.txt b/benchmarks/a_500_vs_a.txt
diff --git a/benchmarks/a_600_vs_a.txt b/benchmarks/a_600_vs_a.txt
diff --git a/benchmarks/a_700_vs_a.txt b/benchmarks/a_700_vs_a.txt
diff --git a/benchmarks/a_800_vs_a.txt b/benchmarks/a_800_vs_a.txt
diff --git a/benchmarks/a_900_vs_a.txt b/benchmarks/a_900_vs_a.txt
diff --git a/benchmarks/a_vs_a.txt b/benchmarks/a_vs_a.txt
diff --git a/benchmarks/a_vs_a_1000.txt b/benchmarks/a_vs_a_1000.txt
diff --git a/benchmarks/a_vs_a_1000_t16.txt b/benchmarks/a_vs_a_1000_t16.txt
diff --git a/benchmarks/a_vs_a_1000_t36.txt b/benchmarks/a_vs_a_1000_t36.txt
diff --git a/benchmarks/a_vs_a_1000_t4.txt b/benchmarks/a_vs_a_1000_t4.txt
diff --git a/benchmarks/a_vs_a_1000_t40.txt b/benchmarks/a_vs_a_1000_t40.txt
diff --git a/benchmarks/a_vs_a_1000_t44.txt b/benchmarks/a_vs_a_1000_t44.txt
diff --git a/benchmarks/a_vs_a_1000_t48.txt b/benchmarks/a_vs_a_1000_t48.txt
diff --git a/benchmarks/a_vs_a_1000_t52.txt b/benchmarks/a_vs_a_1000_t52.txt
diff --git a/benchmarks/a_vs_a_1000_t56.txt b/benchmarks/a_vs_a_1000_t56.txt
diff --git a/benchmarks/a_vs_a_1000_t60.txt b/benchmarks/a_vs_a_1000_t60.txt
diff --git a/benchmarks/a_vs_a_1000_t64.txt b/benchmarks/a_vs_a_1000_t64.txt
diff --git a/benchmarks/a_vs_a_1000_t8.txt b/benchmarks/a_vs_a_1000_t8.txt
diff --git a/benchmarks/a_vs_a_2000.txt b/benchmarks/a_vs_a_2000.txt
diff --git a/benchmarks/a_vs_a_3000.txt b/benchmarks/a_vs_a_3000.txt
diff --git a/benchmarks/a_vs_a_4000.txt b/benchmarks/a_vs_a_4000.txt
diff --git a/benchmarks/a_vs_a_5000.txt b/benchmarks/a_vs_a_5000.txt
diff --git a/benchmarks/a_vs_a_6000.txt b/benchmarks/a_vs_a_6000.txt
diff --git a/benchmarks/a_vs_a_7000.txt b/benchmarks/a_vs_a_7000.txt
diff --git a/benchmarks/a_vs_a_8000.txt b/benchmarks/a_vs_a_8000.txt
diff --git a/benchmarks/a_vs_a_9000.txt b/benchmarks/a_vs_a_9000.txt
diff --git a/benchmarks/a_vs_b.txt b/benchmarks/a_vs_b.txt
diff --git a/benchmarks/a_vs_c.txt b/benchmarks/a_vs_c.txt
diff --git a/benchmarks/a_vs_catalog.txt b/benchmarks/a_vs_catalog.txt
diff --git a/benchmarks/a_vs_d.txt b/benchmarks/a_vs_d.txt
@@ -1,2 +1,2 @@
 s	h:m:s	max_rss	max_vms	max_uss	max_pss	io_in	io_out	mean_load	cpu_time
-1505.0091	0:25:05	16107.48	19187.17	16137.28	16137.32	93738.74	1.02	2605.24	39209.39
+559.7348	0:09:19	38303.28	110077.33	38193.48	38200.80	34302.56	160.97	2433.26	13623.33
diff --git a/benchmarks/a_vs_e.txt b/benchmarks/a_vs_e.txt
diff --git a/benchmarks/a_vs_largest_10k.txt b/benchmarks/a_vs_largest_10k.txt
diff --git a/environment.yml b/environment.yml
@@ -0,0 +1,13 @@
+name: branchwater-bench
+channels:
+    - conda-forge
+    - bioconda
+    - defaults
+dependencies:
+    - sourmash>=4.8.3,<5
+    - snakemake-minimal>=7,<8
+    - pip
+    - rust
+    - maturin>=1,<2
+    - pip:
+      - pyo3_branchwater==0.7.0