Merge branch 'dev' into 'master'

v1.0.0 rls

See merge request research/medaka!403

.gitlab-ci.yml

@@ -15,7 +15,7 @@ test:
           libssl-dev libffi-dev valgrind
     script:
         - make venv
-        - make -j 4 test mem_check
+        - make -j 1 test mem_check
         - make docs
 
 .many-linux: &many-linux-def

CHANGELOG.md

@@ -1,3 +1,13 @@
+v1.0.0
+-------
+Models, features and fixes release
+
+* Consensus models for Guppy 3.6.0.
+* Add functionality for auto-download of older models.
+* Fix to methylation aggregation.
+* VCF annotation tool.
+
+
 v0.12.1
 -------
 Minor release harmonising versions of htslib/samtools dependencies.

Makefile

@@ -45,25 +45,36 @@ $(BINCACHEDIR)/tabix: | libhts.a $(BINCACHEDIR)
 $(BINCACHEDIR)/bgzip: | libhts.a $(BINCACHEDIR)
 	cp submodules/samtools-${SAMVER}/htslib-${SAMVER}/$(@F) $@
 
+
 .PHONY: clean_htslib
 clean_htslib:
 	cd submodules/samtools-${SAMVER} && make clean || exit 0
 	cd submodules/samtools-${SAMVER}/htslib-${SAMVER} && make clean || exit 0
 
 
 MINIMAPVER=2.17
-submodules/minimap2-$(MINIMAPVER)/Makefile:
+$(BINCACHEDIR)/minimap2: | $(BINCACHEDIR)
+	@echo Compiling $(@F)
 	cd submodules; \
 		curl -L -o minimap2-${MINIMAPVER}.tar.bz2 https://github.com/lh3/minimap2/releases/download/v${MINIMAPVER}/minimap2-${MINIMAPVER}.tar.bz2; \
 		tar -xjf minimap2-${MINIMAPVER}.tar.bz2; \
 	    rm -rf minimap2-${MINIMAPVER}.tar.bz2
-
-$(BINCACHEDIR)/minimap2: submodules/minimap2-$(MINIMAPVER)/Makefile | $(BINCACHEDIR)
-	@echo Compiling $(@F)
 	cd submodules/minimap2-${MINIMAPVER} && make
 	cp submodules/minimap2-${MINIMAPVER}/minimap2 $@
 
 
+$(BINCACHEDIR)/bcftools: | $(BINCACHEDIR)
+	@echo Making $(@F)
+	if [ ! -d submodules/bcftools-v${SAMVER} ]; then \
+		cd submodules; \
+		curl -L -o bcftools-v${SAMVER}.tar.bz2 https://github.com/samtools/bcftools/releases/download/${SAMVER}/bcftools-${SAMVER}.tar.bz2; \
+		tar -xjf bcftools-v${SAMVER}.tar.bz2; \
+		cd bcftools-${SAMVER}; \
+		make; \
+	fi
+	cp submodules/bcftools-${SAMVER}/bcftools $@
+
+
 SPOAVER=3.0.0
 $(BINCACHEDIR)/spoa: | $(BINCACHEDIR)
 	@echo Making $(@F)
@@ -99,17 +110,6 @@ $(BINCACHEDIR)/racon: | $(BINCACHEDIR)
 		make;
 	cp submodules/racon-v${RACONVER}/build/bin/racon $@
 
-$(BINCACHEDIR)/bcftools: | $(BINCACHEDIR)
-	@echo Making $(@F)
-	if [ ! -d submodules/bcftools-v${SAMVER} ]; then \
-		cd submodules; \
-		curl -L -o bcftools-v${SAMVER}.tar.bz2 https://github.com/samtools/bcftools/releases/download/${SAMVER}/bcftools-${SAMVER}.tar.bz2; \
-		tar -xjf bcftools-v${SAMVER}.tar.bz2; \
-		cd bcftools-${SAMVER}; \
-		make; \
-	fi
-	cp submodules/bcftools-${SAMVER}/bcftools $@
-
 
 $(BINCACHEDIR)/vcf2fasta: | $(BINCACHEDIR)
 	cd src/vcf2fasta && g++ -std=c++11 \
@@ -157,27 +157,36 @@ test: install
 		--cov-fail-under=80 --cov-report term-missing
 
 
-.PHONY: mem_check
-mem_check: install pileup
-	${IN_VENV} && python -c "import medaka.test.test_counts as tc; tc.create_simple_bam('mem_test.bam', tc.simple_data['calls'])"
-	valgrind --error-exitcode=1 --tool=memcheck ./pileup mem_test.bam ref:1-8 || (ret=$$?; rm mem_test.bam* && exit $$ret)
-	rm -rf mem_test.bam*
-
-
 .PHONY: clean
 clean: clean_htslib
 	(${IN_VENV} && python setup.py clean) || echo "Failed to run setup.py clean"
 	rm -rf libhts.a libmedaka.abi3.so venv build dist/ medaka.egg-info/ __pycache__ medaka.egg-info
 	find . -name '*.pyc' -delete
 
 
+.PHONY: mem_check
+mem_check: install pileup
+	${IN_VENV} && python -c "import medaka.test.test_counts as tc; tc.create_simple_bam('mem_test.bam', tc.simple_data['calls'])"
+	valgrind --error-exitcode=1 --tool=memcheck ./pileup mem_test.bam ref:1-8 || (ret=$$?; rm mem_test.bam* && exit $$ret)
+	rm -rf mem_test.bam*
+
+
 pileup: libhts.a
 	gcc -pthread  -g -Wall -fstack-protector-strong -D_FORTIFY_SOURCE=2 -fPIC -std=c99 -msse3 -O3 \
 		-Isrc -Isubmodules/samtools-1.9/htslib-1.9 \
 		src/medaka_common.c src/medaka_counts.c src/medaka_bamiter.c libhts.a \
 		-lm -lz -llzma -lbz2 -lpthread -lcurl -lcrypto \
 		-o $(@) -std=c99 -msse3 -O3
 
+
+trim_reads: libhts.a
+	gcc -pthread -pg -g -Wall -fstack-protector-strong -D_FORTIFY_SOURCE=2 -fPIC -std=c99 -msse3 -O3 \
+		-Isrc -Isubmodules/samtools-1.9/htslib-1.9 \
+		src/medaka_common.c src/medaka_trimbam.c src/medaka_bamiter.c libhts.a \
+		-lz -llzma -lbz2 -lpthread -lcurl -lcrypto \
+		-o $(@) -std=c99 -msse3 -O3
+
+
 .PHONY: wheels
 wheels:
 	docker run -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/build-wheels.sh /io 5 6
@@ -192,7 +201,7 @@ pypi_build/bin/activate:
 
 .PHONY: sdist
 sdist: pypi_build/bin/activate scripts/mini_align submodules/samtools-$(SAMVER)/Makefile
-	${IN_BUILD} && python setup.py sdist
+	${IN_BUILD} && MEDAKA_DIST=1 python setup.py sdist
 
 
 # Documentation

build-wheels.sh

@@ -2,6 +2,7 @@
 # Usage: ./build-wheels.sh <workdir> <pyminorversion1> <pyminorversion2> ...
 set -e -x
 export MANYLINUX=1
+export MEDAKA_DIST=1
 
 PACKAGE_NAME='medaka'
 

build.py

@@ -1,3 +1,4 @@
+import itertools
 import os
 
 from cffi import FFI
@@ -13,23 +14,26 @@
 ffibuilder = FFI()
 ffibuilder.set_source("libmedaka",
     r"""
+    #include "kvec.h"
     #include "medaka_bamiter.h"
     #include "medaka_common.h"
     #include "medaka_counts.h"
+    #include "medaka_trimbam.h"
+    #include "medaka_pytrimbam.h"
     #include "fastrle.h"
     #include "kseq.h"
 
     """,
     libraries=libraries,
     library_dirs=library_dirs,
     include_dirs=[src_dir, htslib_dir],
-    sources=[os.path.join(src_dir, x) for x in ('medaka_bamiter.c', 'medaka_common.c', 'medaka_counts.c', 'fastrle.c')],
+    sources=[os.path.join(src_dir, x) for x in ('medaka_bamiter.c', 'medaka_common.c', 'medaka_counts.c', 'fastrle.c', 'medaka_trimbam.c', 'medaka_pytrimbam.c')],
     extra_compile_args=['-std=c99', '-msse3', '-O3'],
     extra_objects=['libhts.a']
 )
 
 cdef = []
-for header in ('medaka_counts.h','fastrle.h'):
+for header in ('medaka_counts.h','fastrle.h', 'medaka_pytrimbam.h'):
     with open(os.path.join(src_dir, header), 'r') as fh:
         # remove directives
         lines = ''.join(x for x in fh.readlines() if not x.startswith('#'))

medaka/__init__.py

@@ -4,7 +4,7 @@
 import os
 import subprocess
 
-__version__ = '0.12.1'
+__version__ = '1.0.0'
 
 
 def check_minimap2_version():

medaka/common.py

@@ -616,6 +616,19 @@ def grouper(gen, batch_size=4):
         yield batch
 
 
+def roundrobin(*iterables):
+    """Take items from iterables in a round-robin."""
+    pending = len(iterables)
+    nexts = itertools.cycle(iter(it).__next__ for it in iterables)
+    while pending:
+        try:
+            for next in nexts:
+                yield next()
+        except StopIteration:
+            pending -= 1
+            nexts = itertools.cycle(itertools.islice(nexts, pending))
+
+
 def print_data_path():
     """Print data directory containing models."""
     print(resource_filename(__package__, 'data'))