diff --git a/Dockerfile b/Dockerfile index 7094f81b1..f2adb7d03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,34 @@ -# syntax=docker/dockerfile:1 +# syntax=docker/dockerfile:1.3 + +# ==================================================================================== +# NOTE this docker uses `--network=none` option for RUN directives, you need to either +# - activate BuildKit with `export DOCKER_BUILDKIT=1` +# - run this using buildx +# ==================================================================================== + +# NOTE the following must match the ENTRYPOINT +ARG install_path=/opt/V-dock +ARG vpipe_path=${install_path}/V-pipe +ARG envs_path=${install_path}/conda_envs +ARG test_data=/test-data +ARG virus_download_list ### ### Stage 0: download conda environments ### FROM snakemake/snakemake:latest AS create-envs +ARG install_path +ARG vpipe_path +ARG envs_path +ARG test_data + RUN apt-get update && apt-get install -y --no-install-recommends \ jdupes # TODO: only move workflow files -#COPY . /V-dock/V-pipe -WORKDIR /V-dock/V-pipe/ +#COPY . ${vpipe_path} +WORKDIR ${vpipe_path}/ COPY README.md ./README.md COPY LICENSE ./LICENSE COPY vpipe.snake ./vpipe.snake @@ -23,7 +41,7 @@ COPY functions.sh ./functions.sh COPY utils ./utils COPY init_project.sh ./init_project.sh -COPY tests/data /test-data +COPY tests/data ${test_data} WORKDIR /work @@ -31,20 +49,20 @@ WORKDIR /work RUN echo 'output:\n snv: true\n local: true\n global: true\n visualization: true\n QA: true' > config.yaml # TODO harmonize list with CI tests and Docker tests -RUN for virus in $(ls /test-data/); do echo "\n\n\e[36;1mvirus: ${virus}\e[0m\n" \ - && ln -sf "/test-data/${virus}/" ./samples \ +RUN for virus in ${virus_download_list:-$(ls ${test_data}/)}; do echo "\n\n\e[36;1mvirus: ${virus}\e[0m\n" \ + && ln -sf "${test_data}/${virus}/" ./samples \ && if test -e samples/samples.tsv; then cp -f samples/samples.tsv ./samples.tsv; fi \ - && snakemake -s /V-dock/V-pipe/vpipe.snake -j 1 --conda-create-envs-only --use-conda --conda-prefix /V-dock/conda_prefix --config "general={virus_base_config: ${virus}}" \ + && PYTHONUNBUFFERED=1 snakemake -s ${vpipe_path}/vpipe.snake -j 1 --conda-create-envs-only --use-conda --conda-prefix ${envs_path} --config "general={virus_base_config: ${virus}}" \ && rm -f samples samples.tsv \ - ; done - -RUN jdupes -Lr /V-dock/conda_prefix/ + ; done \ + && jdupes -Lr ${envs_path}/ ### ### Stage 1: base layer with V-pipe and environments ### -FROM snakemake/snakemake:latest AS vpipe-base +FROM snakemake/snakemake:latest AS vpipe-tests-base +ARG install_path # NOTE rsync only used with local scratch RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -53,47 +71,97 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # NOTE V-pipe/envs/*.yaml and conda_prefix/* must be in sync so that env checksums match -COPY --from=create-envs /V-dock /V-dock +COPY --from=create-envs ${install_path} ${install_path} ### ### Test 1: test the base layer with hiv ### -FROM vpipe-base AS test_hiv +FROM vpipe-tests-base AS test_hiv +ARG install_path +ARG vpipe_path +ARG envs_path +ARG test_data ENV virus=hiv + WORKDIR /work RUN echo 'output:\n snv: true\n local: true\n global: false\n visualization: true\n QA: true' > config.yaml -COPY --from=create-envs /test-data/${virus} ./samples +COPY --from=create-envs ${test_data}/${virus} ./samples RUN if test -e samples/samples.tsv; then cp -f samples/samples.tsv ./samples.tsv; fi -RUN snakemake -s /V-dock/V-pipe/vpipe.snake -j 4 --use-conda --conda-prefix /V-dock/conda_prefix --config "general={virus_base_config: ${virus}}" +# NOTE see top comment if `--network=none` breaks build process +RUN --network=none \ + PYTHONUNBUFFERED=1 snakemake -s ${vpipe_path}/vpipe.snake -j 4 --use-conda --conda-prefix ${envs_path} --config "general={virus_base_config: ${virus}}" \ + && echo "$(date --iso-8601=sec ; grep -E 'failed|for error' .snakemake/log/*.snakemake.log)" > ${install_path}/${virus}.teststamp ### ### Test 2: test the base layer with sars-cov-2 ### -FROM vpipe-base AS test_sars-cov-2 +FROM vpipe-tests-base AS test_sars-cov-2 +ARG install_path +ARG vpipe_path +ARG envs_path +ARG test_data ENV virus=sars-cov-2 + WORKDIR /work RUN echo 'output:\n snv: true\n local: true\n global: false\n visualization: true\n QA: true' > config.yaml -COPY --from=create-envs /test-data/${virus} ./samples +COPY --from=create-envs ${test_data}/${virus} ./samples RUN if test -e samples/samples.tsv; then cp -f samples/samples.tsv ./samples.tsv; fi -RUN snakemake -s /V-dock/V-pipe/vpipe.snake -j 4 --use-conda --conda-prefix /V-dock/conda_prefix --config "general={virus_base_config: ${virus}}" +# NOTE see top comment if `--network=none` breaks build process +RUN --network=none \ + PYTHONUNBUFFERED=1 snakemake -s ${vpipe_path}/vpipe.snake -j 4 --use-conda --conda-prefix ${envs_path} --config "general={virus_base_config: ${virus}}" \ + && echo "$(date --iso-8601=sec ; grep -E 'failed|for error' .snakemake/log/*.snakemake.log)" > ${install_path}/${virus}.teststamp + + + +### +### Final base: gather tests +### +FROM vpipe-tests-base as vpipe-final-base +ARG install_path + +# NOTE individual test can be forced using the following on github actions +COPY --from=test_hiv ${install_path}/hiv.teststamp ${install_path} +COPY --from=test_sars-cov-2 ${install_path}/sars-cov-2.teststamp ${install_path} ### ### Final stage: setup image ready to run ### -FROM vpipe-base +# ============================================= +# NOTE this will *skip* tests on GitHub actions +#FROM vpipe-test-base +#ARG install_path +#ARG vpipe_path +#ARG envs_path +# --------------------------------------------- +# HACK this will *force* tests on GitHub actions +FROM snakemake/snakemake:latest +ARG install_path +ARG vpipe_path +ARG envs_path + +RUN apt-get update && apt-get install -y --no-install-recommends \ + rsync \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=vpipe-final-base ${install_path} ${install_path} +# ============================================= + +MAINTAINER V-pipe Dev Team VOLUME /work WORKDIR /work +# NOTE current docker versions do not offer a way to bake the content of an ARG into an ENTRYPOINT ENTRYPOINT [ \ "snakemake", \ - "-s", "/V-dock/V-pipe/vpipe.snake", \ + "-s", "/opt/V-dock/V-pipe/vpipe.snake", \ "--use-conda", \ - "--conda-prefix", "/V-dock/conda_prefix" \ + "--conda-prefix", "/opt/V-dock/conda_envs" \ ]