Merge pull request #1073 from maxplanck-ie/develop

3.1.0

.github/workflows/linux.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.11', '3.12']
+        python-version: ['3.11', '3.12' ,'3.13']
         optdeps: [".", ".[actions]", ".[docs]"]
     steps:
     - uses: actions/checkout@v4
@@ -100,7 +100,8 @@ jobs:
           'CONDA_PREPROCESSING_ENV',
           'CONDA_NONCODING_RNASEQ_ENV',
           'CONDA_SAMBAMBA_ENV',
-          'CONDA_pysam_ENV'
+          'CONDA_pysam_ENV',
+          'CONDA_FQLINT_ENV'
         ]
     runs-on: ubuntu-latest
     steps:

.github/workflows/osx.yml

@@ -31,7 +31,8 @@ jobs:
           'CONDA_PREPROCESSING_ENV',
           'CONDA_NONCODING_RNASEQ_ENV',
           'CONDA_SAMBAMBA_ENV',
-          'CONDA_pysam_ENV'
+          'CONDA_pysam_ENV',
+          'CONDA_FQLINT_ENV'
         ]
     runs-on: macos-latest
     steps:
@@ -46,6 +47,6 @@ jobs:
           pip install .
       - name: createEnvsOSX
         run: |
-          conda config --add subdirs osx-64
+          conda config --set subdir osx-64
           snakePipes createEnvs --only ${{matrix.envs}}
      

conda-recipe/meta.yaml

@@ -1,4 +1,4 @@
-{% set version = "3.0.0" %}
+{% set version = "3.1.0" %}
 
 package:
   name: snakepipes
@@ -14,9 +14,10 @@ build:
 
 requirements:
   host:
-    - python >=3
+    - python >=3, < 3.13
     - pip
     - seaborn
+    - setuptools
   run:
     - python >=3.11
     - snakemake >=8

docs/conf.py

@@ -15,7 +15,6 @@
 import sys
 import os
 from importlib.metadata import version as importlibversion
-import sphinx_rtd_theme
 
 # to allow readthedocs to compile without installing some dependencies
 import mock
@@ -150,7 +149,7 @@
 
 # import them both locally and on rtd
 html_theme = 'sphinx_rtd_theme'  # 'alabaster' 'sphinx_rtd_theme'
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the

docs/content/News.rst

@@ -1,6 +1,21 @@
 snakePipes News
 ===============
 
+snakePipes 3.1.0
+________________
+
+ * installation test for python 3.13
+ * bulk mode in makePairs wf
+ * Removal of --user flag in createIndices. Organism yamls are by default written to both installation folder and output folder.
+
+snakePipes 3.0.0
+________________
+
+* clusteryaml omitted for profiles
+* All workflows have '-' removed from their name
+* toml file installation
+* makePairs mode introduced
+
 snakePipes 2.9.0
 ________________
 

docs/content/setting_up.rst

@@ -63,7 +63,7 @@ or for the pytests:
 Configuring snakePipes
 ----------------------
 
-Finally, at least one file (``defaults.yaml``) should be modified to match your compute infrastructure. The location of this file can be found out by executing:
+Finally, config files ``defaults.yaml`` and snakemake profile should be modified to match your compute infrastructure. The location of this file can be found out by executing:
 
 .. code:: bash
 
@@ -86,7 +86,7 @@ If you want to use the snakepipes_genericprofile, make sure to review the follow
 
  * ``module load slurm &&`` - could be omitted
  * ``resources.partition`` - set to your slurm partition
- * ``conda-prefix`` - set to your preferred location where snakePipes environments should be stored
+ * ``conda-prefix`` - set to your preferred location where snakePipes environments should be stored. You can set this value by running `snakePipes config --condaEnvDir` and providing the respective path.
  * ``resources`` - make sure default resources make sense for your infrastructure
  * ``ccancel.sh`` - refers to the ccancel.sh file inside the profile directory and contains instructions on how to kill submitted jobs (on failure / interruption of snakemake). The module command could be omitted here as before
 

docs/content/workflows/makePairs.rst

@@ -6,8 +6,8 @@ makePairs
 What it does
 ------------
 
-The snakePipes makePairs workflow allows users to process their HiC data from raw fastq files to HiC matrices in
-an allele-specific manner. The workflow utilized mapping by bwa, followed by analysis
+The snakePipes makePairs workflow allows users to process their HiC/uC data from raw fastq files to HiC matrices (in
+an allele-specific manner). The workflow utilizes mapping by bwa, followed by analysis
 using `pairtools <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC9949071/>`__ . The workflow follows the `example workflow described in the documentation of pairtools <https://pairtools.readthedocs.io/en/latest/examples/pairtools_phase_walkthrough.html>`__ , 
 which explains each step in detail and would be useful for new users to have a look at. 
 Currently the output matrices are produced in the `.pairs <https://pairtools.readthedocs.io/en/latest/formats.html>`__ format.

docs/index.rst

@@ -45,7 +45,8 @@ Quick start
 
 * Download genome fasta and annotations for an your organism, and build indexes, Check in :ref:`createIndices`
 
-* Configure snakePipes with paths to organism and cluster configs on your system using snakePipes config. For detailed information, run:
+* Configure snakePipes with paths to organism and snakemake configs on your system using snakePipes config. Importantly, take care to set `--condaEnvDir` parameter, which defaults to `/tmp`. 
+  For detailed information, run:
 
 .. code:: bash
 

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "snakePipes"
 description = 'Snakemake workflows and wrappers for NGS data processing from the MPI-IE'
 readme = "README.md"
-version = "3.0.0"
+version = "3.1.0"
 keywords = [
     "DNAmapping",
     "ChIPSeq",
@@ -24,8 +24,7 @@ authors = [
 ]
 
 classifiers = [
-    "Intended Audience :: Bioinformaticians",
-    "Intended Audience :: Biologists",
+    "Intended Audience :: Science/Research",
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python :: 3",   
 ]

snakePipes/common_functions.py

@@ -42,7 +42,9 @@ def set_env_yamls():
             'CONDA_NONCODING_RNASEQ_ENV': 'envs/noncoding.yaml',
             'CONDA_SAMBAMBA_ENV': 'envs/sambamba.yaml',
             'CONDA_pysam_ENV': 'envs/pysam.yaml',
-            'CONDA_SEACR_ENV': 'envs/chip_seacr.yaml'}
+            'CONDA_SEACR_ENV': 'envs/chip_seacr.yaml',
+            'CONDA_FQLINT_ENV': 'envs/fqlint.yaml'
+            }
 
 
 def merge_dicts(x, y):

snakePipes/parserCommon.py

@@ -139,7 +139,7 @@ def snpArguments(defaults):
     snpargs = parser.add_argument_group('Allele-specific mapping arguments')
     snpargs.add_argument("--VCFfile",
                          default='',
-                         help="VCF file to create N-masked genomes (default: 'None')")
+                         help="VCF file to create N-masked genomes (default: 'None'). Note that for the makePairs workflow this file is assumed to be gzipped and indexed (with tabix).")
 
     snpargs.add_argument("--strains",
                          default='',

snakePipes/shared/defaults.yaml

@@ -19,4 +19,4 @@ max_thread: 25
 #print tools versions
 toolsVersion: True
 oldConfig:
-configMode: manual
+configMode: manual

snakePipes/shared/rules/ChIP_peak_calling.snakefile

@@ -122,7 +122,7 @@ rule MACS2_peak_qc:
 
         # compute peak genome coverage
         peak_len=`awk '{{total+=$3-$2}}END{{print total}}' {params.peaks}`
-        genome_size=`awk '{{total+=$3-$2}}END{{print total}}' {params.genome_index}`
+        genome_size=`awk '{{total+=$3-$2}}END{{printf( "%14d",total) }}' {params.genome_index}`
         genomecov=`bc -l <<< "$peak_len/$genome_size"`
 
         # write peak-based QC metrics to output file

snakePipes/shared/rules/FASTQ.snakefile

@@ -5,7 +5,7 @@ if pairedEnd or pipeline=="scrnaseq":
             r2 = indir+"/{sample}"+reads[1]+ext
         output:
             temp("originalFASTQ/{sample}.valid")
-        conda: CONDA_SHARED_ENV
+        conda: CONDA_FQLINT_ENV
         shell:"""
             fq lint {input.r1} {input.r2}
             touch {output}
@@ -16,7 +16,7 @@ else:
             r1 = indir+"/{sample}"+reads[0]+ext
         output:
             temp("originalFASTQ/{sample}.valid")
-        conda: CONDA_SHARED_ENV
+        conda: CONDA_FQLINT_ENV
         shell:"""
             fq lint {input.r1}
             touch {output}

snakePipes/shared/rules/envs/fqlint.yaml

@@ -0,0 +1,4 @@
+channels:
+ - bioconda
+dependencies:
+ - fq = 0.12.0

snakePipes/shared/rules/envs/makePairs.yaml

@@ -2,9 +2,9 @@ name: pairtools_phased
 channels:
   - conda-forge
   - bioconda
-  - defaults
 dependencies:
   - bwa
+  - cooler
   - samtools
   - tabix
   - bcftools

snakePipes/shared/rules/envs/shared.yaml

@@ -12,9 +12,8 @@ dependencies:
  - fastqc = 0.12.1
  - cutadapt = 4.6
  - trim-galore = 0.6.10
- - multiqc = 1.19
+ - multiqc = 1.23
  - fastp = 0.23.4
  - umi_tools = 1.1.4
- - fq = 0.11.0
  - pybigwig = 0.3.22
 

snakePipes/shared/rules/masked_genomeIndex.snakefile

@@ -2,7 +2,8 @@ from os.path import join, dirname
 import glob
 
 GENOMEDIR = os.path.dirname(genome_fasta)
-BASENAME = genome
+genome_alias = os.path.splitext(os.path.basename(genome))[0]
+BASENAME = genome_alias
 # define snpgenome_dir
 if allele_hybrid == 'dual':
     SNPdir = "snp_genome/" + strains[0] + "_" + \
@@ -50,17 +51,15 @@ else:
         params:
             strain1 = strains[0],
             SNPpath = os.path.abspath(VCFfile),
-
-            temp_out=temp("all_SNPs_" + strains[0] + "_GRCm38.txt.gz"),
+            temp_out=temp("all_SNPs_" + strains[0] + "_" + genome_alias + ".txt.gz"),
             out_bname=os.path.basename(SNPFile)
         conda: CONDA_SHARED_ENV
         shell:
             " ( [ -d snp_genome ] || mkdir -p snp_genome ) && cd snp_genome &&"
             " SNPsplit_genome_preparation"
             " --genome_build {BASENAME}"
             " --reference_genome {input.genome} --vcf_file {params.SNPpath}"
-            " --strain {params.strain1} && cp "
-            "{params.temp_out} {params.out_bname}"
+            " --strain {params.strain1}"
             "&& cd ../"
 
 if aligner == "STAR":