test data tested and works

wtsi-hgi · Apr 30, 2024 · 509a86d · 509a86d
1 parent 70c5340
commit 509a86d
Show file tree

Hide file tree

Showing 19 changed files with 1,348 additions and 1,258 deletions.
diff --git a/assets/deploy_scripts/module_exacutables/yascp b/assets/deploy_scripts/module_exacutables/yascp
@@ -5,6 +5,7 @@ bold=$(tput bold)
 normal=$(tput sgr0)
 
 
+
 if [[ "$QUEUE" == '' ]];
     then
         export QUEUE='long'

diff --git a/bin/4.add_vdj.R b/bin/4.add_vdj.R
@@ -34,8 +34,7 @@ data_dir <- './'
 args = commandArgs(trailingOnly=TRUE)
 
 wnn_integrated_file = args[1]
-n2 = strsplit(as.character(wnn_integrated_file), split="__all_samples")[[1]][1]
-outname = paste0(n2,'__all_samples_integrated.vdj.RDS')
+
 
 myPalette <- colorRampPalette(rev(brewer.pal(11, "Spectral")))
 
@@ -155,8 +154,14 @@ vdj_dir <- paste0('./vdj/')
 dir.create(vdj_dir,showWarnings = F)
 names(immdata_TCR$data) <- tcr_names
 names(immdata_BCR$data) <- bcr_names
-saveRDS(immdata_BCR, file=paste0(vdj_dir,'/slemap_BCR.rds'))
-saveRDS(immdata_TCR, paste0(vdj_dir,'/slemap_TCR.rds'))
+
+n2 = strsplit(as.character(wnn_integrated_file), split="__all_samples")[[1]][1]
+outname = paste0(n2,'__all_samples_integrated.vdj.RDS')
+outname_BCR = paste0(n2,'__all_samples_integrated.BCR.RDS')
+outname_TCR = paste0(n2,'__all_samples_integrated.TCR.RDS')
+
+saveRDS(immdata_BCR, file=outname_BCR)
+saveRDS(immdata_TCR, file=outname_TCR)
 
 saveRDS(all_samples, file=outname)
 
diff --git a/bin/concordance_calculations.py b/bin/concordance_calculations.py
diff --git a/bin/gather_minimal_dataset.py b/bin/gather_minimal_dataset.py
@@ -413,6 +413,9 @@ def gather_donor(donor_id, ad, ad_lane_raw, azimuth_annot, qc_obs, columns_outpu
     ad.obs = ad.obs.loc[:,~ad.obs.columns.duplicated()]
     if write_h5:
         path1=os.path.join(outdir, oufnam + '.h5ad')
+        ad.obs['qc.filter.pass.AZ:L0'] = ad.obs['qc.filter.pass.AZ:L0'].astype('bool')
+        ad.obs['cell_passes_hard_filters'] = ad.obs['cell_passes_hard_filters'].astype('bool')
+        ad.obs['qc.filter.pass'] = ad.obs['qc.filter.pass'].astype('bool')
         ad.write(path1,compression='gzip')
 
     return {

diff --git a/conf/base.conf b/conf/base.conf
@@ -9,7 +9,7 @@
 */
 
 params{
-    input = 'cellbender' 
+    input = 'cellbender' //# cellbender|cellranger
     rsync_to_web_file = "${launchDir}/yascp/bin/rsync_to_web.sh"
     profile = 'normal_run'
     citeseq = false
@@ -59,11 +59,12 @@ params{
     cluster_validate_resolution_keras = true
     input_tables_column_delimiter = '\t'
     outdir= "${launchDir}/results"
+    tracedir                   = "${params.outdir}/pipeline_info"
     do_deconvolution = true
     split_bam = false
     run_multiplet = true
     utilise_gpu = true
-    split_ad_per_bach = false
+    split_ad_per_bach = true
     cellbender_resolution_to_use='0pt1'
     reference_assembly_fasta_dir = "https://yascp.cog.sanger.ac.uk/public/10x_reference_assembly"
     webtransfer = false
@@ -239,246 +240,31 @@ process {
     withLabel:process_medium_memory {
         memory = { 30.GB * task.attempt }
     }
-
-    withName: SCRUBLET{
-        maxRetries    = 3
-        errorStrategy = { task.attempt > 2 ? 'ignore' : 'retry' }
-    }
-
-
-    withName: DOUBLET_DECON{
-        maxRetries    = 3
-        errorStrategy = { task.attempt > 2 ? 'ignore' : 'retry' }
-        memory = { 100.GB * task.attempt }
-    }
-
-    withName: AZIMUTH{
-        maxRetries    = 3
-        errorStrategy = { task.attempt > 2 ? 'ignore' : 'retry' }
-    }
-    withName: CELLTYPIST{
-        maxRetries    = 3
-        errorStrategy = { task.attempt > 2 ? 'ignore' : 'retry' }
-    }
-    withName: cellbender__remove_background{
-        maxRetries    = 2
-        //# errorStrategy = { task.attempt > 2 ? 'ignore' : 'retry' }
-    }
-
-    withLabel:error_ignore {
-        errorStrategy = 'ignore'
-    }
-    withLabel:error_retry {
-        errorStrategy = 'retry'
-        maxRetries    = 2
-    }
-
-    withName:cluster_validate_resolution_keras{
-        maxForks=4
+    withName: ASSESS_CALL_RATE{
         maxRetries    = 3
-        memory = { 60.GB * task.attempt }
-        time   = { 12.h   * task.attempt }
+        memory = { 10.GB * task.attempt }
         errorStrategy = { task.attempt > 2 ? 'ignore' : 'retry' }
     }
 
-    withName: CELLTYPIST{
-        maxForks=7
-    }
-
-    withName: CELLTYPE_FILE_MERGE{
-        memory = { 60.GB * task.attempt }
-    }
-
-    withName: NORMALISE_AND_PCA{
-        maxForks=7
-        errorStrategy = 'retry'
-        memory = { 50.GB * task.attempt}
-        maxRetries    = 8
-        cpus   = 4
-    }
-
-    withName: LISI{
-        maxForks=7
-        memory =300.GB
-    }
-
-    withName: RESOLVE_POOL_VCFS{
-        cpus =  1
-        memory = { 1.GB * task.attempt }
-        time   = {  12.h   * task.attempt }
-        maxRetries    = 3
-    }
-
-    withName: SPLIT_BATCH_H5AD{
-        cpus =  2
-        memory = { 25.GB * task.attempt * 0.5}
-        time   = {  12.h   * task.attempt }
-        maxRetries    = 3
-    }
-
-    withName: SUBSET_GENOTYPE2{
-        cpus =  2
-        memory = { 1.GB * task.attempt }
-        time   = {  12.h   * task.attempt }
-        maxRetries    = 3
-    }
-
-    withName: JOIN_STUDIES_MERGE{
-        cpus =  1
-        memory = { 20.GB * task.attempt }
-        time   = {  12.h   * task.attempt }
-        maxRetries    = 3
-    }    
-
-    withName: FREEBAYES{
-        cpus =  1
-        time   = {  12.h   * task.attempt }
-        maxRetries    = 2
-    }
-
-    withName: VIREO_ADD_SAMPLE_PREFIX{
-        cpus =  1
-        memory = { 2.GB * task.attempt }
-        time   = {  12.h   * task.attempt }
-        maxRetries    = 3
-    }
-
-
-    withName: REPLACE_GT_DONOR_ID2{
-        cpus =  1
-        memory = { 1.GB * task.attempt }
-        time   = 12.h
-        maxRetries    = 3
-    }
-
-    withName: JOIN_CHROMOSOMES{
-        cpus =  1
-        memory = { 2.GB * task.attempt }
-        time   = 12.h
-        maxRetries    = 3
-    }
-
-    withName: serialize_known_markers{
-        cpus =  1
-        memory = { 1.GB * task.attempt }
-        time   = {  12.h   * task.attempt }
-        maxRetries    = 3
-    }
-
-    withName: OUTLIER_FILTER{
-        errorStrategy = 'retry'
-        memory = { 50.GB * task.attempt}
-        maxRetries    = 8
-    }
-
-
-
-    withName: cluster{
-        cpus   = {  3     * task.attempt }
-    }
-
-    withName: LISI{
-        maxForks=7
-        errorStrategy = 'retry'
-        maxRetries    = 8
-        memory = { 200.GB * task.attempt}
-    }
-
-    withName: VIREO_GT_FIX_HEADER{
-        errorStrategy = 'retry'
-        maxRetries    = 4
-        cpus = 1
-        memory = { 1.GB * task.attempt }
-    }
-
-
-    withName: JOIN_CHROMOSOMES{
-        errorStrategy = 'retry'
-        maxRetries    = 4
-    }
-
-    withName: cluster{
-        errorStrategy = 'retry'
-        maxRetries    = 4
-    }
-
-    withName: SPLIT_BAM_BY_CELL_BARCODES {
-      cpus = 1
-      memory = { 8.GB * task.attempt}
-      time = 4.h
-    }
-
-    withName: CONCORDANCE_CALCLULATIONS{
-        cpus = 5
-        time   = {  12.h   * task.attempt }
-        memory = { 70.GB * task.attempt }
-    }
-
-    withName: OTHER_DONOR_CONCORDANCE_CALCLULATIONS{
-        cpus = 3
-        time   = {  6.h   * task.attempt }
-        memory = { 20.GB * task.attempt }
-    }
-
-
-    withName: CELLSNP{
-        memory = { 5.GB * task.attempt }
-    }
-
-    withName: DYNAMIC_DONOR_EXCLUSIVE_SNP_SELECTION{
-        cpus = 5
-        time   = { 12.h   * task.attempt }
-        memory = { 20.GB * task.attempt }
-    }
-
-    withName: prep_collectmetadata{
-        memory = { 150.MB * task.attempt }
-    }
-
-    withName: VIREO{
-        //# maxForks=7
-        cpus   = 5
-        time   = { 12.h   * task.attempt }
-        memory = { 70.GB * task.attempt }
-    }
-    withName: DSB_INTEGRATE{
-        memory = { 200.GB * task.attempt }
-        cpus   = {  4     * task.attempt }
-        maxRetries    = 3
-    }
-
-    withName: MULTIMODAL_INTEGRATION{
-        memory = { 200.GB * task.attempt }
-        cpus   = {  4     * task.attempt }
-        maxRetries    = 3
-    }
-
-    withName: umap_gather{
-        memory = { 100.GB * task.attempt }
-        errorStrategy = 'retry'
-        maxRetries    = 3
-    }
-
-    withName: DOUBLET_FINDER{
-        memory = { 100.GB * task.attempt }
-    }
-
-
-
-    withName: GT_MATCH_POOL_AGAINST_PANEL{
-        time   = { 24.h   * task.attempt }
-    }
-
-    withName: plot_predicted_sex{
-        memory = { 50.GB * task.attempt }
-        maxRetries = 5
-        cpus = 2
-
-    }
-
-
+}
 
 
+def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
+timeline {
+    enabled = true
+    file    = "${params.tracedir}/execution_timeline_${trace_timestamp}.html"
+}
+report {
+    enabled = true
+    file    = "${params.tracedir}/execution_report_${trace_timestamp}.html"
+}
+trace {
+    enabled = true
+    file    = "${params.tracedir}/execution_trace_${trace_timestamp}.txt"
+}
+dag {
+    enabled = true
+    file    = "${params.tracedir}/pipeline_dag_${trace_timestamp}.svg"
 }
 
 singularity {

diff --git a/conf/cellbender.conf b/conf/cellbender.conf
@@ -59,8 +59,6 @@ params {
                 value{
                     expected_nemptydroplets_umi_cutoff = 0
                     method_estimate_ncells = 'dropletutils::barcoderanks::inflection'
-                    //method_estimate_ncells = 'cellrangerv2::expected' //this method feeds in the cellranger estimate ncells
-                    //method_estimate_ncells = 'expected'
                     lower_bound_umis_estimate_ncells = 1000
                     method_estimate_nemptydroplets = 'dropletutils::barcoderanks::inflection,dropletutils::barcoderanks::knee,0.33'
                     lower_bound_umis_estimate_nemptydroplets = 10