Merge pull request #191 from maxulysse/better_tests

Update SRA workflow tests

.github/workflows/ci.yml

@@ -80,7 +80,7 @@ jobs:
         if: failure()
         run: |
           sudo apt install bat > /dev/null
-          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/output/pipeline_info/software_versions.yml
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
 
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v3

main.nf

@@ -72,8 +72,8 @@ if (params.input_type == 'synapse') {
 ========================================================================================
 */
 
-include { CUSTOM_DUMPSOFTWAREVERSIONS } from './modules/nf-core/custom/dumpsoftwareversions'
 include { MULTIQC_MAPPINGS_CONFIG     } from './modules/local/multiqc_mappings_config'
+include { CUSTOM_DUMPSOFTWAREVERSIONS } from './modules/nf-core/custom/dumpsoftwareversions'
 
 /*
 ========================================================================================
@@ -97,33 +97,25 @@ workflow NFCORE_FETCHNGS {
 
     ch_versions = Channel.empty()
 
-    //
     // WORKFLOW: Download FastQ files for SRA / ENA / GEO / DDBJ ids
-    //
     if (params.input_type == 'sra') {
         SRA(ch_ids)
         ch_versions = ch_versions.mix(SRA.out.versions)
-    //
-    // MODULE: Create a MultiQC config file with sample name mappings
-    //
+
+        // MODULE: Create a MultiQC config file with sample name mappings
         if (params.sample_mapping_fields) {
             MULTIQC_MAPPINGS_CONFIG(SRA.out.mappings)
             ch_versions = ch_versions.mix(MULTIQC_MAPPINGS_CONFIG.out.versions)
         }
-    //
+
     // WORKFLOW: Download FastQ files for Synapse ids
-    //
     } else if (params.input_type == 'synapse') {
         SYNAPSE(ch_ids, ch_synapse_config)
         ch_versions = ch_versions.mix(SYNAPSE.out.versions)
     }
 
-    //
     // MODULE: Dump software versions for all tools used in the workflow
-    //
-    CUSTOM_DUMPSOFTWAREVERSIONS (
-        ch_versions.unique().collectFile(name: 'collated_versions.yml')
-    )
+    CUSTOM_DUMPSOFTWAREVERSIONS(ch_versions.unique().collectFile(name: 'collated_versions.yml'))
 }
 
 /*

modules.json

@@ -8,8 +8,7 @@
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
                         "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
-                        "installed_by": ["modules"],
-                        "patch": "modules/nf-core/custom/dumpsoftwareversions/custom-dumpsoftwareversions.diff"
+                        "installed_by": ["modules"]
                     },
                     "custom/sratoolsncbisettings": {
                         "branch": "master",

nf-test.config

@@ -13,4 +13,7 @@ config {
 
     // run all test with the defined docker profile from the main nextflow.config
     profile ""
+
+    // Disable autosort Channels
+    autoSort false
 }

workflows/sra/main.nf

@@ -12,7 +12,7 @@ include { SRA_MERGE_SAMPLESHEET   } from '../../modules/local/sra_merge_samplesh
 
 /*
 ========================================================================================
-    IMPORT NF-CORE MODULES/SUBWORKFLOWS
+    IMPORT NF-CORE SUBWORKFLOWS
 ========================================================================================
 */
 
@@ -49,25 +49,24 @@ workflow SRA {
     )
     ch_versions = ch_versions.mix(SRA_RUNINFO_TO_FTP.out.versions.first())
 
-    SRA_RUNINFO_TO_FTP
+    ch_sra_metadata = SRA_RUNINFO_TO_FTP
         .out
         .tsv
         .splitCsv(header:true, sep:'\t')
-        .map {
-            meta ->
-                def meta_clone = meta.clone()
-                meta_clone.single_end = meta_clone.single_end.toBoolean()
-                return meta_clone
+        .map{ meta ->
+            def meta_clone = meta.clone()
+            meta_clone.single_end = meta_clone.single_end.toBoolean()
+            return meta_clone
         }
         .unique()
-        .set { ch_sra_metadata }
+
     ch_versions = ch_versions.mix(SRA_RUNINFO_TO_FTP.out.versions.first())
 
     fastq_files = Channel.empty()
 
     if (!params.skip_fastq_download) {
 
-        ch_sra_metadata
+        ch_sra_reads = ch_sra_metadata
             .map {
                 meta ->
                     [ meta, [ meta.fastq_1, meta.fastq_2 ] ]
@@ -76,7 +75,6 @@ workflow SRA {
                 ftp: it[0].fastq_1  && !params.force_sratools_download
                 sra: !it[0].fastq_1 || params.force_sratools_download
             }
-            .set { ch_sra_reads }
 
         //
         // MODULE: If FTP link is provided in run information then download FastQ directly via FTP and validate with md5sums
@@ -101,16 +99,15 @@ workflow SRA {
             FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads
         )
 
-        fastq_files
-            .map {
-                meta, fastq ->
-                    def reads = fastq instanceof List ? fastq.flatten() : [ fastq ]
-                    def meta_clone = meta.clone()
-                    meta_clone.fastq_1 = reads[0] ? "${params.outdir}/fastq/${reads[0].getName()}" : ''
-                    meta_clone.fastq_2 = reads[1] && !meta.single_end ? "${params.outdir}/fastq/${reads[1].getName()}" : ''
-                    return meta_clone
-            }
-            .set { ch_sra_metadata }
+        ch_sra_metadata = fastq_files.map { meta, fastq ->
+            def reads = fastq instanceof List ? fastq.flatten() : [ fastq ]
+            def meta_clone = meta.clone()
+
+            meta_clone.fastq_1 = reads[0] ? "${params.outdir}/fastq/${reads[0].getName()}" : ''
+            meta_clone.fastq_2 = reads[1] && !meta.single_end ? "${params.outdir}/fastq/${reads[1].getName()}" : ''
+
+            return meta_clone
+        }
     }
 
     //
@@ -133,10 +130,10 @@ workflow SRA {
     ch_versions = ch_versions.mix(SRA_MERGE_SAMPLESHEET.out.versions)
 
     emit:
-        fastq         = fastq_files
-        samplesheet   = SRA_MERGE_SAMPLESHEET.out.samplesheet
-        mappings      = SRA_MERGE_SAMPLESHEET.out.mappings
-        versions      = ch_versions.unique()
+    fastq         = fastq_files
+    samplesheet   = SRA_MERGE_SAMPLESHEET.out.samplesheet
+    mappings      = SRA_MERGE_SAMPLESHEET.out.mappings
+    versions      = ch_versions.unique()
 }
 
 /*

workflows/sra/tests/main.nf.test

@@ -1,14 +1,20 @@
-nextflow_pipeline {
+nextflow_workflow {
 
-    name "Test workflow: sra.nf"
-    script "main.nf"
+    name "Test workflow: sra/main.nf"
+    script "workflows/sra/main.nf"
+    workflow "SRA"
     tag "workflows"
     tag "workflows_sra"
     tag "sra_default_parameters"
 
     test("Parameters: default") {
 
         when {
+            workflow {
+                """
+                input[0] = Channel.from('DRR026872','DRR028935','ERR1109373','ERR1160846','GSE214215','GSM4907283','SRR12848126','SRR13191702','SRR14593545','SRR14709033','SRR9984183')
+                """
+            }
             params {
                 outdir = "$outputDir"
             }
@@ -17,7 +23,12 @@ nextflow_pipeline {
         then {
             assertAll(
                 { assert workflow.success },
-                { assert snapshot(process.out).match() }
+//                { assert snapshot(workflow.out).match() }
+                { assert new File("$outputDir/custom").exists() },
+                { assert new File("$outputDir/metadata").exists() },
+                { assert new File("$outputDir/pipeline_info").exists() },
+                { assert new File("$outputDir/samplesheet").exists() },
+                { assert snapshot(path("$outputDir/fastq/").list()).match("fastq") }
             )
         }
     }