From 1c5a396d686d53d58833c1be10c9b555ffb0c1e0 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 23 Aug 2023 14:06:42 +0200 Subject: [PATCH 1/4] split up config files to be more modular --- conf/modules/base.config | 32 +++++++++ conf/{modules.config => modules/sra.config} | 79 --------------------- conf/modules/synapse.config | 69 ++++++++++++++++++ nextflow.config | 4 +- 4 files changed, 104 insertions(+), 80 deletions(-) create mode 100644 conf/modules/base.config rename conf/{modules.config => modules/sra.config} (58%) create mode 100644 conf/modules/synapse.config diff --git a/conf/modules/base.config b/conf/modules/base.config new file mode 100644 index 00000000..d7eb6c86 --- /dev/null +++ b/conf/modules/base.config @@ -0,0 +1,32 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Config file for defining DSL2 per module options and publishing paths +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Available keys to override module options: + ext.args = Additional arguments appended to command in module. + ext.args2 = Second set of arguments appended to command in module (multi-tool modules). + ext.args3 = Third set of arguments appended to command in module (multi-tool modules). + ext.prefix = File name prefix for output files. +---------------------------------------------------------------------------------------- +*/ + +// +// Generic process options for all workflows +// +process { + + publishDir = [ + path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + + withName: CUSTOM_DUMPSOFTWAREVERSIONS { + publishDir = [ + path: { "${params.outdir}/pipeline_info" }, + mode: params.publish_dir_mode, + pattern: '*_versions.yml' + ] + } + +} diff --git a/conf/modules.config b/conf/modules/sra.config similarity index 58% rename from conf/modules.config rename to conf/modules/sra.config index f5cb1c77..e6e9d168 100644 --- a/conf/modules.config +++ b/conf/modules/sra.config @@ -10,27 +10,6 @@ ---------------------------------------------------------------------------------------- */ -// -// Generic process options for all workflows -// -process { - - publishDir = [ - path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - - withName: CUSTOM_DUMPSOFTWAREVERSIONS { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - pattern: '*_versions.yml' - ] - } - -} - // // Process options for the SRA workflow // @@ -111,61 +90,3 @@ if (params.input_type == 'sra') { } } - -// -// Process options for the Synapse workflow -// -if (params.input_type == 'synapse') { - - process { - - withName: SYNAPSE_LIST { - ext.args = '--long' - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - - withName: SYNAPSE_GET { - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] - } - - withName: SYNAPSE_SHOW { - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - - withName: SYNAPSE_TO_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - enabled: false - ] - } - - withName: SYNAPSE_MERGE_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - - } - -} diff --git a/conf/modules/synapse.config b/conf/modules/synapse.config new file mode 100644 index 00000000..bcca94db --- /dev/null +++ b/conf/modules/synapse.config @@ -0,0 +1,69 @@ +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Config file for defining DSL2 per module options and publishing paths +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Available keys to override module options: + ext.args = Additional arguments appended to command in module. + ext.args2 = Second set of arguments appended to command in module (multi-tool modules). + ext.args3 = Third set of arguments appended to command in module (multi-tool modules). + ext.prefix = File name prefix for output files. +---------------------------------------------------------------------------------------- +*/ + +// +// Process options for the Synapse workflow +// +if (params.input_type == 'synapse') { + + process { + + withName: SYNAPSE_LIST { + ext.args = '--long' + publishDir = [ + path: { "${params.outdir}/metadata" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: SYNAPSE_GET { + publishDir = [ + [ + path: { "${params.outdir}/fastq" }, + mode: params.publish_dir_mode, + pattern: "*.fastq.gz" + ], + [ + path: { "${params.outdir}/fastq/md5" }, + mode: params.publish_dir_mode, + pattern: "*.md5" + ] + ] + } + + withName: SYNAPSE_SHOW { + publishDir = [ + path: { "${params.outdir}/metadata" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + withName: SYNAPSE_TO_SAMPLESHEET { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + enabled: false + ] + } + + withName: SYNAPSE_MERGE_SAMPLESHEET { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + + } + +} diff --git a/nextflow.config b/nextflow.config index 6eb5c18d..486c058c 100644 --- a/nextflow.config +++ b/nextflow.config @@ -217,7 +217,9 @@ manifest { } // Load modules.config for DSL2 module specific options -includeConfig 'conf/modules.config' +includeConfig 'conf/modules/base.config' +includeConfig 'conf/modules/sra.config' +includeConfig 'conf/modules/synapse.config' // Function to ensure that resource requirements don't go beyond // a maximum limit From bc7fd25c21e1a058d00870513d1eb742847d3b0f Mon Sep 17 00:00:00 2001 From: maxulysse Date: Wed, 23 Aug 2023 16:21:07 +0200 Subject: [PATCH 2/4] fix lint --- .nf-core.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.nf-core.yml b/.nf-core.yml index 253f86e2..269ee3ed 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -1,5 +1,8 @@ repository_type: pipeline lint: + actions_ci: false + files_exist: + - conf/modules.config files_unchanged: - .github/CONTRIBUTING.md - .github/ISSUE_TEMPLATE/bug_report.yml @@ -7,7 +10,6 @@ lint: - assets/sendmail_template.txt - lib/NfcoreTemplate.groovy - .gitattributes - actions_ci: false multiqc_config: false nextflow_config: - "params.validationShowHiddenParams" From 82fa5000fe8a468a4e202f5765d78b12a93c7e0f Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 24 Aug 2023 15:50:52 +0200 Subject: [PATCH 3/4] Update nextflow.config Co-authored-by: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> --- nextflow.config | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nextflow.config b/nextflow.config index 486c058c..6b7463d3 100644 --- a/nextflow.config +++ b/nextflow.config @@ -218,8 +218,12 @@ manifest { // Load modules.config for DSL2 module specific options includeConfig 'conf/modules/base.config' -includeConfig 'conf/modules/sra.config' -includeConfig 'conf/modules/synapse.config' +if ( params.input_type == 'sra' ) { + includeConfig 'conf/modules/sra.config' +} +if ( params.input_type == 'synapse' ) { + includeConfig 'conf/modules/synapse.config' +} // Function to ensure that resource requirements don't go beyond // a maximum limit From 011addc3cf5d6ad036adc435cd30eaf88df2582d Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 24 Aug 2023 15:52:05 +0200 Subject: [PATCH 4/4] remove if statement --- conf/modules/sra.config | 122 +++++++++++++++++------------------- conf/modules/synapse.config | 83 ++++++++++++------------ 2 files changed, 99 insertions(+), 106 deletions(-) diff --git a/conf/modules/sra.config b/conf/modules/sra.config index e6e9d168..15802b41 100644 --- a/conf/modules/sra.config +++ b/conf/modules/sra.config @@ -13,80 +13,76 @@ // // Process options for the SRA workflow // -if (params.input_type == 'sra') { +process { - process { - - withName: SRA_IDS_TO_RUNINFO { - publishDir = [ - path: { "${params.outdir}/metadata" }, - enabled: false - ] - } - - withName: SRA_RUNINFO_TO_FTP { - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - - withName: SRA_FASTQ_FTP { - ext.args = '--retry 5 --continue-at - --max-time 1200' - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] - } + withName: SRA_IDS_TO_RUNINFO { + publishDir = [ + path: { "${params.outdir}/metadata" }, + enabled: false + ] + } - withName: SRATOOLS_PREFETCH { - publishDir = [ - path: { "${params.outdir}/sra" }, - enabled: false - ] - } + withName: SRA_RUNINFO_TO_FTP { + publishDir = [ + path: { "${params.outdir}/metadata" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } - withName: SRATOOLS_FASTERQDUMP { - ext.args = '--split-files --include-technical' - publishDir = [ + withName: SRA_FASTQ_FTP { + ext.args = '--retry 5 --continue-at - --max-time 1200' + publishDir = [ + [ path: { "${params.outdir}/fastq" }, mode: params.publish_dir_mode, pattern: "*.fastq.gz" + ], + [ + path: { "${params.outdir}/fastq/md5" }, + mode: params.publish_dir_mode, + pattern: "*.md5" ] - } + ] + } - withName: SRA_TO_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - enabled: false - ] - } + withName: SRATOOLS_PREFETCH { + publishDir = [ + path: { "${params.outdir}/sra" }, + enabled: false + ] + } - withName: SRA_MERGE_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } + withName: SRATOOLS_FASTERQDUMP { + ext.args = '--split-files --include-technical' + publishDir = [ + path: { "${params.outdir}/fastq" }, + mode: params.publish_dir_mode, + pattern: "*.fastq.gz" + ] + } - withName: MULTIQC_MAPPINGS_CONFIG { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } + withName: SRA_TO_SAMPLESHEET { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + enabled: false + ] + } + + withName: SRA_MERGE_SAMPLESHEET { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } + withName: MULTIQC_MAPPINGS_CONFIG { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] } } diff --git a/conf/modules/synapse.config b/conf/modules/synapse.config index bcca94db..2f52aa6f 100644 --- a/conf/modules/synapse.config +++ b/conf/modules/synapse.config @@ -13,57 +13,54 @@ // // Process options for the Synapse workflow // -if (params.input_type == 'synapse') { - process { +process { - withName: SYNAPSE_LIST { - ext.args = '--long' - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } - - withName: SYNAPSE_GET { - publishDir = [ - [ - path: { "${params.outdir}/fastq" }, - mode: params.publish_dir_mode, - pattern: "*.fastq.gz" - ], - [ - path: { "${params.outdir}/fastq/md5" }, - mode: params.publish_dir_mode, - pattern: "*.md5" - ] - ] - } + withName: SYNAPSE_LIST { + ext.args = '--long' + publishDir = [ + path: { "${params.outdir}/metadata" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } - withName: SYNAPSE_SHOW { - publishDir = [ - path: { "${params.outdir}/metadata" }, + withName: SYNAPSE_GET { + publishDir = [ + [ + path: { "${params.outdir}/fastq" }, + mode: params.publish_dir_mode, + pattern: "*.fastq.gz" + ], + [ + path: { "${params.outdir}/fastq/md5" }, mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + pattern: "*.md5" ] - } + ] + } - withName: SYNAPSE_TO_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - enabled: false - ] - } + withName: SYNAPSE_SHOW { + publishDir = [ + path: { "${params.outdir}/metadata" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] + } - withName: SYNAPSE_MERGE_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } + withName: SYNAPSE_TO_SAMPLESHEET { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + enabled: false + ] + } + withName: SYNAPSE_MERGE_SAMPLESHEET { + publishDir = [ + path: { "${params.outdir}/samplesheet" }, + mode: params.publish_dir_mode, + saveAs: { filename -> filename.equals('versions.yml') ? null : filename } + ] } }