From 577ebb83c2c2a27d4bf91bc3c56bb2bd6d4e7c15 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Mon, 7 Mar 2022 22:50:50 +0100 Subject: [PATCH 1/8] Make outdir a mandatory parameter --- CHANGELOG.md | 1 + .../pipeline-template/.github/PULL_REQUEST_TEMPLATE.md | 2 +- .../pipeline-template/.github/workflows/awsfulltest.yml | 2 ++ nf_core/pipeline-template/.github/workflows/ci.yml | 4 ++-- nf_core/pipeline-template/README.md | 8 ++++++-- nf_core/pipeline-template/conf/modules.config | 8 ++++---- nf_core/pipeline-template/conf/test.config | 2 +- nf_core/pipeline-template/conf/test_full.config | 2 +- nf_core/pipeline-template/docs/usage.md | 2 +- nf_core/pipeline-template/nextflow.config | 4 ++-- nf_core/pipeline-template/nextflow_schema.json | 4 ++-- 11 files changed, 23 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e23f42b3c..4294886cf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Add `.yamllint.yml` to avoid line length and document start errors ([#1407](https://github.com/nf-core/tools/issues/1407)) * Add `--publish_dir_mode` back into the pipeline template ([nf-core/rnaseq#752](https://github.com/nf-core/rnaseq/issues/752#issuecomment-1039451607)) * Add optional loading of of pipeline-specific institutional configs to `nextflow.config` +* Make `--outdir` a mandatory parameter ([nf-core/tools#1415](https://github.com/nf-core/tools/issues/1415)) ## General diff --git a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md index 230705bca7..a6eeaaccef 100644 --- a/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md +++ b/nf_core/pipeline-template/.github/PULL_REQUEST_TEMPLATE.md @@ -19,7 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/{{ name }}/t - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/{{ name }}/tree/master/.github/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the {{ name }} _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker`). +- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker` --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml index 4e3c168cc1..d2ba27e3a2 100644 --- a/nf_core/pipeline-template/.github/workflows/awsfulltest.yml +++ b/nf_core/pipeline-template/.github/workflows/awsfulltest.yml @@ -23,6 +23,8 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + pipeline: ${{ github.repository }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} parameters: | { diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index d8a7240f73..94edf32438 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -14,7 +14,7 @@ env: jobs: test: - name: Run workflow tests + name: Run pipeline with test data # Only run on push if this is the nf-core dev branch (merged PRs) if: {% raw %}${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ name }}') {% raw %}}}{% endraw %} runs-on: ubuntu-latest @@ -49,4 +49,4 @@ jobs: # For example: adding multiple test runs with different parameters # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker + nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 9d56da69e2..6dc1b11723 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -40,7 +40,7 @@ On release, automated continuous integration tests run the pipeline on a full-si 3. Download the pipeline and test it on a minimal dataset with a single command: ```console - nextflow run {{ name }} -profile test,YOURPROFILE + nextflow run {{ name }} -profile test,YOURPROFILE --outdir ``` Note that some form of configuration will be needed so that Nextflow knows how to fetch the required software. This is usually done in the form of a config profile (`YOURPROFILE` in the example command above). You can chain multiple config profiles in a comma-separated string. @@ -55,7 +55,11 @@ On release, automated continuous integration tests run the pipeline on a full-si ```console - nextflow run {{ name }} -profile --input samplesheet.csv --genome GRCh37 + nextflow run {{ name }} \ + --input samplesheet.csv \ + --outdir \ + --genome GRCh37 \ + -profile ``` ## Documentation diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index 6b9870439c..da58a5d881 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -3,10 +3,10 @@ Config file for defining DSL2 per module options and publishing paths ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Available keys to override module options: - ext.args = Additional arguments appended to command in module. - ext.args2 = Second set of arguments appended to command in module (multi-tool modules). - ext.args3 = Third set of arguments appended to command in module (multi-tool modules). - ext.prefix = File name prefix for output files. + ext.args = Additional arguments appended to command in module. + ext.args2 = Second set of arguments appended to command in module (multi-tool modules). + ext.args3 = Third set of arguments appended to command in module (multi-tool modules). + ext.prefix = File name prefix for output files. ---------------------------------------------------------------------------------------- */ diff --git a/nf_core/pipeline-template/conf/test.config b/nf_core/pipeline-template/conf/test.config index 9dc33f2d14..1e2ea2e6bb 100644 --- a/nf_core/pipeline-template/conf/test.config +++ b/nf_core/pipeline-template/conf/test.config @@ -5,7 +5,7 @@ Defines input files and everything required to run a fast and simple pipeline test. Use as follows: - nextflow run {{ name }} -profile test, + nextflow run {{ name }} -profile test, --outdir ---------------------------------------------------------------------------------------- */ diff --git a/nf_core/pipeline-template/conf/test_full.config b/nf_core/pipeline-template/conf/test_full.config index ad37132e98..87e4c96289 100644 --- a/nf_core/pipeline-template/conf/test_full.config +++ b/nf_core/pipeline-template/conf/test_full.config @@ -5,7 +5,7 @@ Defines input files and everything required to run a full size pipeline test. Use as follows: - nextflow run {{ name }} -profile test_full, + nextflow run {{ name }} -profile test_full, --outdir ---------------------------------------------------------------------------------------- */ diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index c5300c9700..8d84911d67 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -57,7 +57,7 @@ An [example samplesheet](../assets/samplesheet.csv) has been provided with the p The typical command for running the pipeline is as follows: ```console -nextflow run {{ name }} --input samplesheet.csv --genome GRCh37 -profile docker +nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile docker ``` This will launch the pipeline with the `docker` configuration profile. See below for more information about profiles. diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 005fa664f9..001c3261aa 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -24,7 +24,7 @@ params { max_multiqc_email_size = '25.MB' // Boilerplate options - outdir = './results' + outdir = null tracedir = "${params.outdir}/pipeline_info" publish_dir_mode = 'copy' email = null @@ -63,7 +63,7 @@ try { System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config") } -// Load {{ name }} custom profiles from different institutions. +// Load {{ name }} custom profiles from different institutions. // Warning: Uncomment only if a pipeline-specific instititutional config already exists on nf-core/configs! // try { // includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index fd95d67238..3daa4b36e3 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -26,8 +26,8 @@ }, "outdir": { "type": "string", - "description": "Path to the output directory where the results will be saved.", - "default": "./results", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", "fa_icon": "fas fa-folder-open" }, "email": { From d787f61f3c49cd7bf5aa76e9015e23e8105b77e2 Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 8 Mar 2022 09:25:45 +0000 Subject: [PATCH 2/8] Update nextflow_schema.json --- nf_core/pipeline-template/nextflow_schema.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 3daa4b36e3..4e64f555de 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -11,7 +11,8 @@ "fa_icon": "fas fa-terminal", "description": "Define where the pipeline should find input data and save output data.", "required": [ - "input" + "input", + "output" ], "properties": { "input": { From 03993e4452e139744d26cb345518946d9765112b Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 8 Mar 2022 09:26:02 +0000 Subject: [PATCH 3/8] Update nextflow_schema.json --- nf_core/pipeline-template/nextflow_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/nextflow_schema.json b/nf_core/pipeline-template/nextflow_schema.json index 4e64f555de..598f8a84b8 100644 --- a/nf_core/pipeline-template/nextflow_schema.json +++ b/nf_core/pipeline-template/nextflow_schema.json @@ -12,7 +12,7 @@ "description": "Define where the pipeline should find input data and save output data.", "required": [ "input", - "output" + "outdir" ], "properties": { "input": { From 610ccce666ebd2b86494f99f234b33e7a0e1d36e Mon Sep 17 00:00:00 2001 From: Harshil Patel Date: Tue, 8 Mar 2022 09:34:52 +0000 Subject: [PATCH 4/8] Update create-test-wf.yml --- .github/workflows/create-test-wf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 9b13cadd39..90013bc439 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -47,7 +47,7 @@ jobs: - name: Run nf-core/tools run: | nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" - nextflow run nf-core-testpipeline -profile test,docker + nextflow run nf-core-testpipeline -profile test,docker --outdir ./results - name: Upload log file artifact if: ${{ always() }} From 20dcad12408641b15c49c7868170021efaaf50c8 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 8 Mar 2022 11:37:01 +0100 Subject: [PATCH 5/8] Add pipeline and revision info to awstest.yml --- nf_core/pipeline-template/.github/workflows/awstest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nf_core/pipeline-template/.github/workflows/awstest.yml b/nf_core/pipeline-template/.github/workflows/awstest.yml index e980264d9d..b5ceb32297 100644 --- a/nf_core/pipeline-template/.github/workflows/awstest.yml +++ b/nf_core/pipeline-template/.github/workflows/awstest.yml @@ -17,6 +17,8 @@ jobs: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_COMPUTE_ENV }} + pipeline: ${{ github.repository }} + revision: ${{ github.sha }} workdir: s3://${{ secrets.AWS_S3_BUCKET }}{% endraw %}/work/{{ short_name }}/{% raw %}work-${{ github.sha }}{% endraw %} parameters: | { From 4c63fb4406acb16ce010e2f35c0d1d536c65f111 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Tue, 8 Mar 2022 11:39:39 +0100 Subject: [PATCH 6/8] Revert cmd to a single line as discuss in PR comment --- nf_core/pipeline-template/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/README.md b/nf_core/pipeline-template/README.md index 6dc1b11723..e0049a72e9 100644 --- a/nf_core/pipeline-template/README.md +++ b/nf_core/pipeline-template/README.md @@ -55,11 +55,7 @@ On release, automated continuous integration tests run the pipeline on a full-si ```console - nextflow run {{ name }} \ - --input samplesheet.csv \ - --outdir \ - --genome GRCh37 \ - -profile + nextflow run {{ name }} --input samplesheet.csv --outdir --genome GRCh37 -profile ``` ## Documentation From cdab166c7cd3e7fe41ed02b71f62c1ec2bdca3cf Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Wed, 9 Mar 2022 16:52:26 +0100 Subject: [PATCH 7/8] Check for outdir, otherwise if not provided is set to null --- nf_core/pipeline-template/workflows/pipeline.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 6ac463f8e8..b6967ae1c3 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -16,6 +16,7 @@ for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true // Check mandatory parameters if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } +if (!params.outdir) { exit 1, 'Parameter outdir not specified!' } /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From ae991eab9f217e4c0fba0b69e9e92fe09167a5d6 Mon Sep 17 00:00:00 2001 From: JoseEspinosa Date: Wed, 9 Mar 2022 19:27:50 +0100 Subject: [PATCH 8/8] Undo previous change --- nf_core/pipeline-template/workflows/pipeline.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index b6967ae1c3..6ac463f8e8 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -16,7 +16,6 @@ for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true // Check mandatory parameters if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } -if (!params.outdir) { exit 1, 'Parameter outdir not specified!' } /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~