From 1c927e9481702c9c7d48915592a83a5df04cf1bb Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 2 Aug 2021 16:16:10 +0200 Subject: [PATCH 01/44] Change filter to support multiple version file outputs --- nf_core/pipeline-template/workflows/pipeline.nf | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index fe1882b420..c27d4b4df3 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -92,15 +92,12 @@ workflow {{ short_name|upper }} { // MODULE: Pipeline reporting // ch_software_versions - .map { it -> if (it) [ it.baseName, it ] } - .groupTuple() - .map { it[1][0] } .flatten() - .collect() + .unique { it.getName() + it.getText() } .set { ch_software_versions } GET_SOFTWARE_VERSIONS ( - ch_software_versions.map { it }.collect() + ch_software_versions.collect() ) // From e147cc931d004ef7dbd4870bfe63464255e4a952 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 2 Aug 2021 22:18:29 +0200 Subject: [PATCH 02/44] Add collectFile to combine different versions --- nf_core/pipeline-template/workflows/pipeline.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index c27d4b4df3..880886fd64 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -94,6 +94,7 @@ workflow {{ short_name|upper }} { ch_software_versions .flatten() .unique { it.getName() + it.getText() } + .collectFile(sort:true) { it -> [ it.getName(), it.getText()] } .set { ch_software_versions } GET_SOFTWARE_VERSIONS ( From 2942f8384d9420ad5a6aa803aaf6968ccdc5f2b9 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 2 Aug 2021 22:27:29 +0200 Subject: [PATCH 03/44] Change scrape versions to comma separate multiple versions --- nf_core/pipeline-template/bin/scrape_software_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/bin/scrape_software_versions.py b/nf_core/pipeline-template/bin/scrape_software_versions.py index 241dc8b7a6..057c99ffd3 100755 --- a/nf_core/pipeline-template/bin/scrape_software_versions.py +++ b/nf_core/pipeline-template/bin/scrape_software_versions.py @@ -11,7 +11,7 @@ software = "{{ name }}" with open(version_file) as fin: - version = fin.read().strip() + version = fin.read().strip().replace('\n',', ') results[software] = version # Dump to YAML From 826661037e95706df18280a2d883bc33ac12c7b6 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 2 Aug 2021 22:37:15 +0200 Subject: [PATCH 04/44] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe87bf81dc..dc696843c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +* Modify software version channel handling to support multiple software version emissions (e.g. from mulled containers), and multiple software versions. + ### General * Changed `questionary` `ask()` to `unsafe_ask()` to not catch `KeyboardInterupts` ([#1237](https://github.com/nf-core/tools/issues/1237)) From deaf0c12c92215c0051831adac39b4dbbcc8944e Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Mon, 2 Aug 2021 22:47:57 +0200 Subject: [PATCH 05/44] black lint update --- nf_core/pipeline-template/bin/scrape_software_versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/bin/scrape_software_versions.py b/nf_core/pipeline-template/bin/scrape_software_versions.py index 057c99ffd3..a554188966 100755 --- a/nf_core/pipeline-template/bin/scrape_software_versions.py +++ b/nf_core/pipeline-template/bin/scrape_software_versions.py @@ -11,7 +11,7 @@ software = "{{ name }}" with open(version_file) as fin: - version = fin.read().strip().replace('\n',', ') + version = fin.read().strip().replace("\n", ", ") results[software] = version # Dump to YAML From e3c5fe0873612527af76b56e831daf0968a5fdc5 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 3 Aug 2021 15:07:41 +0200 Subject: [PATCH 06/44] Software versions as YAML output --- nf_core/module-template/modules/main.nf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 6e4dcde636..c568852139 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -78,6 +78,9 @@ process {{ tool_name_underscore|upper }} { {%- endif %} $bam - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${task.process}: + - samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) + END_VERSIONS """ } From 7a894d668b4429829fea4acd6f8acf1fe50994cd Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 3 Aug 2021 15:23:38 +0200 Subject: [PATCH 07/44] Add comment to add version for each tool --- nf_core/module-template/modules/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index c568852139..741f8b04d0 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -62,6 +62,7 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/software/homer/annotatepeaks/main.nf + // Each tool MUST provide the tool name and version number in the YAML version file (versions.yml) // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "$options.args" variable // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter // using the Nextflow "task" variable e.g. "--threads $task.cpus" From d9267e901bc06ce3d46c3b55780a420dcbea0d89 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 3 Aug 2021 15:23:57 +0200 Subject: [PATCH 08/44] Update output --- nf_core/module-template/modules/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 741f8b04d0..ee3293f445 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -52,7 +52,7 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Named file extensions MUST be emitted for ALL output channels {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam // TODO nf-core: List additional required output channels/values here - path "*.version.txt" , emit: version + path "versions.yml" , emit: version script: def software = getSoftwareName(task.process) From 71d20e1c2ea7e1743931c1c26891ce5595087697 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 3 Aug 2021 15:27:19 +0200 Subject: [PATCH 09/44] Collect YAML files together --- nf_core/pipeline-template/workflows/pipeline.nf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 880886fd64..bbd9ae62a4 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -91,14 +91,14 @@ workflow {{ short_name|upper }} { // // MODULE: Pipeline reporting // - ch_software_versions - .flatten() - .unique { it.getName() + it.getText() } - .collectFile(sort:true) { it -> [ it.getName(), it.getText()] } - .set { ch_software_versions } + // ch_software_versions + // .flatten() + // .unique { it.getName() + it.getText() } + // .collectFile(sort:true) { it -> [ it.getName(), it.getText()] } + // .set { ch_software_versions } GET_SOFTWARE_VERSIONS ( - ch_software_versions.collect() + ch_software_versions.collectFile() ) // From 221e49252509f5aee2e77754be59dd0ff9da62bd Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 3 Aug 2021 15:59:35 +0200 Subject: [PATCH 10/44] Add workflow data to software versions --- .../modules/local/get_software_versions.nf | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index 8af8af1735..b902834c50 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -26,8 +26,13 @@ process GET_SOFTWARE_VERSIONS { script: // This script is bundled with the pipeline, in {{ name }}/bin/ """ - echo $workflow.manifest.version > pipeline.version.txt - echo $workflow.nextflow.version > nextflow.version.txt - scrape_software_versions.py &> software_versions_mqc.yaml + # echo $workflow.manifest.version > pipeline.version.txt + # echo $workflow.nextflow.version > nextflow.version.txt + # scrape_software_versions.py &> software_versions_mqc.yaml + cat - $versions <<-END_WORKFLOW_VERSION > software_versions_mqc.yaml + Workflow: + - Nextflow: $workflow.nextflow.version + - $workflow.manifest.name: $workflow.manifest.version + END_WORKFLOW_VERSION """ } From 89d291863412e3dac601d8781c8175ef0de0e7f3 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 3 Aug 2021 20:17:45 +0200 Subject: [PATCH 11/44] Replace software versions tsv with yml --- .../modules/local/get_software_versions.nf | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index b902834c50..fc7fcbca37 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -21,18 +21,23 @@ process GET_SOFTWARE_VERSIONS { path versions output: - path "software_versions.tsv" , emit: tsv - path 'software_versions_mqc.yaml', emit: yaml + path "software_versions.yml" , emit: yml + path 'software_versions_mqc.yaml', emit: mqc_yaml script: // This script is bundled with the pipeline, in {{ name }}/bin/ """ - # echo $workflow.manifest.version > pipeline.version.txt - # echo $workflow.nextflow.version > nextflow.version.txt - # scrape_software_versions.py &> software_versions_mqc.yaml - cat - $versions <<-END_WORKFLOW_VERSION > software_versions_mqc.yaml + cat - $versions <<-END_WORKFLOW_VERSION > software_versions.yml Workflow: - Nextflow: $workflow.nextflow.version - $workflow.manifest.name: $workflow.manifest.version END_WORKFLOW_VERSION + cat - <( sed 's/^/ /' software_versions.yml ) <<-END_MQC_YAML > software_versions_mqc.yaml + id: 'software_versions' + section_name: '{{ name }} Software Versions' + section_href: 'https://github.com/{{ name }}' + plot_type: 'table' + description: 'are collected at run time from the software output.' + data: + END_MQC_YAML """ } From 3945ee8901217e49ea9c64fe5ab9c708b8fcf973 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 3 Aug 2021 20:21:34 +0200 Subject: [PATCH 12/44] Update GET_SOFTWARE_VERSION channel names --- nf_core/pipeline-template/workflows/pipeline.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index bbd9ae62a4..9fb4d0d0f5 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -38,7 +38,7 @@ def modules = params.modules.clone() // // MODULE: Local to the pipeline // -include { GET_SOFTWARE_VERSIONS } from '../modules/local/get_software_versions' addParams( options: [publish_files : ['tsv':'']] ) +include { GET_SOFTWARE_VERSIONS } from '../modules/local/get_software_versions' addParams( options: [publish_files : ['yml':'']] ) // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules @@ -111,7 +111,7 @@ workflow {{ short_name|upper }} { ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(GET_SOFTWARE_VERSIONS.out.yaml.collect()) + ch_multiqc_files = ch_multiqc_files.mix(GET_SOFTWARE_VERSIONS.out.mqc_yaml.collect()) ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) MULTIQC ( From e6bc9ff61e2ca735cce2fe20003867b3715ab1b4 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 4 Aug 2021 10:34:36 +0200 Subject: [PATCH 13/44] Update nf_core/pipeline-template/modules/local/get_software_versions.nf Co-authored-by: Gregor Sturm --- nf_core/pipeline-template/modules/local/get_software_versions.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index fc7fcbca37..943adcf5cf 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -31,6 +31,7 @@ process GET_SOFTWARE_VERSIONS { - Nextflow: $workflow.nextflow.version - $workflow.manifest.name: $workflow.manifest.version END_WORKFLOW_VERSION + cat - <( sed 's/^/ /' software_versions.yml ) <<-END_MQC_YAML > software_versions_mqc.yaml id: 'software_versions' section_name: '{{ name }} Software Versions' From 85839d32de84461119b6174c98a20e19b23ca554 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 4 Aug 2021 10:42:47 +0200 Subject: [PATCH 14/44] Move sed for readability --- .../pipeline-template/modules/local/get_software_versions.nf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index 943adcf5cf..0059db317a 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -31,14 +31,15 @@ process GET_SOFTWARE_VERSIONS { - Nextflow: $workflow.nextflow.version - $workflow.manifest.name: $workflow.manifest.version END_WORKFLOW_VERSION - - cat - <( sed 's/^/ /' software_versions.yml ) <<-END_MQC_YAML > software_versions_mqc.yaml + + cat - <<-END_MQC_YAML > software_versions_mqc.yaml id: 'software_versions' section_name: '{{ name }} Software Versions' section_href: 'https://github.com/{{ name }}' plot_type: 'table' description: 'are collected at run time from the software output.' data: + \$( sed 's/^/ /' software_versions.yml ) END_MQC_YAML """ } From e99da1e4377bf7b4abe1c2c0525d5333cde93af1 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Wed, 4 Aug 2021 10:55:53 +0200 Subject: [PATCH 15/44] Update nf_core/module-template/modules/main.nf --- nf_core/module-template/modules/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index ee3293f445..9dfb70d706 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -80,7 +80,7 @@ process {{ tool_name_underscore|upper }} { $bam cat <<-END_VERSIONS > versions.yml - ${task.process}: + ${task.process.tokenize(':')[-1]}: - samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) END_VERSIONS """ From 5ce54a41011f752066b4f6349928fb61c78e32b5 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 10 Aug 2021 09:16:45 +0200 Subject: [PATCH 16/44] Remove Dash from YAML --- nf_core/module-template/modules/main.nf | 2 +- .../modules/local/get_software_versions.nf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 9dfb70d706..97d4eb7ccd 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -81,7 +81,7 @@ process {{ tool_name_underscore|upper }} { cat <<-END_VERSIONS > versions.yml ${task.process.tokenize(':')[-1]}: - - samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) + samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) END_VERSIONS """ } diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index 0059db317a..acfec1f288 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -28,8 +28,8 @@ process GET_SOFTWARE_VERSIONS { """ cat - $versions <<-END_WORKFLOW_VERSION > software_versions.yml Workflow: - - Nextflow: $workflow.nextflow.version - - $workflow.manifest.name: $workflow.manifest.version + Nextflow: $workflow.nextflow.version + $workflow.manifest.name: $workflow.manifest.version END_WORKFLOW_VERSION cat - <<-END_MQC_YAML > software_versions_mqc.yaml @@ -39,7 +39,7 @@ process GET_SOFTWARE_VERSIONS { plot_type: 'table' description: 'are collected at run time from the software output.' data: - \$( sed 's/^/ /' software_versions.yml ) + \$( sed 's/^/ /' software_versions.yml ) END_MQC_YAML """ } From fd93ebbc7000a7e54ddac09ca240a5a5e70c3e41 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 10:25:40 +0200 Subject: [PATCH 17/44] Port changes from prototype https://github.com/nf-core/rnaseq/pull/689 Co-Authored-By: Gregor Sturm --- nf_core/module-template/modules/functions.nf | 7 ++ nf_core/module-template/modules/main.nf | 8 +- .../bin/scrape_software_versions.py | 36 -------- .../modules/local/functions.nf | 7 ++ .../modules/local/get_software_versions.nf | 88 +++++++++++++++---- .../nf-core/modules/fastqc/functions.nf | 7 ++ .../modules/nf-core/modules/fastqc/main.nf | 14 ++- .../nf-core/modules/multiqc/functions.nf | 7 ++ .../modules/nf-core/modules/multiqc/main.nf | 8 +- .../pipeline-template/workflows/pipeline.nf | 9 -- 10 files changed, 118 insertions(+), 73 deletions(-) delete mode 100755 nf_core/pipeline-template/bin/scrape_software_versions.py diff --git a/nf_core/module-template/modules/functions.nf b/nf_core/module-template/modules/functions.nf index da9da093d3..4860a36278 100644 --- a/nf_core/module-template/modules/functions.nf +++ b/nf_core/module-template/modules/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 97d4eb7ccd..164329e965 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/software @@ -62,7 +62,7 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/software/homer/annotatepeaks/main.nf - // Each tool MUST provide the tool name and version number in the YAML version file (versions.yml) + // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "$options.args" variable // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter // using the Nextflow "task" variable e.g. "--threads $task.cpus" @@ -80,8 +80,8 @@ process {{ tool_name_underscore|upper }} { $bam cat <<-END_VERSIONS > versions.yml - ${task.process.tokenize(':')[-1]}: - samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) + ${getProcessName(task.process)}: + $software: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) END_VERSIONS """ } diff --git a/nf_core/pipeline-template/bin/scrape_software_versions.py b/nf_core/pipeline-template/bin/scrape_software_versions.py deleted file mode 100755 index a554188966..0000000000 --- a/nf_core/pipeline-template/bin/scrape_software_versions.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -import os - -results = {} -version_files = [x for x in os.listdir(".") if x.endswith(".version.txt")] -for version_file in version_files: - - software = version_file.replace(".version.txt", "") - if software == "pipeline": - software = "{{ name }}" - - with open(version_file) as fin: - version = fin.read().strip().replace("\n", ", ") - results[software] = version - -# Dump to YAML -print( - """ -id: 'software_versions' -section_name: '{{ name }} Software Versions' -section_href: 'https://github.com/{{ name }}' -plot_type: 'html' -description: 'are collected at run time from the software output.' -data: | -
-""" -) -for k, v in sorted(results.items()): - print("
{}
{}
".format(k, v)) -print("
") - -# Write out as tsv file: -with open("software_versions.tsv", "w") as f: - for k, v in sorted(results.items()): - f.write("{}\t{}\n".format(k, v)) diff --git a/nf_core/pipeline-template/modules/local/functions.nf b/nf_core/pipeline-template/modules/local/functions.nf index da9da093d3..4860a36278 100644 --- a/nf_core/pipeline-template/modules/local/functions.nf +++ b/nf_core/pipeline-template/modules/local/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index acfec1f288..0120a80baf 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -8,11 +8,12 @@ process GET_SOFTWARE_VERSIONS { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) } - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) + // This module requires only the PyYAML library, but rather than create a new container on biocontainers, we reuse the multiqc container. + conda (params.enable_conda ? "bioconda::multiqc=1.10.1" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/python:3.8.3" + container "https://depot.galaxyproject.org/singularity/multiqc:1.10.1--pyhdfd78af_1" } else { - container "quay.io/biocontainers/python:3.8.3" + container "quay.io/biocontainers/multiqc:1.10.1--pyhdfd78af_1" } cache false @@ -24,22 +25,71 @@ process GET_SOFTWARE_VERSIONS { path "software_versions.yml" , emit: yml path 'software_versions_mqc.yaml', emit: mqc_yaml - script: // This script is bundled with the pipeline, in {{ name }}/bin/ + script: """ - cat - $versions <<-END_WORKFLOW_VERSION > software_versions.yml - Workflow: - Nextflow: $workflow.nextflow.version - $workflow.manifest.name: $workflow.manifest.version - END_WORKFLOW_VERSION - - cat - <<-END_MQC_YAML > software_versions_mqc.yaml - id: 'software_versions' - section_name: '{{ name }} Software Versions' - section_href: 'https://github.com/{{ name }}' - plot_type: 'table' - description: 'are collected at run time from the software output.' - data: - \$( sed 's/^/ /' software_versions.yml ) - END_MQC_YAML + #!/usr/bin/env python + + import yaml + from textwrap import dedent + + def _make_versions_html(versions): + html = [ + dedent( + '''\\ + + + + + + + + + + ''' + ) + ] + for process, tmp_versions in sorted(versions.items()): + html.append("") + for i, (tool, version) in enumerate(sorted(tmp_versions.items())): + html.append( + dedent( + f'''\\ + + + + + + ''' + ) + ) + html.append("") + html.append("
Process Name Software Version
{process if (i == 0) else ''}{tool}{version}
") + return "\\n".join(html) + + with open("$versions") as f: + versions = yaml.safe_load(f) + + versions["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version" + } + + versions_mqc = { + 'id': 'software_versions', + 'section_name': '${workflow.manifest.name} Software Versions', + 'section_href': 'https://github.com/${workflow.manifest.name}', + 'plot_type': 'html', + 'description': 'are collected at run time from the software output.', + 'data': _make_versions_html(versions) + } + + with open("software_versions.yml", 'w') as f: + yaml.dump(versions, f, default_flow_style=False) + with open("software_versions_mqc.yml", 'w') as f: + yaml.dump(versions_mqc, f, default_flow_style=False) """ } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf index da9da093d3..4860a36278 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 39c327b261..f90aae1a55 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -34,14 +34,22 @@ process FASTQC { """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz - fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + $software: \$(fastqc --version | sed -e "s/FastQC v//g") + END_VERSIONS """ } else { """ [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + $software: \$(fastqc --version | sed -e "s/FastQC v//g") + END_VERSIONS """ } } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf index da9da093d3..4860a36278 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index da78080024..17a3097671 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) @@ -30,6 +30,10 @@ process MULTIQC { def software = getSoftwareName(task.process) """ multiqc -f $options.args . - multiqc --version | sed -e "s/multiqc, version //g" > ${software}.version.txt + + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + $software: \$(multiqc --version | sed -e "s/multiqc, version //g") + END_VERSIONS """ } diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 9fb4d0d0f5..f99538d4c8 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -88,15 +88,6 @@ workflow {{ short_name|upper }} { ) ch_software_versions = ch_software_versions.mix(FASTQC.out.version.first().ifEmpty(null)) - // - // MODULE: Pipeline reporting - // - // ch_software_versions - // .flatten() - // .unique { it.getName() + it.getText() } - // .collectFile(sort:true) { it -> [ it.getName(), it.getText()] } - // .set { ch_software_versions } - GET_SOFTWARE_VERSIONS ( ch_software_versions.collectFile() ) From 3a10d60617a3958da9dd36a594b6a0387cf833e5 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 10:27:16 +0200 Subject: [PATCH 18/44] Update comment --- .../pipeline-template/modules/local/get_software_versions.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index 0120a80baf..ede47ed68d 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -8,7 +8,7 @@ process GET_SOFTWARE_VERSIONS { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) } - // This module requires only the PyYAML library, but rather than create a new container on biocontainers, we reuse the multiqc container. + // This module only requires the PyYAML library, but rather than create a new container on biocontainers we reuse the multiqc container. conda (params.enable_conda ? "bioconda::multiqc=1.10.1" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { container "https://depot.galaxyproject.org/singularity/multiqc:1.10.1--pyhdfd78af_1" From 1a7993d28a2cb14449aeadf10436085199c4f79f Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 10:30:27 +0200 Subject: [PATCH 19/44] Update process output --- .../pipeline-template/modules/nf-core/modules/fastqc/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/multiqc/main.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index f90aae1a55..97183b62c9 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -24,7 +24,7 @@ process FASTQC { output: tuple val(meta), path("*.html"), emit: html tuple val(meta), path("*.zip") , emit: zip - path "*.version.txt" , emit: version + path "versions.yml" , emit: version script: // Add soft-links to original FastQs for consistent naming in pipeline diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index 17a3097671..77c8f4361a 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -24,7 +24,7 @@ process MULTIQC { path "*multiqc_report.html", emit: report path "*_data" , emit: data path "*_plots" , optional:true, emit: plots - path "*.version.txt" , emit: version + path "versions.yml" , emit: version script: def software = getSoftwareName(task.process) From d18174d69a52aa5b56037a9d33e8a5e2ff314869 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 10:40:59 +0200 Subject: [PATCH 20/44] Add getProcessName to modules functions_nf lint --- nf_core/modules/lint/functions_nf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/modules/lint/functions_nf.py b/nf_core/modules/lint/functions_nf.py index 600a1ae7fd..aef0d115ea 100644 --- a/nf_core/modules/lint/functions_nf.py +++ b/nf_core/modules/lint/functions_nf.py @@ -22,7 +22,7 @@ def functions_nf(module_lint_object, module): return # Test whether all required functions are present - required_functions = ["getSoftwareName", "initOptions", "getPathFromList", "saveFiles"] + required_functions = ["getSoftwareName", "getProcessName", "initOptions", "getPathFromList", "saveFiles"] lines = "\n".join(lines) contains_all_functions = True for f in required_functions: From edae68b5b147712462ea4826523c86b2451178cc Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 10:44:50 +0200 Subject: [PATCH 21/44] Update file_exists lint to remove scrape_versions.py --- nf_core/lint/files_exist.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 7b97aa3fb5..e1b28bc9a3 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -36,7 +36,6 @@ def files_exist(self): assets/email_template.txt assets/nf-core-PIPELINE_logo.png assets/sendmail_template.txt - bin/scrape_software_versions.py conf/modules.config conf/test.config conf/test_full.config @@ -121,7 +120,6 @@ def files_exist(self): [os.path.join("assets", "email_template.txt")], [os.path.join("assets", "sendmail_template.txt")], [os.path.join("assets", f"nf-core-{short_name}_logo.png")], - [os.path.join("bin", "scrape_software_versions.py")], [os.path.join("conf", "modules.config")], [os.path.join("conf", "test.config")], [os.path.join("conf", "test_full.config")], From 8ae12e3d6a2e7893ea6b137697023245764904a8 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 10:47:34 +0200 Subject: [PATCH 22/44] Remove scrape software version from files unchanged. --- nf_core/lint/files_unchanged.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 37728b8b06..262e8c4449 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -92,7 +92,6 @@ def files_unchanged(self): [os.path.join("assets", "email_template.txt")], [os.path.join("assets", "sendmail_template.txt")], [os.path.join("assets", f"nf-core-{short_name}_logo.png")], - [os.path.join("bin", "scrape_software_versions.py")], [os.path.join("docs", "images", f"nf-core-{short_name}_logo.png")], [os.path.join("docs", "README.md")], [os.path.join("lib", "nfcore_external_java_deps.jar")], From 074f1c496c6b2427df25b5d0fddd74b12ccbe1ca Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 10:50:25 +0200 Subject: [PATCH 23/44] Fix YAML output --- .../pipeline-template/modules/local/get_software_versions.nf | 2 +- nf_core/pipeline-template/workflows/pipeline.nf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index ede47ed68d..08d58f9c52 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -23,7 +23,7 @@ process GET_SOFTWARE_VERSIONS { output: path "software_versions.yml" , emit: yml - path 'software_versions_mqc.yaml', emit: mqc_yaml + path "software_versions_mqc.yml" , emit: mqc_yml script: """ diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index f99538d4c8..44924b8116 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -102,7 +102,7 @@ workflow {{ short_name|upper }} { ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(GET_SOFTWARE_VERSIONS.out.mqc_yaml.collect()) + ch_multiqc_files = ch_multiqc_files.mix(GET_SOFTWARE_VERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) MULTIQC ( From c21dabdd26ee8a46d32bcb78387916216e45a2f3 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 11:33:55 +0200 Subject: [PATCH 24/44] Update nf_core/module-template/modules/main.nf Co-authored-by: Gregor Sturm --- nf_core/module-template/modules/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 164329e965..aa2a96021a 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -81,7 +81,7 @@ process {{ tool_name_underscore|upper }} { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - $software: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) + samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) END_VERSIONS """ } From 70d14aaf950c0e825310e1229017b01ccacc6662 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 11:39:43 +0200 Subject: [PATCH 25/44] Remove getSoftwareName function --- nf_core/module-template/modules/functions.nf | 7 ------- nf_core/module-template/modules/main.nf | 3 +-- nf_core/modules/lint/functions_nf.py | 2 +- nf_core/pipeline-template/modules/local/functions.nf | 7 ------- .../modules/nf-core/modules/fastqc/functions.nf | 7 ------- .../modules/nf-core/modules/fastqc/main.nf | 2 +- .../modules/nf-core/modules/multiqc/functions.nf | 7 ------- .../modules/nf-core/modules/multiqc/main.nf | 2 +- 8 files changed, 4 insertions(+), 33 deletions(-) diff --git a/nf_core/module-template/modules/functions.nf b/nf_core/module-template/modules/functions.nf index 4860a36278..0dfaa3a1ca 100644 --- a/nf_core/module-template/modules/functions.nf +++ b/nf_core/module-template/modules/functions.nf @@ -2,13 +2,6 @@ // Utility functions used in nf-core DSL2 module files // -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - // // Extract name of module from process name using $task.process // diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index aa2a96021a..b72659f824 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' +include { initOptions; saveFiles; getProcessName } from './functions' // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/software @@ -55,7 +55,6 @@ process {{ tool_name_underscore|upper }} { path "versions.yml" , emit: version script: - def software = getSoftwareName(task.process) {% if has_meta -%} def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" {%- endif %} diff --git a/nf_core/modules/lint/functions_nf.py b/nf_core/modules/lint/functions_nf.py index aef0d115ea..f03f0c39a3 100644 --- a/nf_core/modules/lint/functions_nf.py +++ b/nf_core/modules/lint/functions_nf.py @@ -22,7 +22,7 @@ def functions_nf(module_lint_object, module): return # Test whether all required functions are present - required_functions = ["getSoftwareName", "getProcessName", "initOptions", "getPathFromList", "saveFiles"] + required_functions = ["getProcessName", "initOptions", "getPathFromList", "saveFiles"] lines = "\n".join(lines) contains_all_functions = True for f in required_functions: diff --git a/nf_core/pipeline-template/modules/local/functions.nf b/nf_core/pipeline-template/modules/local/functions.nf index 4860a36278..0dfaa3a1ca 100644 --- a/nf_core/pipeline-template/modules/local/functions.nf +++ b/nf_core/pipeline-template/modules/local/functions.nf @@ -2,13 +2,6 @@ // Utility functions used in nf-core DSL2 module files // -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - // // Extract name of module from process name using $task.process // diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf index 4860a36278..0dfaa3a1ca 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf @@ -2,13 +2,6 @@ // Utility functions used in nf-core DSL2 module files // -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - // // Extract name of module from process name using $task.process // diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 97183b62c9..8becdcb48d 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' +include { initOptions; saveFiles; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf index 4860a36278..0dfaa3a1ca 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf @@ -2,13 +2,6 @@ // Utility functions used in nf-core DSL2 module files // -// -// Extract name of software tool from process name using $task.process -// -def getSoftwareName(task_process) { - return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() -} - // // Extract name of module from process name using $task.process // diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index 77c8f4361a..a9ba0fe3a2 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' +include { initOptions; saveFiles; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) From 1ee14e9a98608a87391d6b0739950fc85efcff4b Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 11:44:03 +0200 Subject: [PATCH 26/44] remove software var --- .../pipeline-template/modules/nf-core/modules/fastqc/main.nf | 5 ++--- .../modules/nf-core/modules/multiqc/main.nf | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 8becdcb48d..26eb12cdad 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -28,7 +28,6 @@ process FASTQC { script: // Add soft-links to original FastQs for consistent naming in pipeline - def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" if (meta.single_end) { """ @@ -37,7 +36,7 @@ process FASTQC { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - $software: \$(fastqc --version | sed -e "s/FastQC v//g") + fastqc: \$(fastqc --version | sed -e "s/FastQC v//g") END_VERSIONS """ } else { @@ -48,7 +47,7 @@ process FASTQC { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - $software: \$(fastqc --version | sed -e "s/FastQC v//g") + fastqc: \$(fastqc --version | sed -e "s/FastQC v//g") END_VERSIONS """ } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index a9ba0fe3a2..65b1e64bb7 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -27,13 +27,12 @@ process MULTIQC { path "versions.yml" , emit: version script: - def software = getSoftwareName(task.process) """ multiqc -f $options.args . cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - $software: \$(multiqc --version | sed -e "s/multiqc, version //g") + multiqc: \$(multiqc --version | sed -e "s/multiqc, version //g") END_VERSIONS """ } From a45c66d3b3a9dbdbbab26d07d904bb9e1e55c5e1 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 11:52:12 +0200 Subject: [PATCH 27/44] Rename publish_dir directory --- nf_core/module-template/modules/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/fastqc/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/multiqc/main.nf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index b72659f824..21b079ab3e 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -26,7 +26,7 @@ process {{ tool_name_underscore|upper }} { label '{{ process_label }}' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:{{ 'meta' if has_meta else "[:]" }}, publish_by_meta:{{ "['id']" if has_meta else "[]" }}) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:{{ 'meta' if has_meta else "[:]" }}, publish_by_meta:{{ "['id']" if has_meta else "[]" }}) } // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 26eb12cdad..a725ecce8d 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -9,7 +9,7 @@ process FASTQC { label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:meta, publish_by_meta:['id']) } conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index 65b1e64bb7..c97ed17c10 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -8,7 +8,7 @@ process MULTIQC { label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:[:], publish_by_meta:[]) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:[:], publish_by_meta:[]) } conda (params.enable_conda ? "bioconda::multiqc=1.10.1" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { From 6682211194bb8e923f3a7a1cd603c0c2af997ae3 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 13:13:24 +0200 Subject: [PATCH 28/44] Update nf_core/module-template/modules/main.nf Co-authored-by: Gregor Sturm --- nf_core/module-template/modules/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 21b079ab3e..16d35aa675 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -80,7 +80,7 @@ process {{ tool_name_underscore|upper }} { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$// ) + samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$//' ) END_VERSIONS """ } From 30bbc8aef6ba9f5791d4a8d089af6a3502e9b2d9 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 13:31:24 +0200 Subject: [PATCH 29/44] Update version.txt reference to versions.yml --- nf_core/module-template/modules/functions.nf | 2 +- nf_core/module-template/modules/meta.yml | 2 +- nf_core/pipeline-template/modules/local/functions.nf | 2 +- .../modules/nf-core/modules/fastqc/functions.nf | 2 +- .../pipeline-template/modules/nf-core/modules/fastqc/meta.yml | 2 +- .../modules/nf-core/modules/multiqc/functions.nf | 2 +- .../pipeline-template/modules/nf-core/modules/multiqc/meta.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nf_core/module-template/modules/functions.nf b/nf_core/module-template/modules/functions.nf index 0dfaa3a1ca..8171d73fa4 100644 --- a/nf_core/module-template/modules/functions.nf +++ b/nf_core/module-template/modules/functions.nf @@ -37,7 +37,7 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { + if (!args.filename.equals('versions.yml')) { def ioptions = initOptions(args.options) def path_list = [ ioptions.publish_dir ?: args.publish_dir ] if (ioptions.publish_by_meta) { diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index be6d3e5f93..c42dd613bd 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -40,7 +40,7 @@ output: - version: type: file description: File containing software version - pattern: "*.{version.txt}" + pattern: "versions.yml" ## TODO nf-core: Delete / customise this example output - bam: type: file diff --git a/nf_core/pipeline-template/modules/local/functions.nf b/nf_core/pipeline-template/modules/local/functions.nf index 0dfaa3a1ca..8171d73fa4 100644 --- a/nf_core/pipeline-template/modules/local/functions.nf +++ b/nf_core/pipeline-template/modules/local/functions.nf @@ -37,7 +37,7 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { + if (!args.filename.equals('versions.yml')) { def ioptions = initOptions(args.options) def path_list = [ ioptions.publish_dir ?: args.publish_dir ] if (ioptions.publish_by_meta) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf index 0dfaa3a1ca..8171d73fa4 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf @@ -37,7 +37,7 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { + if (!args.filename.equals('versions.yml')) { def ioptions = initOptions(args.options) def path_list = [ ioptions.publish_dir ?: args.publish_dir ] if (ioptions.publish_by_meta) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml index 8eb9953dce..48031356b5 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml @@ -43,7 +43,7 @@ output: - version: type: file description: File containing software version - pattern: "*.{version.txt}" + pattern: "versions.yml" authors: - "@drpatelh" - "@grst" diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf index 0dfaa3a1ca..8171d73fa4 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf @@ -37,7 +37,7 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { + if (!args.filename.equals('versions.yml')) { def ioptions = initOptions(args.options) def path_list = [ ioptions.publish_dir ?: args.publish_dir ] if (ioptions.publish_by_meta) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml index 532a8bb1ef..2d99ec0d12 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml @@ -32,7 +32,7 @@ output: - version: type: file description: File containing software version - pattern: "*.{version.txt}" + pattern: "versions.yml" authors: - "@abhi18av" - "@bunop" From a98588b0a25eb98fb76cfdccfedf25699873963b Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 13:34:13 +0200 Subject: [PATCH 30/44] Update lint check for software --- nf_core/modules/lint/main_nf.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 018dc99af2..ac29e9ff26 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -116,16 +116,10 @@ def main_nf(module_lint_object, module): def check_script_section(self, lines): """ Lint the script section - Checks whether 'def sotware' and 'def prefix' are defined + Checks whether 'def prefix' is defined """ script = "".join(lines) - # check for software - if re.search("\s*def\s*software\s*=\s*getSoftwareName", script): - self.passed.append(("main_nf_version_script", "Software version specified in script section", self.main_nf)) - else: - self.warned.append(("main_nf_version_script", "Software version unspecified in script section", self.main_nf)) - # check for prefix (only if module has a meta map as input) if self.has_meta: if re.search("\s*prefix\s*=\s*options.suffix", script): From c46e2194e91f1878b3a714550eec6bf8b7d5c40e Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Thu, 12 Aug 2021 14:28:25 +0200 Subject: [PATCH 31/44] Make publish_dirs lowercase --- nf_core/module-template/modules/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/fastqc/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/multiqc/main.nf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 16d35aa675..4047c0bf04 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -26,7 +26,7 @@ process {{ tool_name_underscore|upper }} { label '{{ process_label }}' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:{{ 'meta' if has_meta else "[:]" }}, publish_by_meta:{{ "['id']" if has_meta else "[]" }}) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process).toLowerCase(), meta:{{ 'meta' if has_meta else "[:]" }}, publish_by_meta:{{ "['id']" if has_meta else "[]" }}) } // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index a725ecce8d..35ed481c9c 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -9,7 +9,7 @@ process FASTQC { label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:meta, publish_by_meta:['id']) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process).toLowerCase(), meta:meta, publish_by_meta:['id']) } conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index c97ed17c10..2fc36ad1f3 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -8,7 +8,7 @@ process MULTIQC { label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:[:], publish_by_meta:[]) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process).toLowerCase(), meta:[:], publish_by_meta:[]) } conda (params.enable_conda ? "bioconda::multiqc=1.10.1" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { From 77a89e2b6ac18952c2fbf96a9601292587a1160e Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 Aug 2021 09:17:06 +0200 Subject: [PATCH 32/44] Revert "Make publish_dirs lowercase" This reverts commit c46e2194e91f1878b3a714550eec6bf8b7d5c40e. --- nf_core/module-template/modules/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/fastqc/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/multiqc/main.nf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 4047c0bf04..16d35aa675 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -26,7 +26,7 @@ process {{ tool_name_underscore|upper }} { label '{{ process_label }}' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process).toLowerCase(), meta:{{ 'meta' if has_meta else "[:]" }}, publish_by_meta:{{ "['id']" if has_meta else "[]" }}) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:{{ 'meta' if has_meta else "[:]" }}, publish_by_meta:{{ "['id']" if has_meta else "[]" }}) } // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 35ed481c9c..a725ecce8d 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -9,7 +9,7 @@ process FASTQC { label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process).toLowerCase(), meta:meta, publish_by_meta:['id']) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:meta, publish_by_meta:['id']) } conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index 2fc36ad1f3..c97ed17c10 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -8,7 +8,7 @@ process MULTIQC { label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process).toLowerCase(), meta:[:], publish_by_meta:[]) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:[:], publish_by_meta:[]) } conda (params.enable_conda ? "bioconda::multiqc=1.10.1" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { From bd394ba3a7f6e6e5f3f99e252c506d83ce92fe82 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 Aug 2021 09:17:28 +0200 Subject: [PATCH 33/44] Revert "Update lint check for software" This reverts commit a98588b0a25eb98fb76cfdccfedf25699873963b. --- nf_core/modules/lint/main_nf.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index ac29e9ff26..018dc99af2 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -116,10 +116,16 @@ def main_nf(module_lint_object, module): def check_script_section(self, lines): """ Lint the script section - Checks whether 'def prefix' is defined + Checks whether 'def sotware' and 'def prefix' are defined """ script = "".join(lines) + # check for software + if re.search("\s*def\s*software\s*=\s*getSoftwareName", script): + self.passed.append(("main_nf_version_script", "Software version specified in script section", self.main_nf)) + else: + self.warned.append(("main_nf_version_script", "Software version unspecified in script section", self.main_nf)) + # check for prefix (only if module has a meta map as input) if self.has_meta: if re.search("\s*prefix\s*=\s*options.suffix", script): From 610585cbcdc47390744224e525298be983b96a7f Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 Aug 2021 09:17:44 +0200 Subject: [PATCH 34/44] Revert "Rename publish_dir directory" This reverts commit a45c66d3b3a9dbdbbab26d07d904bb9e1e55c5e1. --- nf_core/module-template/modules/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/fastqc/main.nf | 2 +- .../pipeline-template/modules/nf-core/modules/multiqc/main.nf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 16d35aa675..b91d44307b 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -26,7 +26,7 @@ process {{ tool_name_underscore|upper }} { label '{{ process_label }}' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:{{ 'meta' if has_meta else "[:]" }}, publish_by_meta:{{ "['id']" if has_meta else "[]" }}) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:{{ 'meta' if has_meta else "[:]" }}, publish_by_meta:{{ "['id']" if has_meta else "[]" }}) } // TODO nf-core: List required Conda package(s). // Software MUST be pinned to channel (i.e. "bioconda"), version (i.e. "1.10"). diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index a725ecce8d..26eb12cdad 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -9,7 +9,7 @@ process FASTQC { label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:meta, publish_by_meta:['id']) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:meta, publish_by_meta:['id']) } conda (params.enable_conda ? "bioconda::fastqc=0.11.9" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index c97ed17c10..65b1e64bb7 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -8,7 +8,7 @@ process MULTIQC { label 'process_medium' publishDir "${params.outdir}", mode: params.publish_dir_mode, - saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getProcessName(task.process), meta:[:], publish_by_meta:[]) } + saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:getSoftwareName(task.process), meta:[:], publish_by_meta:[]) } conda (params.enable_conda ? "bioconda::multiqc=1.10.1" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { From 21983765f28e432273285eaa8f013dc1ac9dc7df Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 Aug 2021 09:18:01 +0200 Subject: [PATCH 35/44] Revert "remove software var" This reverts commit 1ee14e9a98608a87391d6b0739950fc85efcff4b. --- .../pipeline-template/modules/nf-core/modules/fastqc/main.nf | 5 +++-- .../modules/nf-core/modules/multiqc/main.nf | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 26eb12cdad..8becdcb48d 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -28,6 +28,7 @@ process FASTQC { script: // Add soft-links to original FastQs for consistent naming in pipeline + def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" if (meta.single_end) { """ @@ -36,7 +37,7 @@ process FASTQC { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - fastqc: \$(fastqc --version | sed -e "s/FastQC v//g") + $software: \$(fastqc --version | sed -e "s/FastQC v//g") END_VERSIONS """ } else { @@ -47,7 +48,7 @@ process FASTQC { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - fastqc: \$(fastqc --version | sed -e "s/FastQC v//g") + $software: \$(fastqc --version | sed -e "s/FastQC v//g") END_VERSIONS """ } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index 65b1e64bb7..a9ba0fe3a2 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -27,12 +27,13 @@ process MULTIQC { path "versions.yml" , emit: version script: + def software = getSoftwareName(task.process) """ multiqc -f $options.args . cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - multiqc: \$(multiqc --version | sed -e "s/multiqc, version //g") + $software: \$(multiqc --version | sed -e "s/multiqc, version //g") END_VERSIONS """ } From 42404591f04cc614a70b225aacbb9c6828c64540 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 Aug 2021 09:18:12 +0200 Subject: [PATCH 36/44] Revert "Remove getSoftwareName function" This reverts commit 70d14aaf950c0e825310e1229017b01ccacc6662. --- nf_core/module-template/modules/functions.nf | 7 +++++++ nf_core/module-template/modules/main.nf | 3 ++- nf_core/modules/lint/functions_nf.py | 2 +- nf_core/pipeline-template/modules/local/functions.nf | 7 +++++++ .../modules/nf-core/modules/fastqc/functions.nf | 7 +++++++ .../modules/nf-core/modules/fastqc/main.nf | 2 +- .../modules/nf-core/modules/multiqc/functions.nf | 7 +++++++ .../modules/nf-core/modules/multiqc/main.nf | 2 +- 8 files changed, 33 insertions(+), 4 deletions(-) diff --git a/nf_core/module-template/modules/functions.nf b/nf_core/module-template/modules/functions.nf index 8171d73fa4..9fbf2c4992 100644 --- a/nf_core/module-template/modules/functions.nf +++ b/nf_core/module-template/modules/functions.nf @@ -2,6 +2,13 @@ // Utility functions used in nf-core DSL2 module files // +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + // // Extract name of module from process name using $task.process // diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index b91d44307b..460f5f1b76 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getProcessName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/software @@ -55,6 +55,7 @@ process {{ tool_name_underscore|upper }} { path "versions.yml" , emit: version script: + def software = getSoftwareName(task.process) {% if has_meta -%} def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" {%- endif %} diff --git a/nf_core/modules/lint/functions_nf.py b/nf_core/modules/lint/functions_nf.py index f03f0c39a3..aef0d115ea 100644 --- a/nf_core/modules/lint/functions_nf.py +++ b/nf_core/modules/lint/functions_nf.py @@ -22,7 +22,7 @@ def functions_nf(module_lint_object, module): return # Test whether all required functions are present - required_functions = ["getProcessName", "initOptions", "getPathFromList", "saveFiles"] + required_functions = ["getSoftwareName", "getProcessName", "initOptions", "getPathFromList", "saveFiles"] lines = "\n".join(lines) contains_all_functions = True for f in required_functions: diff --git a/nf_core/pipeline-template/modules/local/functions.nf b/nf_core/pipeline-template/modules/local/functions.nf index 8171d73fa4..9fbf2c4992 100644 --- a/nf_core/pipeline-template/modules/local/functions.nf +++ b/nf_core/pipeline-template/modules/local/functions.nf @@ -2,6 +2,13 @@ // Utility functions used in nf-core DSL2 module files // +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + // // Extract name of module from process name using $task.process // diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf index 8171d73fa4..9fbf2c4992 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf @@ -2,6 +2,13 @@ // Utility functions used in nf-core DSL2 module files // +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + // // Extract name of module from process name using $task.process // diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 8becdcb48d..97183b62c9 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getProcessName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf index 8171d73fa4..9fbf2c4992 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf @@ -2,6 +2,13 @@ // Utility functions used in nf-core DSL2 module files // +// +// Extract name of software tool from process name using $task.process +// +def getSoftwareName(task_process) { + return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() +} + // // Extract name of module from process name using $task.process // diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index a9ba0fe3a2..77c8f4361a 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getProcessName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' params.options = [:] options = initOptions(params.options) From 3bf1b7f76230807c3291a9f40f221e7061d10971 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 Aug 2021 09:25:44 +0200 Subject: [PATCH 37/44] Remove def software line --- nf_core/module-template/modules/main.nf | 1 - nf_core/modules/lint/main_nf.py | 6 ------ .../modules/nf-core/modules/fastqc/main.nf | 1 - .../modules/nf-core/modules/multiqc/main.nf | 1 - 4 files changed, 9 deletions(-) diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 460f5f1b76..457f2b39bc 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -55,7 +55,6 @@ process {{ tool_name_underscore|upper }} { path "versions.yml" , emit: version script: - def software = getSoftwareName(task.process) {% if has_meta -%} def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" {%- endif %} diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 018dc99af2..0393d352eb 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -120,12 +120,6 @@ def check_script_section(self, lines): """ script = "".join(lines) - # check for software - if re.search("\s*def\s*software\s*=\s*getSoftwareName", script): - self.passed.append(("main_nf_version_script", "Software version specified in script section", self.main_nf)) - else: - self.warned.append(("main_nf_version_script", "Software version unspecified in script section", self.main_nf)) - # check for prefix (only if module has a meta map as input) if self.has_meta: if re.search("\s*prefix\s*=\s*options.suffix", script): diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 97183b62c9..07585128ec 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -28,7 +28,6 @@ process FASTQC { script: // Add soft-links to original FastQs for consistent naming in pipeline - def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" if (meta.single_end) { """ diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index 77c8f4361a..a92cca59b0 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -27,7 +27,6 @@ process MULTIQC { path "versions.yml" , emit: version script: - def software = getSoftwareName(task.process) """ multiqc -f $options.args . From d965d62d5fb894a8bd12460b19dc7a1f2722872d Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Fri, 13 Aug 2021 09:30:14 +0200 Subject: [PATCH 38/44] Remove reference to $software --- .../pipeline-template/modules/nf-core/modules/fastqc/main.nf | 4 ++-- .../pipeline-template/modules/nf-core/modules/multiqc/main.nf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 07585128ec..8173cd8962 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -36,7 +36,7 @@ process FASTQC { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - $software: \$(fastqc --version | sed -e "s/FastQC v//g") + fastqc: \$(fastqc --version | sed -e "s/FastQC v//g") END_VERSIONS """ } else { @@ -47,7 +47,7 @@ process FASTQC { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - $software: \$(fastqc --version | sed -e "s/FastQC v//g") + fastqc: \$(fastqc --version | sed -e "s/FastQC v//g") END_VERSIONS """ } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index a92cca59b0..d1e26b6fbe 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -32,7 +32,7 @@ process MULTIQC { cat <<-END_VERSIONS > versions.yml ${getProcessName(task.process)}: - $software: \$(multiqc --version | sed -e "s/multiqc, version //g") + multiqc: \$(multiqc --version | sed -e "s/multiqc, version //g") END_VERSIONS """ } From 8635612b3ee3b1e414414553c4fd72cb048464d3 Mon Sep 17 00:00:00 2001 From: Mahesh Binzer-Panchal Date: Tue, 14 Sep 2021 15:06:48 +0200 Subject: [PATCH 39/44] Update output docs (software_versions.tsv ->software_versions.yml) --- nf_core/pipeline-template/docs/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index 9646e12290..9fd9e5e127 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -60,7 +60,7 @@ Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQ * `pipeline_info/` * Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. - * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.tsv`. + * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. * Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. From 2af8709fc757c65e2191db0151e48f574d273386 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Thu, 23 Sep 2021 13:17:42 +0200 Subject: [PATCH 40/44] Do not update nf-core modules in pipeline template yet --- .../modules/nf-core/modules/fastqc/functions.nf | 9 +-------- .../modules/nf-core/modules/fastqc/main.nf | 17 +++++------------ .../modules/nf-core/modules/fastqc/meta.yml | 2 +- .../nf-core/modules/multiqc/functions.nf | 9 +-------- .../modules/nf-core/modules/multiqc/main.nf | 11 ++++------- .../modules/nf-core/modules/multiqc/meta.yml | 2 +- 6 files changed, 13 insertions(+), 37 deletions(-) diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf index 9fbf2c4992..da9da093d3 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/functions.nf @@ -9,13 +9,6 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } -// -// Extract name of module from process name using $task.process -// -def getProcessName(task_process) { - return task_process.tokenize(':')[-1] -} - // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -44,7 +37,7 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.equals('versions.yml')) { + if (!args.filename.endsWith('.version.txt')) { def ioptions = initOptions(args.options) def path_list = [ ioptions.publish_dir ?: args.publish_dir ] if (ioptions.publish_by_meta) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf index 8173cd8962..39c327b261 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' +include { initOptions; saveFiles; getSoftwareName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,31 +24,24 @@ process FASTQC { output: tuple val(meta), path("*.html"), emit: html tuple val(meta), path("*.zip") , emit: zip - path "versions.yml" , emit: version + path "*.version.txt" , emit: version script: // Add soft-links to original FastQs for consistent naming in pipeline + def software = getSoftwareName(task.process) def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" if (meta.single_end) { """ [ ! -f ${prefix}.fastq.gz ] && ln -s $reads ${prefix}.fastq.gz fastqc $options.args --threads $task.cpus ${prefix}.fastq.gz - - cat <<-END_VERSIONS > versions.yml - ${getProcessName(task.process)}: - fastqc: \$(fastqc --version | sed -e "s/FastQC v//g") - END_VERSIONS + fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt """ } else { """ [ ! -f ${prefix}_1.fastq.gz ] && ln -s ${reads[0]} ${prefix}_1.fastq.gz [ ! -f ${prefix}_2.fastq.gz ] && ln -s ${reads[1]} ${prefix}_2.fastq.gz fastqc $options.args --threads $task.cpus ${prefix}_1.fastq.gz ${prefix}_2.fastq.gz - - cat <<-END_VERSIONS > versions.yml - ${getProcessName(task.process)}: - fastqc: \$(fastqc --version | sed -e "s/FastQC v//g") - END_VERSIONS + fastqc --version | sed -e "s/FastQC v//g" > ${software}.version.txt """ } } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml index 48031356b5..8eb9953dce 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/modules/fastqc/meta.yml @@ -43,7 +43,7 @@ output: - version: type: file description: File containing software version - pattern: "versions.yml" + pattern: "*.{version.txt}" authors: - "@drpatelh" - "@grst" diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf index 9fbf2c4992..da9da093d3 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/functions.nf @@ -9,13 +9,6 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } -// -// Extract name of module from process name using $task.process -// -def getProcessName(task_process) { - return task_process.tokenize(':')[-1] -} - // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -44,7 +37,7 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.equals('versions.yml')) { + if (!args.filename.endsWith('.version.txt')) { def ioptions = initOptions(args.options) def path_list = [ ioptions.publish_dir ?: args.publish_dir ] if (ioptions.publish_by_meta) { diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf index d1e26b6fbe..da78080024 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' +include { initOptions; saveFiles; getSoftwareName } from './functions' params.options = [:] options = initOptions(params.options) @@ -24,15 +24,12 @@ process MULTIQC { path "*multiqc_report.html", emit: report path "*_data" , emit: data path "*_plots" , optional:true, emit: plots - path "versions.yml" , emit: version + path "*.version.txt" , emit: version script: + def software = getSoftwareName(task.process) """ multiqc -f $options.args . - - cat <<-END_VERSIONS > versions.yml - ${getProcessName(task.process)}: - multiqc: \$(multiqc --version | sed -e "s/multiqc, version //g") - END_VERSIONS + multiqc --version | sed -e "s/multiqc, version //g" > ${software}.version.txt """ } diff --git a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml index 2d99ec0d12..532a8bb1ef 100644 --- a/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml +++ b/nf_core/pipeline-template/modules/nf-core/modules/multiqc/meta.yml @@ -32,7 +32,7 @@ output: - version: type: file description: File containing software version - pattern: "versions.yml" + pattern: "*.{version.txt}" authors: - "@abhi18av" - "@bunop" From 6c52385764a2c8a736f7199919b5979041fc8530 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Thu, 23 Sep 2021 13:19:57 +0200 Subject: [PATCH 41/44] Update functions.nf in module template and in local pipeline modules --- nf_core/module-template/modules/functions.nf | 47 ++++++++++--------- .../modules/local/functions.nf | 47 ++++++++++--------- 2 files changed, 50 insertions(+), 44 deletions(-) diff --git a/nf_core/module-template/modules/functions.nf b/nf_core/module-template/modules/functions.nf index 9fbf2c4992..85628ee0eb 100644 --- a/nf_core/module-template/modules/functions.nf +++ b/nf_core/module-template/modules/functions.nf @@ -44,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.equals('versions.yml')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/nf_core/pipeline-template/modules/local/functions.nf b/nf_core/pipeline-template/modules/local/functions.nf index 9fbf2c4992..85628ee0eb 100644 --- a/nf_core/pipeline-template/modules/local/functions.nf +++ b/nf_core/pipeline-template/modules/local/functions.nf @@ -44,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.equals('versions.yml')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } From da8dadba76bd02343b16d9b89561b92e78a0b9c8 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Thu, 23 Sep 2021 13:31:58 +0200 Subject: [PATCH 42/44] Show linting messages when linting tests failed --- tests/modules/lint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index de29371c58..1cf99c07b9 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -8,7 +8,7 @@ def test_modules_lint_trimgalore(self): module_lint.lint(print_results=False, module="trimgalore") assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 0, module_lint.failed def test_modules_lint_empty(self): @@ -19,7 +19,7 @@ def test_modules_lint_empty(self): module_lint.lint(print_results=False, all_modules=True) assert len(module_lint.passed) == 0 assert len(module_lint.warned) == 0 - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 0, module_lint.failed def test_modules_lint_new_modules(self): @@ -28,4 +28,4 @@ def test_modules_lint_new_modules(self): module_lint.lint(print_results=True, all_modules=True) assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 0, module_lint.failed From 9a0dfd9e0e83dd588cbfe6f09b618c96f620b7d3 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Thu, 23 Sep 2021 13:41:06 +0200 Subject: [PATCH 43/44] Print linting error message --- tests/modules/lint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 1cf99c07b9..8371b92fb7 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -8,7 +8,7 @@ def test_modules_lint_trimgalore(self): module_lint.lint(print_results=False, module="trimgalore") assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - assert len(module_lint.failed) == 0, module_lint.failed + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" def test_modules_lint_empty(self): @@ -19,7 +19,7 @@ def test_modules_lint_empty(self): module_lint.lint(print_results=False, all_modules=True) assert len(module_lint.passed) == 0 assert len(module_lint.warned) == 0 - assert len(module_lint.failed) == 0, module_lint.failed + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" def test_modules_lint_new_modules(self): @@ -28,4 +28,4 @@ def test_modules_lint_new_modules(self): module_lint.lint(print_results=True, all_modules=True) assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - assert len(module_lint.failed) == 0, module_lint.failed + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" From 82ce4b4679f026c802cf41f2e78a3e0a87d271c6 Mon Sep 17 00:00:00 2001 From: Gregor Sturm Date: Thu, 23 Sep 2021 15:39:31 +0200 Subject: [PATCH 44/44] Update nf_core/pipeline-template/docs/output.md Co-authored-by: Harshil Patel --- nf_core/pipeline-template/docs/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index 9fd9e5e127..4ef9a4ea01 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -60,7 +60,7 @@ Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQ * `pipeline_info/` * Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. - * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. + * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. * Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`.