diff --git a/CHANGELOG.md b/CHANGELOG.md index fe87bf81dc..dc696843c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +* Modify software version channel handling to support multiple software version emissions (e.g. from mulled containers), and multiple software versions. + ### General * Changed `questionary` `ask()` to `unsafe_ask()` to not catch `KeyboardInterupts` ([#1237](https://github.com/nf-core/tools/issues/1237)) diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 7b97aa3fb5..e1b28bc9a3 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -36,7 +36,6 @@ def files_exist(self): assets/email_template.txt assets/nf-core-PIPELINE_logo.png assets/sendmail_template.txt - bin/scrape_software_versions.py conf/modules.config conf/test.config conf/test_full.config @@ -121,7 +120,6 @@ def files_exist(self): [os.path.join("assets", "email_template.txt")], [os.path.join("assets", "sendmail_template.txt")], [os.path.join("assets", f"nf-core-{short_name}_logo.png")], - [os.path.join("bin", "scrape_software_versions.py")], [os.path.join("conf", "modules.config")], [os.path.join("conf", "test.config")], [os.path.join("conf", "test_full.config")], diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 37728b8b06..262e8c4449 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -92,7 +92,6 @@ def files_unchanged(self): [os.path.join("assets", "email_template.txt")], [os.path.join("assets", "sendmail_template.txt")], [os.path.join("assets", f"nf-core-{short_name}_logo.png")], - [os.path.join("bin", "scrape_software_versions.py")], [os.path.join("docs", "images", f"nf-core-{short_name}_logo.png")], [os.path.join("docs", "README.md")], [os.path.join("lib", "nfcore_external_java_deps.jar")], diff --git a/nf_core/module-template/modules/functions.nf b/nf_core/module-template/modules/functions.nf index da9da093d3..85628ee0eb 100644 --- a/nf_core/module-template/modules/functions.nf +++ b/nf_core/module-template/modules/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/nf_core/module-template/modules/main.nf b/nf_core/module-template/modules/main.nf index 6e4dcde636..457f2b39bc 100644 --- a/nf_core/module-template/modules/main.nf +++ b/nf_core/module-template/modules/main.nf @@ -1,5 +1,5 @@ // Import generic module functions -include { initOptions; saveFiles; getSoftwareName } from './functions' +include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions' // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :) // https://github.com/nf-core/modules/tree/master/software @@ -52,16 +52,16 @@ process {{ tool_name_underscore|upper }} { // TODO nf-core: Named file extensions MUST be emitted for ALL output channels {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam // TODO nf-core: List additional required output channels/values here - path "*.version.txt" , emit: version + path "versions.yml" , emit: version script: - def software = getSoftwareName(task.process) {% if has_meta -%} def prefix = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}" {%- endif %} // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10 // If the software is unable to output a version number on the command-line then it can be manually specified // e.g. https://github.com/nf-core/modules/blob/master/software/homer/annotatepeaks/main.nf + // Each software used MUST provide the software name and version number in the YAML version file (versions.yml) // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "$options.args" variable // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter // using the Nextflow "task" variable e.g. "--threads $task.cpus" @@ -78,6 +78,9 @@ process {{ tool_name_underscore|upper }} { {%- endif %} $bam - echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt + cat <<-END_VERSIONS > versions.yml + ${getProcessName(task.process)}: + samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$//' ) + END_VERSIONS """ } diff --git a/nf_core/module-template/modules/meta.yml b/nf_core/module-template/modules/meta.yml index be6d3e5f93..c42dd613bd 100644 --- a/nf_core/module-template/modules/meta.yml +++ b/nf_core/module-template/modules/meta.yml @@ -40,7 +40,7 @@ output: - version: type: file description: File containing software version - pattern: "*.{version.txt}" + pattern: "versions.yml" ## TODO nf-core: Delete / customise this example output - bam: type: file diff --git a/nf_core/modules/lint/functions_nf.py b/nf_core/modules/lint/functions_nf.py index 600a1ae7fd..aef0d115ea 100644 --- a/nf_core/modules/lint/functions_nf.py +++ b/nf_core/modules/lint/functions_nf.py @@ -22,7 +22,7 @@ def functions_nf(module_lint_object, module): return # Test whether all required functions are present - required_functions = ["getSoftwareName", "initOptions", "getPathFromList", "saveFiles"] + required_functions = ["getSoftwareName", "getProcessName", "initOptions", "getPathFromList", "saveFiles"] lines = "\n".join(lines) contains_all_functions = True for f in required_functions: diff --git a/nf_core/modules/lint/main_nf.py b/nf_core/modules/lint/main_nf.py index 018dc99af2..0393d352eb 100644 --- a/nf_core/modules/lint/main_nf.py +++ b/nf_core/modules/lint/main_nf.py @@ -120,12 +120,6 @@ def check_script_section(self, lines): """ script = "".join(lines) - # check for software - if re.search("\s*def\s*software\s*=\s*getSoftwareName", script): - self.passed.append(("main_nf_version_script", "Software version specified in script section", self.main_nf)) - else: - self.warned.append(("main_nf_version_script", "Software version unspecified in script section", self.main_nf)) - # check for prefix (only if module has a meta map as input) if self.has_meta: if re.search("\s*prefix\s*=\s*options.suffix", script): diff --git a/nf_core/pipeline-template/bin/scrape_software_versions.py b/nf_core/pipeline-template/bin/scrape_software_versions.py deleted file mode 100755 index 241dc8b7a6..0000000000 --- a/nf_core/pipeline-template/bin/scrape_software_versions.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function -import os - -results = {} -version_files = [x for x in os.listdir(".") if x.endswith(".version.txt")] -for version_file in version_files: - - software = version_file.replace(".version.txt", "") - if software == "pipeline": - software = "{{ name }}" - - with open(version_file) as fin: - version = fin.read().strip() - results[software] = version - -# Dump to YAML -print( - """ -id: 'software_versions' -section_name: '{{ name }} Software Versions' -section_href: 'https://github.com/{{ name }}' -plot_type: 'html' -description: 'are collected at run time from the software output.' -data: | -
-""" -) -for k, v in sorted(results.items()): - print("
{}
{}
".format(k, v)) -print("
") - -# Write out as tsv file: -with open("software_versions.tsv", "w") as f: - for k, v in sorted(results.items()): - f.write("{}\t{}\n".format(k, v)) diff --git a/nf_core/pipeline-template/docs/output.md b/nf_core/pipeline-template/docs/output.md index 9646e12290..4ef9a4ea01 100644 --- a/nf_core/pipeline-template/docs/output.md +++ b/nf_core/pipeline-template/docs/output.md @@ -60,7 +60,7 @@ Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQ * `pipeline_info/` * Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`. - * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.tsv`. + * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. * Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. diff --git a/nf_core/pipeline-template/modules/local/functions.nf b/nf_core/pipeline-template/modules/local/functions.nf index da9da093d3..85628ee0eb 100644 --- a/nf_core/pipeline-template/modules/local/functions.nf +++ b/nf_core/pipeline-template/modules/local/functions.nf @@ -9,6 +9,13 @@ def getSoftwareName(task_process) { return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase() } +// +// Extract name of module from process name using $task.process +// +def getProcessName(task_process) { + return task_process.tokenize(':')[-1] +} + // // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules // @@ -37,32 +44,35 @@ def getPathFromList(path_list) { // Function to save/publish module results // def saveFiles(Map args) { - if (!args.filename.endsWith('.version.txt')) { - def ioptions = initOptions(args.options) - def path_list = [ ioptions.publish_dir ?: args.publish_dir ] - if (ioptions.publish_by_meta) { - def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta - for (key in key_list) { - if (args.meta && key instanceof String) { - def path = key - if (args.meta.containsKey(key)) { - path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] - } - path = path instanceof String ? path : '' - path_list.add(path) + def ioptions = initOptions(args.options) + def path_list = [ ioptions.publish_dir ?: args.publish_dir ] + + // Do not publish versions.yml unless running from pytest workflow + if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) { + return null + } + if (ioptions.publish_by_meta) { + def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta + for (key in key_list) { + if (args.meta && key instanceof String) { + def path = key + if (args.meta.containsKey(key)) { + path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key] } + path = path instanceof String ? path : '' + path_list.add(path) } } - if (ioptions.publish_files instanceof Map) { - for (ext in ioptions.publish_files) { - if (args.filename.endsWith(ext.key)) { - def ext_list = path_list.collect() - ext_list.add(ext.value) - return "${getPathFromList(ext_list)}/$args.filename" - } + } + if (ioptions.publish_files instanceof Map) { + for (ext in ioptions.publish_files) { + if (args.filename.endsWith(ext.key)) { + def ext_list = path_list.collect() + ext_list.add(ext.value) + return "${getPathFromList(ext_list)}/$args.filename" } - } else if (ioptions.publish_files == null) { - return "${getPathFromList(path_list)}/$args.filename" } + } else if (ioptions.publish_files == null) { + return "${getPathFromList(path_list)}/$args.filename" } } diff --git a/nf_core/pipeline-template/modules/local/get_software_versions.nf b/nf_core/pipeline-template/modules/local/get_software_versions.nf index 8af8af1735..08d58f9c52 100644 --- a/nf_core/pipeline-template/modules/local/get_software_versions.nf +++ b/nf_core/pipeline-template/modules/local/get_software_versions.nf @@ -8,11 +8,12 @@ process GET_SOFTWARE_VERSIONS { mode: params.publish_dir_mode, saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) } - conda (params.enable_conda ? "conda-forge::python=3.8.3" : null) + // This module only requires the PyYAML library, but rather than create a new container on biocontainers we reuse the multiqc container. + conda (params.enable_conda ? "bioconda::multiqc=1.10.1" : null) if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { - container "https://depot.galaxyproject.org/singularity/python:3.8.3" + container "https://depot.galaxyproject.org/singularity/multiqc:1.10.1--pyhdfd78af_1" } else { - container "quay.io/biocontainers/python:3.8.3" + container "quay.io/biocontainers/multiqc:1.10.1--pyhdfd78af_1" } cache false @@ -21,13 +22,74 @@ process GET_SOFTWARE_VERSIONS { path versions output: - path "software_versions.tsv" , emit: tsv - path 'software_versions_mqc.yaml', emit: yaml + path "software_versions.yml" , emit: yml + path "software_versions_mqc.yml" , emit: mqc_yml - script: // This script is bundled with the pipeline, in {{ name }}/bin/ + script: """ - echo $workflow.manifest.version > pipeline.version.txt - echo $workflow.nextflow.version > nextflow.version.txt - scrape_software_versions.py &> software_versions_mqc.yaml + #!/usr/bin/env python + + import yaml + from textwrap import dedent + + def _make_versions_html(versions): + html = [ + dedent( + '''\\ + + + + + + + + + + ''' + ) + ] + for process, tmp_versions in sorted(versions.items()): + html.append("") + for i, (tool, version) in enumerate(sorted(tmp_versions.items())): + html.append( + dedent( + f'''\\ + + + + + + ''' + ) + ) + html.append("") + html.append("
Process Name Software Version
{process if (i == 0) else ''}{tool}{version}
") + return "\\n".join(html) + + with open("$versions") as f: + versions = yaml.safe_load(f) + + versions["Workflow"] = { + "Nextflow": "$workflow.nextflow.version", + "$workflow.manifest.name": "$workflow.manifest.version" + } + + versions_mqc = { + 'id': 'software_versions', + 'section_name': '${workflow.manifest.name} Software Versions', + 'section_href': 'https://github.com/${workflow.manifest.name}', + 'plot_type': 'html', + 'description': 'are collected at run time from the software output.', + 'data': _make_versions_html(versions) + } + + with open("software_versions.yml", 'w') as f: + yaml.dump(versions, f, default_flow_style=False) + with open("software_versions_mqc.yml", 'w') as f: + yaml.dump(versions_mqc, f, default_flow_style=False) """ } diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index fe1882b420..44924b8116 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -38,7 +38,7 @@ def modules = params.modules.clone() // // MODULE: Local to the pipeline // -include { GET_SOFTWARE_VERSIONS } from '../modules/local/get_software_versions' addParams( options: [publish_files : ['tsv':'']] ) +include { GET_SOFTWARE_VERSIONS } from '../modules/local/get_software_versions' addParams( options: [publish_files : ['yml':'']] ) // // SUBWORKFLOW: Consisting of a mix of local and nf-core/modules @@ -88,19 +88,8 @@ workflow {{ short_name|upper }} { ) ch_software_versions = ch_software_versions.mix(FASTQC.out.version.first().ifEmpty(null)) - // - // MODULE: Pipeline reporting - // - ch_software_versions - .map { it -> if (it) [ it.baseName, it ] } - .groupTuple() - .map { it[1][0] } - .flatten() - .collect() - .set { ch_software_versions } - GET_SOFTWARE_VERSIONS ( - ch_software_versions.map { it }.collect() + ch_software_versions.collectFile() ) // @@ -113,7 +102,7 @@ workflow {{ short_name|upper }} { ch_multiqc_files = ch_multiqc_files.mix(Channel.from(ch_multiqc_config)) ch_multiqc_files = ch_multiqc_files.mix(ch_multiqc_custom_config.collect().ifEmpty([])) ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(GET_SOFTWARE_VERSIONS.out.yaml.collect()) + ch_multiqc_files = ch_multiqc_files.mix(GET_SOFTWARE_VERSIONS.out.mqc_yml.collect()) ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}.ifEmpty([])) MULTIQC ( diff --git a/tests/modules/lint.py b/tests/modules/lint.py index de29371c58..8371b92fb7 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -8,7 +8,7 @@ def test_modules_lint_trimgalore(self): module_lint.lint(print_results=False, module="trimgalore") assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" def test_modules_lint_empty(self): @@ -19,7 +19,7 @@ def test_modules_lint_empty(self): module_lint.lint(print_results=False, all_modules=True) assert len(module_lint.passed) == 0 assert len(module_lint.warned) == 0 - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" def test_modules_lint_new_modules(self): @@ -28,4 +28,4 @@ def test_modules_lint_new_modules(self): module_lint.lint(print_results=True, all_modules=True) assert len(module_lint.passed) > 0 assert len(module_lint.warned) >= 0 - assert len(module_lint.failed) == 0 + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}"