Support multiple file versions

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 ### Template
 
+* Modify software version channel handling to support multiple software version emissions (e.g. from mulled containers), and multiple software versions.
+
 ### General
 
 * Changed `questionary` `ask()` to `unsafe_ask()` to not catch `KeyboardInterupts` ([#1237](https://github.com/nf-core/tools/issues/1237))

nf_core/lint/files_exist.py

@@ -36,7 +36,6 @@ def files_exist(self):
         assets/email_template.txt
         assets/nf-core-PIPELINE_logo.png
         assets/sendmail_template.txt
-        bin/scrape_software_versions.py
         conf/modules.config
         conf/test.config
         conf/test_full.config
@@ -121,7 +120,6 @@ def files_exist(self):
         [os.path.join("assets", "email_template.txt")],
         [os.path.join("assets", "sendmail_template.txt")],
         [os.path.join("assets", f"nf-core-{short_name}_logo.png")],
-        [os.path.join("bin", "scrape_software_versions.py")],
         [os.path.join("conf", "modules.config")],
         [os.path.join("conf", "test.config")],
         [os.path.join("conf", "test_full.config")],

nf_core/lint/files_unchanged.py

@@ -92,7 +92,6 @@ def files_unchanged(self):
         [os.path.join("assets", "email_template.txt")],
         [os.path.join("assets", "sendmail_template.txt")],
         [os.path.join("assets", f"nf-core-{short_name}_logo.png")],
-        [os.path.join("bin", "scrape_software_versions.py")],
         [os.path.join("docs", "images", f"nf-core-{short_name}_logo.png")],
         [os.path.join("docs", "README.md")],
         [os.path.join("lib", "nfcore_external_java_deps.jar")],

nf_core/module-template/modules/functions.nf

@@ -9,6 +9,13 @@ def getSoftwareName(task_process) {
     return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()
 }
 
+//
+// Extract name of module from process name using $task.process
+//
+def getProcessName(task_process) {
+    return task_process.tokenize(':')[-1]
+}
+
 //
 // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules
 //
@@ -37,32 +44,35 @@ def getPathFromList(path_list) {
 // Function to save/publish module results
 //
 def saveFiles(Map args) {
-    if (!args.filename.endsWith('.version.txt')) {
-        def ioptions  = initOptions(args.options)
-        def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
-        if (ioptions.publish_by_meta) {
-            def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
-            for (key in key_list) {
-                if (args.meta && key instanceof String) {
-                    def path = key
-                    if (args.meta.containsKey(key)) {
-                        path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
-                    }
-                    path = path instanceof String ? path : ''
-                    path_list.add(path)
+    def ioptions  = initOptions(args.options)
+    def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
+
+    // Do not publish versions.yml unless running from pytest workflow
+    if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) {
+        return null
+    }
+    if (ioptions.publish_by_meta) {
+        def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
+        for (key in key_list) {
+            if (args.meta && key instanceof String) {
+                def path = key
+                if (args.meta.containsKey(key)) {
+                    path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
                 }
+                path = path instanceof String ? path : ''
+                path_list.add(path)
             }
         }
-        if (ioptions.publish_files instanceof Map) {
-            for (ext in ioptions.publish_files) {
-                if (args.filename.endsWith(ext.key)) {
-                    def ext_list = path_list.collect()
-                    ext_list.add(ext.value)
-                    return "${getPathFromList(ext_list)}/$args.filename"
-                }
+    }
+    if (ioptions.publish_files instanceof Map) {
+        for (ext in ioptions.publish_files) {
+            if (args.filename.endsWith(ext.key)) {
+                def ext_list = path_list.collect()
+                ext_list.add(ext.value)
+                return "${getPathFromList(ext_list)}/$args.filename"
             }
-        } else if (ioptions.publish_files == null) {
-            return "${getPathFromList(path_list)}/$args.filename"
         }
+    } else if (ioptions.publish_files == null) {
+        return "${getPathFromList(path_list)}/$args.filename"
     }
 }

nf_core/module-template/modules/main.nf

@@ -1,5 +1,5 @@
 // Import generic module functions
-include { initOptions; saveFiles; getSoftwareName } from './functions'
+include { initOptions; saveFiles; getSoftwareName; getProcessName } from './functions'
 
 // TODO nf-core: If in doubt look at other nf-core/modules to see how we are doing things! :)
 //               https://github.com/nf-core/modules/tree/master/software
@@ -52,16 +52,16 @@ process {{ tool_name_underscore|upper }} {
     // TODO nf-core: Named file extensions MUST be emitted for ALL output channels
     {{ 'tuple val(meta), path("*.bam")' if has_meta else 'path "*.bam"' }}, emit: bam
     // TODO nf-core: List additional required output channels/values here
-    path "*.version.txt"          , emit: version
+    path "versions.yml"          , emit: version
 
     script:
-    def software = getSoftwareName(task.process)
     {% if has_meta -%}
     def prefix   = options.suffix ? "${meta.id}${options.suffix}" : "${meta.id}"
     {%- endif %}
     // TODO nf-core: Where possible, a command MUST be provided to obtain the version number of the software e.g. 1.10
     //               If the software is unable to output a version number on the command-line then it can be manually specified
     //               e.g. https://github.com/nf-core/modules/blob/master/software/homer/annotatepeaks/main.nf
+    //               Each software used MUST provide the software name and version number in the YAML version file (versions.yml)
     // TODO nf-core: It MUST be possible to pass additional parameters to the tool as a command-line string via the "$options.args" variable
     // TODO nf-core: If the tool supports multi-threading then you MUST provide the appropriate parameter
     //               using the Nextflow "task" variable e.g. "--threads $task.cpus"
@@ -78,6 +78,9 @@ process {{ tool_name_underscore|upper }} {
         {%- endif %}
         $bam
 
-    echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//' > ${software}.version.txt
+    cat <<-END_VERSIONS > versions.yml
+    ${getProcessName(task.process)}:
+        samtools: \$( samtools --version 2>&1 | sed 's/^.*samtools //; s/Using.*\$//' )
+    END_VERSIONS
     """
 }

nf_core/module-template/modules/meta.yml

@@ -40,7 +40,7 @@ output:
   - version:
       type: file
       description: File containing software version
-      pattern: "*.{version.txt}"
+      pattern: "versions.yml"
   ## TODO nf-core: Delete / customise this example output
   - bam:
       type: file

nf_core/modules/lint/functions_nf.py

@@ -22,7 +22,7 @@ def functions_nf(module_lint_object, module):
         return
 
     # Test whether all required functions are present
-    required_functions = ["getSoftwareName", "initOptions", "getPathFromList", "saveFiles"]
+    required_functions = ["getSoftwareName", "getProcessName", "initOptions", "getPathFromList", "saveFiles"]
     lines = "\n".join(lines)
     contains_all_functions = True
     for f in required_functions:

nf_core/modules/lint/main_nf.py

@@ -120,12 +120,6 @@ def check_script_section(self, lines):
     """
     script = "".join(lines)
 
-    # check for software
-    if re.search("\s*def\s*software\s*=\s*getSoftwareName", script):
-        self.passed.append(("main_nf_version_script", "Software version specified in script section", self.main_nf))
-    else:
-        self.warned.append(("main_nf_version_script", "Software version unspecified in script section", self.main_nf))
-
     # check for prefix (only if module has a meta map as input)
     if self.has_meta:
         if re.search("\s*prefix\s*=\s*options.suffix", script):

nf_core/pipeline-template/docs/output.md

@@ -60,7 +60,7 @@ Results generated by MultiQC collate pipeline QC from supported tools e.g. FastQ
 
 * `pipeline_info/`
     * Reports generated by Nextflow: `execution_report.html`, `execution_timeline.html`, `execution_trace.txt` and `pipeline_dag.dot`/`pipeline_dag.svg`.
-    * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.tsv`.
+    * Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline.
     * Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`.
 
 </details>

nf_core/pipeline-template/modules/local/functions.nf

@@ -9,6 +9,13 @@ def getSoftwareName(task_process) {
     return task_process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()
 }
 
+//
+// Extract name of module from process name using $task.process
+//
+def getProcessName(task_process) {
+    return task_process.tokenize(':')[-1]
+}
+
 //
 // Function to initialise default values and to generate a Groovy Map of available options for nf-core modules
 //
@@ -37,32 +44,35 @@ def getPathFromList(path_list) {
 // Function to save/publish module results
 //
 def saveFiles(Map args) {
-    if (!args.filename.endsWith('.version.txt')) {
-        def ioptions  = initOptions(args.options)
-        def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
-        if (ioptions.publish_by_meta) {
-            def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
-            for (key in key_list) {
-                if (args.meta && key instanceof String) {
-                    def path = key
-                    if (args.meta.containsKey(key)) {
-                        path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
-                    }
-                    path = path instanceof String ? path : ''
-                    path_list.add(path)
+    def ioptions  = initOptions(args.options)
+    def path_list = [ ioptions.publish_dir ?: args.publish_dir ]
+
+    // Do not publish versions.yml unless running from pytest workflow
+    if (args.filename.equals('versions.yml') && !System.getenv("NF_CORE_MODULES_TEST")) {
+        return null
+    }
+    if (ioptions.publish_by_meta) {
+        def key_list = ioptions.publish_by_meta instanceof List ? ioptions.publish_by_meta : args.publish_by_meta
+        for (key in key_list) {
+            if (args.meta && key instanceof String) {
+                def path = key
+                if (args.meta.containsKey(key)) {
+                    path = args.meta[key] instanceof Boolean ? "${key}_${args.meta[key]}".toString() : args.meta[key]
                 }
+                path = path instanceof String ? path : ''
+                path_list.add(path)
             }
         }
-        if (ioptions.publish_files instanceof Map) {
-            for (ext in ioptions.publish_files) {
-                if (args.filename.endsWith(ext.key)) {
-                    def ext_list = path_list.collect()
-                    ext_list.add(ext.value)
-                    return "${getPathFromList(ext_list)}/$args.filename"
-                }
+    }
+    if (ioptions.publish_files instanceof Map) {
+        for (ext in ioptions.publish_files) {
+            if (args.filename.endsWith(ext.key)) {
+                def ext_list = path_list.collect()
+                ext_list.add(ext.value)
+                return "${getPathFromList(ext_list)}/$args.filename"
             }
-        } else if (ioptions.publish_files == null) {
-            return "${getPathFromList(path_list)}/$args.filename"
         }
+    } else if (ioptions.publish_files == null) {
+        return "${getPathFromList(path_list)}/$args.filename"
     }
 }

nf_core/pipeline-template/modules/local/get_software_versions.nf

@@ -8,11 +8,12 @@ process GET_SOFTWARE_VERSIONS {
         mode: params.publish_dir_mode,
         saveAs: { filename -> saveFiles(filename:filename, options:params.options, publish_dir:'pipeline_info', meta:[:], publish_by_meta:[]) }
 
-    conda (params.enable_conda ? "conda-forge::python=3.8.3" : null)
+    // This module only requires the PyYAML library, but rather than create a new container on biocontainers we reuse the multiqc container.
+    conda (params.enable_conda ? "bioconda::multiqc=1.10.1" : null)
     if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) {
-        container "https://depot.galaxyproject.org/singularity/python:3.8.3"
+        container "https://depot.galaxyproject.org/singularity/multiqc:1.10.1--pyhdfd78af_1"
     } else {
-        container "quay.io/biocontainers/python:3.8.3"
+        container "quay.io/biocontainers/multiqc:1.10.1--pyhdfd78af_1"
     }
 
     cache false
@@ -21,13 +22,74 @@ process GET_SOFTWARE_VERSIONS {
     path versions
 
     output:
-    path "software_versions.tsv"     , emit: tsv
-    path 'software_versions_mqc.yaml', emit: yaml
+    path "software_versions.yml"     , emit: yml
+    path "software_versions_mqc.yml" , emit: mqc_yml
 
-    script: // This script is bundled with the pipeline, in {{ name }}/bin/
+    script:
     """
-    echo $workflow.manifest.version > pipeline.version.txt
-    echo $workflow.nextflow.version > nextflow.version.txt
-    scrape_software_versions.py &> software_versions_mqc.yaml
+    #!/usr/bin/env python
+
+    import yaml
+    from textwrap import dedent
+
+    def _make_versions_html(versions):
+        html = [
+            dedent(
+                '''\\
+                <style>
+                #nf-core-versions tbody:nth-child(even) {
+                    background-color: #f2f2f2;
+                }
+                </style>
+                <table class="table" style="width:100%" id="nf-core-versions">
+                    <thead>
+                        <tr>
+                            <th> Process Name </th>
+                            <th> Software </th>
+                            <th> Version  </th>
+                        </tr>
+                    </thead>
+                '''
+            )
+        ]
+        for process, tmp_versions in sorted(versions.items()):
+            html.append("<tbody>")
+            for i, (tool, version) in enumerate(sorted(tmp_versions.items())):
+                html.append(
+                    dedent(
+                        f'''\\
+                        <tr>
+                            <td><samp>{process if (i == 0) else ''}</samp></td>
+                            <td><samp>{tool}</samp></td>
+                            <td><samp>{version}</samp></td>
+                        </tr>
+                        '''
+                    )
+                )
+            html.append("</tbody>")
+        html.append("</table>")
+        return "\\n".join(html)
+
+    with open("$versions") as f:
+        versions = yaml.safe_load(f)
+
+    versions["Workflow"] = {
+        "Nextflow": "$workflow.nextflow.version",
+        "$workflow.manifest.name": "$workflow.manifest.version"
+    }
+
+    versions_mqc = {
+        'id': 'software_versions',
+        'section_name': '${workflow.manifest.name} Software Versions',
+        'section_href': 'https://github.com/${workflow.manifest.name}',
+        'plot_type': 'html',
+        'description': 'are collected at run time from the software output.',
+        'data': _make_versions_html(versions)
+    }
+
+    with open("software_versions.yml", 'w') as f:
+        yaml.dump(versions, f, default_flow_style=False)
+    with open("software_versions_mqc.yml", 'w') as f:
+        yaml.dump(versions_mqc, f, default_flow_style=False)
     """
 }